From 3eb7ad8b0a1ac0f6f9cda679108937a01268f184 Mon Sep 17 00:00:00 2001 From: Tobias Fella Date: Mon, 16 May 2022 20:46:34 +0200 Subject: Save and load outgoing megolm session --- lib/database.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib/database.h') diff --git a/lib/database.h b/lib/database.h index 08fe49f3..751ebd1d 100644 --- a/lib/database.h +++ b/lib/database.h @@ -8,6 +8,7 @@ #include #include "e2ee/e2ee.h" + namespace Quotient { class QUOTIENT_API Database : public QObject { @@ -34,6 +35,8 @@ public: std::pair groupSessionIndexRecord(const QString& roomId, const QString& sessionId, qint64 index); void clearRoomData(const QString& roomId); void setOlmSessionLastReceived(const QString& sessionId, const QDateTime& timestamp); + QOlmOutboundGroupSessionPtr loadCurrentOutboundMegolmSession(const QString& roomId, const PicklingMode& picklingMode); + void saveCurrentOutboundMegolmSession(const QString& roomId, const PicklingMode& picklingMode, const QOlmOutboundGroupSessionPtr& data); private: void migrateTo1(); -- cgit v1.2.3 From 6f5ac9b7315d75692960e5eac7b1eb6867c0d203 Mon Sep 17 00:00:00 2001 From: Tobias Fella Date: Sun, 6 Mar 2022 22:54:01 +0100 Subject: Keep log of where we send keys and send keys to new devices and users --- lib/connection.cpp | 1 + lib/database.cpp | 44 ++++++++++++++++++++++++++++++++++++++++++++ lib/database.h | 9 +++++++++ lib/room.cpp | 53 ++++++++++++++++++++++++++++++++++------------------- 4 files changed, 88 insertions(+), 19 deletions(-) (limited to 'lib/database.h') diff --git a/lib/connection.cpp b/lib/connection.cpp index b11ec731..2a1b39f9 100644 --- a/lib/connection.cpp +++ b/lib/connection.cpp @@ -2247,6 +2247,7 @@ bool Connection::hasOlmSession(User* user, const QString& deviceId) const QPair Connection::olmEncryptMessage(User* user, const QString& device, const QByteArray& message) { //TODO be smarter about choosing a session; see e2ee impl guide + //TODO do we need to save the olm session after sending a message? const auto& curveKey = curveKeyForUserDevice(user->id(), device); QOlmMessage::Type type = d->olmSessions[curveKey][0]->encryptMessageType(); auto result = d->olmSessions[curveKey][0]->encrypt(message); diff --git a/lib/database.cpp b/lib/database.cpp index 8cb3a9d1..4a28fd4c 100644 --- a/lib/database.cpp +++ b/lib/database.cpp @@ -13,6 +13,9 @@ #include "e2ee/e2ee.h" #include "e2ee/qolmsession.h" #include "e2ee/qolminboundsession.h" +#include "connection.h" +#include "user.h" +#include "room.h" using namespace Quotient; Database::Database(const QString& matrixId, const QString& deviceId, QObject* parent) @@ -91,6 +94,7 @@ void Database::migrateTo1() execute(QStringLiteral("CREATE TABLE tracked_users (matrixId TEXT);")); execute(QStringLiteral("CREATE TABLE outdated_users (matrixId TEXT);")); execute(QStringLiteral("CREATE TABLE tracked_devices (matrixId TEXT, deviceId TEXT, curveKeyId TEXT, curveKey TEXT, edKeyId TEXT, edKey TEXT);")); + execute(QStringLiteral("CREATE TABLE sent_megolm_sessions (roomId TEXT, userId TEXT, deviceId TEXT, identityKey TEXT, sessionId TEXT, i INTEGER);")); execute(QStringLiteral("PRAGMA user_version = 1;")); commit(); @@ -331,3 +335,43 @@ QOlmOutboundGroupSessionPtr Database::loadCurrentOutboundMegolmSession(const QSt } return nullptr; } + +void Database::setDevicesReceivedKey(const QString& roomId, QHash devices, const QString& sessionId, int index) +{ + //TODO this better + auto connection = dynamic_cast(parent()); + transaction(); + for (const auto& user : devices.keys()) { + for (const auto& device : devices[user]) { + auto query = prepareQuery(QStringLiteral("INSERT INTO sent_megolm_sessions(roomId, userId, deviceId, identityKey, sessionId, i) VALUES(:roomId, :userId, :deviceId, :identityKey, :sessionId, :i);")); + query.bindValue(":roomId", roomId); + query.bindValue(":userId", user->id()); + query.bindValue(":deviceId", device); + query.bindValue(":identityKey", connection->curveKeyForUserDevice(user->id(), device)); + query.bindValue(":sessionId", sessionId); + query.bindValue(":i", index); + execute(query); + } + } + commit(); +} + +QHash Database::devicesWithoutKey(Room* room, const QString &sessionId) +{ + auto connection = dynamic_cast(parent()); + QHash devices; + for (const auto& user : room->users()) {//TODO does this include invited & left? + devices[user->id()] = connection->devicesForUser(user); + } + + auto query = prepareQuery(QStringLiteral("SELECT userId, deviceId FROM sent_megolm_sessions WHERE roomId=:roomId AND sessionId=:sessionId")); + query.bindValue(":roomId", room->id()); + query.bindValue(":sessionId", sessionId); + transaction(); + execute(query); + commit(); + while (query.next()) { + devices[query.value("userId").toString()].removeAll(query.value("deviceId").toString()); + } + return devices; +} diff --git a/lib/database.h b/lib/database.h index 751ebd1d..30f2f203 100644 --- a/lib/database.h +++ b/lib/database.h @@ -7,9 +7,14 @@ #include #include +#include + #include "e2ee/e2ee.h" namespace Quotient { +class User; +class Room; + class QUOTIENT_API Database : public QObject { Q_OBJECT @@ -38,6 +43,10 @@ public: QOlmOutboundGroupSessionPtr loadCurrentOutboundMegolmSession(const QString& roomId, const PicklingMode& picklingMode); void saveCurrentOutboundMegolmSession(const QString& roomId, const PicklingMode& picklingMode, const QOlmOutboundGroupSessionPtr& data); + // Returns a map User -> [Device] that have not received key yet + QHash devicesWithoutKey(Room* room, const QString &sessionId); + void setDevicesReceivedKey(const QString& roomId, QHash devices, const QString& sessionId, int index); + private: void migrateTo1(); void migrateTo2(); diff --git a/lib/room.cpp b/lib/room.cpp index 9e2bd7dd..7db9f8e9 100644 --- a/lib/room.cpp +++ b/lib/room.cpp @@ -447,6 +447,13 @@ public: qCDebug(E2EE) << "Creating new outbound megolm session for room " << q->id(); currentOutboundMegolmSession = QOlmOutboundGroupSession::create(); connection->saveCurrentOutboundMegolmSession(q, currentOutboundMegolmSession); + + const auto sessionKey = currentOutboundMegolmSession->sessionKey(); + if(std::holds_alternative(sessionKey)) { + qCWarning(E2EE) << "Session error"; + //TODO something + } + addInboundGroupSession(q->connection()->olmAccount()->identityKeys().curve25519, currentOutboundMegolmSession->sessionId(), std::get(sessionKey), QString(connection->olmAccount()->identityKeys().ed25519)); } std::unique_ptr payloadForUserDevice(User* user, const QString& device, const QByteArray& sessionId, const QByteArray& sessionKey) @@ -472,13 +479,23 @@ public: return makeEvent(encrypted, connection->olmAccount()->identityKeys().curve25519); } - void sendRoomKeyToDevices(const QByteArray& sessionId, const QByteArray& sessionKey) + QHash getDevicesWithoutKey() const + { + QHash devices; + auto rawDevices = q->connection()->database()->devicesWithoutKey(q, QString(currentOutboundMegolmSession->sessionId())); + for (const auto& user : rawDevices.keys()) { + devices[q->connection()->user(user)] = rawDevices[user]; + } + return devices; + } + + void sendRoomKeyToDevices(const QByteArray& sessionId, const QByteArray& sessionKey, const QHash devices, int index) { qCDebug(E2EE) << "Sending room key to devices" << sessionId, sessionKey.toHex(); QHash> hash; - for (const auto& user : q->users()) { + for (const auto& user : devices.keys()) { QHash u; - for(const auto &device : connection->devicesForUser(user)) { + for(const auto &device : devices[user]) { if (!connection->hasOlmSession(user, device)) { u[device] = "signed_curve25519"_ls; qCDebug(E2EE) << "Adding" << user << device << "to keys to claim"; @@ -489,30 +506,28 @@ public: } } auto job = connection->callApi(hash); - connect(job, &BaseJob::success, q, [job, this, sessionId, sessionKey](){ + connect(job, &BaseJob::success, q, [job, this, sessionId, sessionKey, devices, index](){ Connection::UsersToDevicesToEvents usersToDevicesToEvents; const auto data = job->jsonData(); - for(const auto &user : q->users()) { - for(const auto &device : connection->devicesForUser(user)) { + for(const auto &user : devices.keys()) { + for(const auto &device : devices[user]) { const auto recipientCurveKey = connection->curveKeyForUserDevice(user->id(), device); if (!connection->hasOlmSession(user, device)) { qCDebug(E2EE) << "Creating a new session for" << user << device; - if(data["one_time_keys"].toObject()[user->id()].toObject()[device].toObject().isEmpty()) { + if(data["one_time_keys"][user->id()][device].toObject().isEmpty()) { qWarning() << "No one time key for" << user << device; continue; } - auto keyId = data["one_time_keys"].toObject()[user->id()].toObject()[device].toObject().keys()[0]; - auto oneTimeKey = data["one_time_keys"].toObject()[user->id()].toObject()[device].toObject()[keyId].toObject()["key"].toString(); - auto signature = data["one_time_keys"].toObject()[user->id()].toObject()[device].toObject()[keyId].toObject()["signatures"].toObject()[user->id()].toObject()[QStringLiteral("ed25519:") + device].toString().toLatin1(); - auto signedData = data["one_time_keys"].toObject()[user->id()].toObject()[device].toObject()[keyId].toObject(); + const auto keyId = data["one_time_keys"][user->id()][device].toObject().keys()[0]; + const auto oneTimeKey = data["one_time_keys"][user->id()][device][keyId]["key"].toString(); + const auto signature = data["one_time_keys"][user->id()][device][keyId]["signatures"][user->id()][QStringLiteral("ed25519:") + device].toString().toLatin1(); + auto signedData = data["one_time_keys"][user->id()][device][keyId].toObject(); signedData.remove("unsigned"); signedData.remove("signatures"); auto signatureMatch = QOlmUtility().ed25519Verify(connection->edKeyForUserDevice(user->id(), device).toLatin1(), QJsonDocument(signedData).toJson(QJsonDocument::Compact), signature); if (std::holds_alternative(signatureMatch)) { //TODO i think there are more failed signature checks than expected. Investigate - qDebug() << signedData; qCWarning(E2EE) << "Failed to verify one-time-key signature for" << user->id() << device << ". Skipping this device."; - //Q_ASSERT(false); continue; } else { } @@ -522,10 +537,11 @@ public: } } connection->sendToDevices("m.room.encrypted", usersToDevicesToEvents); + connection->database()->setDevicesReceivedKey(q->id(), devices, sessionId, index); }); } - void sendMegolmSession() { + void sendMegolmSession(const QHash& devices) { // Save the session to this device const auto sessionId = currentOutboundMegolmSession->sessionId(); const auto _sessionKey = currentOutboundMegolmSession->sessionKey(); @@ -536,11 +552,8 @@ public: const auto sessionKey = std::get(_sessionKey); const auto senderKey = q->connection()->olmAccount()->identityKeys().curve25519; - // Send to key to ourself at this device - addInboundGroupSession(senderKey, sessionId, sessionKey); - // Send the session to other people - sendRoomKeyToDevices(sessionId, sessionKey); + sendRoomKeyToDevices(sessionId, sessionKey, devices, currentOutboundMegolmSession->sessionMessageIndex()); } #endif // Quotient_E2EE_ENABLED @@ -2066,8 +2079,10 @@ QString Room::Private::sendEvent(RoomEventPtr&& event) if (q->usesEncryption()) { if (!hasValidMegolmSession() || shouldRotateMegolmSession()) { createMegolmSession(); - sendMegolmSession(); } + const auto devicesWithoutKey = getDevicesWithoutKey(); + sendMegolmSession(devicesWithoutKey); + //TODO check if this is necessary //TODO check if we increment the sent message count event->setRoomId(id); -- cgit v1.2.3 From e437c29654e8f988ad694083401bbef23fbbcb18 Mon Sep 17 00:00:00 2001 From: Tobias Fella Date: Mon, 16 May 2022 20:51:41 +0200 Subject: More work; Update olm pickle & timestamps in database; Remove TODOs --- lib/connection.cpp | 12 ++++++++---- lib/connection.h | 3 +-- lib/database.cpp | 18 +++++++++++++++--- lib/database.h | 3 ++- lib/events/encryptedfile.cpp | 4 ++++ lib/room.cpp | 27 ++++++++++++++++----------- 6 files changed, 46 insertions(+), 21 deletions(-) (limited to 'lib/database.h') diff --git a/lib/connection.cpp b/lib/connection.cpp index 2a1b39f9..82046d53 100644 --- a/lib/connection.cpp +++ b/lib/connection.cpp @@ -33,6 +33,7 @@ #include "jobs/downloadfilejob.h" #include "jobs/mediathumbnailjob.h" #include "jobs/syncjob.h" +#include #ifdef Quotient_E2EE_ENABLED # include "database.h" @@ -2246,21 +2247,24 @@ bool Connection::hasOlmSession(User* user, const QString& deviceId) const QPair Connection::olmEncryptMessage(User* user, const QString& device, const QByteArray& message) { - //TODO be smarter about choosing a session; see e2ee impl guide - //TODO do we need to save the olm session after sending a message? const auto& curveKey = curveKeyForUserDevice(user->id(), device); QOlmMessage::Type type = d->olmSessions[curveKey][0]->encryptMessageType(); auto result = d->olmSessions[curveKey][0]->encrypt(message); + auto pickle = d->olmSessions[curveKey][0]->pickle(picklingMode()); + if (std::holds_alternative(pickle)) { + database()->updateOlmSession(curveKey, d->olmSessions[curveKey][0]->sessionId(), std::get(pickle)); + } else { + qCWarning(E2EE) << "Failed to pickle olm session."; + } return qMakePair(type, result.toCiphertext()); } -//TODO be more consistent with curveKey and identityKey void Connection::createOlmSession(const QString& theirIdentityKey, const QString& theirOneTimeKey) { auto session = QOlmSession::createOutboundSession(olmAccount(), theirIdentityKey, theirOneTimeKey); if (std::holds_alternative(session)) { - //TODO something qCWarning(E2EE) << "Failed to create olm session for " << theirIdentityKey << std::get(session); + return; } d->saveSession(std::get>(session), theirIdentityKey); d->olmSessions[theirIdentityKey].push_back(std::move(std::get>(session))); diff --git a/lib/connection.h b/lib/connection.h index 8bed55da..5a1f1e5c 100644 --- a/lib/connection.h +++ b/lib/connection.h @@ -329,8 +329,7 @@ public: void saveCurrentOutboundMegolmSession(Room *room, const QOlmOutboundGroupSessionPtr& data); - //This currently assumes that an olm session with (user, device) exists - //TODO make this return an event? + //This assumes that an olm session with (user, device) exists QPair olmEncryptMessage(User* user, const QString& device, const QByteArray& message); void createOlmSession(const QString& theirIdentityKey, const QString& theirOneTimeKey); #endif // Quotient_E2EE_ENABLED diff --git a/lib/database.cpp b/lib/database.cpp index 4a28fd4c..74b56a02 100644 --- a/lib/database.cpp +++ b/lib/database.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include "e2ee/e2ee.h" #include "e2ee/qolmsession.h" @@ -182,7 +183,7 @@ void Database::saveOlmSession(const QString& senderKey, const QString& sessionId UnorderedMap> Database::loadOlmSessions(const PicklingMode& picklingMode) { - auto query = prepareQuery(QStringLiteral("SELECT * FROM olm_sessions;")); + auto query = prepareQuery(QStringLiteral("SELECT * FROM olm_sessions ORDER BY lastReceived DESC;")); transaction(); execute(query); commit(); @@ -338,7 +339,6 @@ QOlmOutboundGroupSessionPtr Database::loadCurrentOutboundMegolmSession(const QSt void Database::setDevicesReceivedKey(const QString& roomId, QHash devices, const QString& sessionId, int index) { - //TODO this better auto connection = dynamic_cast(parent()); transaction(); for (const auto& user : devices.keys()) { @@ -360,7 +360,7 @@ QHash Database::devicesWithoutKey(Room* room, const QStrin { auto connection = dynamic_cast(parent()); QHash devices; - for (const auto& user : room->users()) {//TODO does this include invited & left? + for (const auto& user : room->users()) { devices[user->id()] = connection->devicesForUser(user); } @@ -375,3 +375,15 @@ QHash Database::devicesWithoutKey(Room* room, const QStrin } return devices; } + +void Database::updateOlmSession(const QString& senderKey, const QString& sessionId, const QByteArray& pickle) +{ + auto query = prepareQuery(QStringLiteral("UPDATE olm_sessions SET pickle=:pickle WHERE senderKey=:senderKey AND sessionId=:sessionId;")); + query.bindValue(":pickle", pickle); + query.bindValue(":senderKey", senderKey); + query.bindValue(":sessionId", sessionId); + transaction(); + execute(query); + commit(); +} + diff --git a/lib/database.h b/lib/database.h index 30f2f203..8ddd7b6d 100644 --- a/lib/database.h +++ b/lib/database.h @@ -32,7 +32,7 @@ public: QByteArray accountPickle(); void setAccountPickle(const QByteArray &pickle); void clear(); - void saveOlmSession(const QString& senderKey, const QString& sessionId, const QByteArray &pickle, const QDateTime& timestamp); + void saveOlmSession(const QString& senderKey, const QString& sessionId, const QByteArray& pickle, const QDateTime& timestamp); UnorderedMap> loadOlmSessions(const PicklingMode& picklingMode); UnorderedMap loadMegolmSessions(const QString& roomId, const PicklingMode& picklingMode); void saveMegolmSession(const QString& roomId, const QString& sessionId, const QByteArray& pickle, const QString& senderId, const QString& olmSessionId); @@ -42,6 +42,7 @@ public: void setOlmSessionLastReceived(const QString& sessionId, const QDateTime& timestamp); QOlmOutboundGroupSessionPtr loadCurrentOutboundMegolmSession(const QString& roomId, const PicklingMode& picklingMode); void saveCurrentOutboundMegolmSession(const QString& roomId, const PicklingMode& picklingMode, const QOlmOutboundGroupSessionPtr& data); + void updateOlmSession(const QString& senderKey, const QString& sessionId, const QByteArray& pickle); // Returns a map User -> [Device] that have not received key yet QHash devicesWithoutKey(Room* room, const QString &sessionId); diff --git a/lib/events/encryptedfile.cpp b/lib/events/encryptedfile.cpp index e90be428..bb4e26c7 100644 --- a/lib/events/encryptedfile.cpp +++ b/lib/events/encryptedfile.cpp @@ -56,6 +56,7 @@ QByteArray EncryptedFile::decryptFile(const QByteArray& ciphertext) const std::pair EncryptedFile::encryptFile(const QByteArray &plainText) { +#ifdef Quotient_E2EE_ENABLED QByteArray k = getRandom(32); auto kBase64 = k.toBase64(); QByteArray iv = getRandom(16); @@ -73,6 +74,9 @@ std::pair EncryptedFile::encryptFile(const QByteArray auto ivBase64 = iv.toBase64(); EncryptedFile file = {{}, key, ivBase64.left(ivBase64.indexOf('=')), {{QStringLiteral("sha256"), hash.left(hash.indexOf('='))}}, "v2"_ls}; return {file, cipherText}; +#else + return {{}, {}}; +#endif } void JsonObjectConverter::dumpTo(QJsonObject& jo, diff --git a/lib/room.cpp b/lib/room.cpp index a42b7184..0ca8f648 100644 --- a/lib/room.cpp +++ b/lib/room.cpp @@ -449,8 +449,8 @@ public: const auto sessionKey = currentOutboundMegolmSession->sessionKey(); if(std::holds_alternative(sessionKey)) { - qCWarning(E2EE) << "Session error"; - //TODO something + qCWarning(E2EE) << "Failed to load key for new megolm session"; + return; } addInboundGroupSession(q->connection()->olmAccount()->identityKeys().curve25519, currentOutboundMegolmSession->sessionId(), std::get(sessionKey), QString(connection->olmAccount()->identityKeys().ed25519)); } @@ -459,7 +459,6 @@ public: { // Noisy but nice for debugging //qCDebug(E2EE) << "Creating the payload for" << user->id() << device << sessionId << sessionKey.toHex(); - //TODO: store {user->id(), device, sessionId, theirIdentityKey}; required for key requests const auto event = makeEvent("m.megolm.v1.aes-sha2", q->id(), sessionId, sessionKey, q->localUser()->id()); QJsonObject payloadJson = event->fullJson(); payloadJson["recipient"] = user->id(); @@ -504,6 +503,9 @@ public: hash[user->id()] = u; } } + if (hash.isEmpty()) { + return; + } auto job = connection->callApi(hash); connect(job, &BaseJob::success, q, [job, this, sessionId, sessionKey, devices, index](){ Connection::UsersToDevicesToEvents usersToDevicesToEvents; @@ -525,7 +527,6 @@ public: signedData.remove("signatures"); auto signatureMatch = QOlmUtility().ed25519Verify(connection->edKeyForUserDevice(user->id(), device).toLatin1(), QJsonDocument(signedData).toJson(QJsonDocument::Compact), signature); if (std::holds_alternative(signatureMatch)) { - //TODO i think there are more failed signature checks than expected. Investigate qCWarning(E2EE) << "Failed to verify one-time-key signature for" << user->id() << device << ". Skipping this device."; continue; } else { @@ -535,8 +536,10 @@ public: usersToDevicesToEvents[user->id()][device] = payloadForUserDevice(user, device, sessionId, sessionKey); } } - connection->sendToDevices("m.room.encrypted", usersToDevicesToEvents); - connection->database()->setDevicesReceivedKey(q->id(), devices, sessionId, index); + if (!usersToDevicesToEvents.empty()) { + connection->sendToDevices("m.room.encrypted", usersToDevicesToEvents); + connection->database()->setDevicesReceivedKey(q->id(), devices, sessionId, index); + } }); } @@ -545,8 +548,8 @@ public: const auto sessionId = currentOutboundMegolmSession->sessionId(); const auto _sessionKey = currentOutboundMegolmSession->sessionKey(); if(std::holds_alternative(_sessionKey)) { - qCWarning(E2EE) << "Session error"; - //TODO something + qCWarning(E2EE) << "Error loading session key"; + return; } const auto sessionKey = std::get(_sessionKey); const auto senderKey = q->connection()->olmAccount()->identityKeys().curve25519; @@ -581,7 +584,6 @@ Room::Room(Connection* connection, QString id, JoinState initialJoinState) connect(this, &Room::userAdded, this, [this, connection](){ if(usesEncryption()) { connection->encryptionUpdate(this); - //TODO key at currentIndex to all user devices } }); d->groupSessions = connection->loadRoomMegolmSessions(this); @@ -2086,18 +2088,20 @@ QString Room::Private::doSendEvent(const RoomEvent* pEvent) const RoomEvent* _event = pEvent; if (q->usesEncryption()) { +#ifndef Quotient_E2EE_ENABLED + qWarning() << "This build of libQuotient does not support E2EE."; + return {}; +#else if (!hasValidMegolmSession() || shouldRotateMegolmSession()) { createMegolmSession(); } const auto devicesWithoutKey = getDevicesWithoutKey(); sendMegolmSession(devicesWithoutKey); - //TODO check if we increment the sent message count const auto encrypted = currentOutboundMegolmSession->encrypt(QJsonDocument(pEvent->fullJson()).toJson()); currentOutboundMegolmSession->setMessageCount(currentOutboundMegolmSession->messageCount() + 1); connection->saveCurrentOutboundMegolmSession(q, currentOutboundMegolmSession); if(std::holds_alternative(encrypted)) { - //TODO something qWarning(E2EE) << "Error encrypting message" << std::get(encrypted); return {}; } @@ -2110,6 +2114,7 @@ QString Room::Private::doSendEvent(const RoomEvent* pEvent) } // We show the unencrypted event locally while pending. The echo check will throw the encrypted version out _event = encryptedEvent; +#endif } if (auto call = -- cgit v1.2.3 From 8af39e510e550d001e207bdc0177a1480f6ebcba Mon Sep 17 00:00:00 2001 From: Tobias Fella Date: Wed, 23 Mar 2022 20:42:28 +0100 Subject: Add database migration --- lib/database.cpp | 18 +++++++++++++++--- lib/database.h | 1 + 2 files changed, 16 insertions(+), 3 deletions(-) (limited to 'lib/database.h') diff --git a/lib/database.cpp b/lib/database.cpp index 74b56a02..d2d33006 100644 --- a/lib/database.cpp +++ b/lib/database.cpp @@ -34,6 +34,7 @@ Database::Database(const QString& matrixId, const QString& deviceId, QObject* pa case 0: migrateTo1(); case 1: migrateTo2(); case 2: migrateTo3(); + case 3: migrateTo4(); } } @@ -90,12 +91,11 @@ void Database::migrateTo1() execute(QStringLiteral("CREATE TABLE accounts (pickle TEXT);")); execute(QStringLiteral("CREATE TABLE olm_sessions (senderKey TEXT, sessionId TEXT, pickle TEXT);")); execute(QStringLiteral("CREATE TABLE inbound_megolm_sessions (roomId TEXT, senderKey TEXT, sessionId TEXT, pickle TEXT);")); - execute(QStringLiteral("CREATE TABLE outbound_megolm_sessions (roomId TEXT, sessionId TEXT, pickle TEXT, creationTime TEXT, messageCount INTEGER);")); + execute(QStringLiteral("CREATE TABLE outbound_megolm_sessions (roomId TEXT, senderKey TEXT, sessionId TEXT, pickle TEXT);")); execute(QStringLiteral("CREATE TABLE group_session_record_index (roomId TEXT, sessionId TEXT, i INTEGER, eventId TEXT, ts INTEGER);")); execute(QStringLiteral("CREATE TABLE tracked_users (matrixId TEXT);")); execute(QStringLiteral("CREATE TABLE outdated_users (matrixId TEXT);")); execute(QStringLiteral("CREATE TABLE tracked_devices (matrixId TEXT, deviceId TEXT, curveKeyId TEXT, curveKey TEXT, edKeyId TEXT, edKey TEXT);")); - execute(QStringLiteral("CREATE TABLE sent_megolm_sessions (roomId TEXT, userId TEXT, deviceId TEXT, identityKey TEXT, sessionId TEXT, i INTEGER);")); execute(QStringLiteral("PRAGMA user_version = 1;")); commit(); @@ -108,7 +108,7 @@ void Database::migrateTo2() //TODO remove this column again - we don't need it after all execute(QStringLiteral("ALTER TABLE inbound_megolm_sessions ADD ed25519Key TEXT")); execute(QStringLiteral("ALTER TABLE olm_sessions ADD lastReceived TEXT")); - + // Add indexes for improving queries speed on larger database execute(QStringLiteral("CREATE INDEX sessions_session_idx ON olm_sessions(sessionId)")); execute(QStringLiteral("CREATE INDEX outbound_room_idx ON outbound_megolm_sessions(roomId)")); @@ -132,6 +132,18 @@ void Database::migrateTo3() commit(); } +void Database::migrateTo3() +{ + qCDebug(DATABASE) << "Migrating database to version 4"; + transaction(); + + execute(QStringLiteral("CREATE TABLE sent_megolm_sessions (roomId TEXT, userId TEXT, deviceId TEXT, identityKey TEXT, sessionId TEXT, i INTEGER);")); + execute(QStringLiteral("ALTER TABLE outbound_megolm_sessions ADD creationTime TEXT;")); + execute(QStringLiteral("ALTER TABLE outbound_megolm_sessions ADD messageCount INTEGER;")); + execute(QStringLiteral("PRAGMA user_version = 3;")); + commit(); +} + QByteArray Database::accountPickle() { auto query = prepareQuery(QStringLiteral("SELECT pickle FROM accounts;")); diff --git a/lib/database.h b/lib/database.h index 8ddd7b6d..00002204 100644 --- a/lib/database.h +++ b/lib/database.h @@ -52,6 +52,7 @@ private: void migrateTo1(); void migrateTo2(); void migrateTo3(); + void migrateTo4(); QString m_matrixId; }; -- cgit v1.2.3 From b29eb3954b798ac9110906cd79c4f288deaa2596 Mon Sep 17 00:00:00 2001 From: Tobias Fella Date: Wed, 18 May 2022 22:39:27 +0200 Subject: Make database independent of {Room, User, Connection} --- lib/connection.cpp | 12 ++++++------ lib/connection.h | 6 +++--- lib/database.cpp | 23 +++++++---------------- lib/database.h | 6 +++--- lib/room.cpp | 51 ++++++++++++++++++++++++++++----------------------- 5 files changed, 47 insertions(+), 51 deletions(-) (limited to 'lib/database.h') diff --git a/lib/connection.cpp b/lib/connection.cpp index 66e21a2a..dba18cb1 100644 --- a/lib/connection.cpp +++ b/lib/connection.cpp @@ -2214,9 +2214,9 @@ void Connection::saveMegolmSession(const Room* room, session.senderId(), session.olmSessionId()); } -QStringList Connection::devicesForUser(User* user) const +QStringList Connection::devicesForUser(const QString& userId) const { - return d->deviceKeys[user->id()].keys(); + return d->deviceKeys[userId].keys(); } QString Connection::curveKeyForUserDevice(const QString& user, const QString& device) const @@ -2238,15 +2238,15 @@ bool Connection::isKnownCurveKey(const QString& user, const QString& curveKey) return query.next(); } -bool Connection::hasOlmSession(User* user, const QString& deviceId) const +bool Connection::hasOlmSession(const QString& user, const QString& deviceId) const { - const auto& curveKey = curveKeyForUserDevice(user->id(), deviceId); + const auto& curveKey = curveKeyForUserDevice(user, deviceId); return d->olmSessions.contains(curveKey) && !d->olmSessions[curveKey].empty(); } -QPair Connection::olmEncryptMessage(User* user, const QString& device, const QByteArray& message) +QPair Connection::olmEncryptMessage(const QString& user, const QString& device, const QByteArray& message) { - const auto& curveKey = curveKeyForUserDevice(user->id(), device); + const auto& curveKey = curveKeyForUserDevice(user, device); QOlmMessage::Type type = d->olmSessions[curveKey][0]->encryptMessageType(); auto result = d->olmSessions[curveKey][0]->encrypt(message); auto pickle = d->olmSessions[curveKey][0]->pickle(picklingMode()); diff --git a/lib/connection.h b/lib/connection.h index 5b266aad..f8744752 100644 --- a/lib/connection.h +++ b/lib/connection.h @@ -323,14 +323,14 @@ public: const Room* room); void saveMegolmSession(const Room* room, const QOlmInboundGroupSession& session); - bool hasOlmSession(User* user, const QString& deviceId) const; + bool hasOlmSession(const QString& user, const QString& deviceId) const; QOlmOutboundGroupSessionPtr loadCurrentOutboundMegolmSession(Room* room); void saveCurrentOutboundMegolmSession(Room *room, const QOlmOutboundGroupSessionPtr& data); //This assumes that an olm session with (user, device) exists - QPair olmEncryptMessage(User* user, const QString& device, const QByteArray& message); + QPair olmEncryptMessage(const QString& userId, const QString& device, const QByteArray& message); void createOlmSession(const QString& theirIdentityKey, const QString& theirOneTimeKey); #endif // Quotient_E2EE_ENABLED Q_INVOKABLE Quotient::SyncJob* syncJob() const; @@ -694,7 +694,7 @@ public Q_SLOTS: PicklingMode picklingMode() const; QJsonObject decryptNotification(const QJsonObject ¬ification); - QStringList devicesForUser(User* user) const; + QStringList devicesForUser(const QString& user) const; QString curveKeyForUserDevice(const QString &user, const QString& device) const; QString edKeyForUserDevice(const QString& user, const QString& device) const; bool isKnownCurveKey(const QString& user, const QString& curveKey); diff --git a/lib/database.cpp b/lib/database.cpp index 99c6f358..3255e5e7 100644 --- a/lib/database.cpp +++ b/lib/database.cpp @@ -13,9 +13,7 @@ #include "e2ee/e2ee.h" #include "e2ee/qolmsession.h" #include "e2ee/qolminboundsession.h" -#include "connection.h" -#include "user.h" -#include "room.h" +#include "e2ee/qolmoutboundsession.h" using namespace Quotient; Database::Database(const QString& matrixId, const QString& deviceId, QObject* parent) @@ -348,17 +346,16 @@ QOlmOutboundGroupSessionPtr Database::loadCurrentOutboundMegolmSession(const QSt return nullptr; } -void Database::setDevicesReceivedKey(const QString& roomId, QHash devices, const QString& sessionId, int index) +void Database::setDevicesReceivedKey(const QString& roomId, const QHash>>& devices, const QString& sessionId, int index) { - auto connection = dynamic_cast(parent()); transaction(); for (const auto& user : devices.keys()) { - for (const auto& device : devices[user]) { + for (const auto& [device, curveKey] : devices[user]) { auto query = prepareQuery(QStringLiteral("INSERT INTO sent_megolm_sessions(roomId, userId, deviceId, identityKey, sessionId, i) VALUES(:roomId, :userId, :deviceId, :identityKey, :sessionId, :i);")); query.bindValue(":roomId", roomId); - query.bindValue(":userId", user->id()); + query.bindValue(":userId", user); query.bindValue(":deviceId", device); - query.bindValue(":identityKey", connection->curveKeyForUserDevice(user->id(), device)); + query.bindValue(":identityKey", curveKey); query.bindValue(":sessionId", sessionId); query.bindValue(":i", index); execute(query); @@ -367,16 +364,10 @@ void Database::setDevicesReceivedKey(const QString& roomId, QHash Database::devicesWithoutKey(Room* room, const QString &sessionId) +QHash Database::devicesWithoutKey(const QString& roomId, QHash& devices, const QString &sessionId) { - auto connection = dynamic_cast(parent()); - QHash devices; - for (const auto& user : room->users()) { - devices[user->id()] = connection->devicesForUser(user); - } - auto query = prepareQuery(QStringLiteral("SELECT userId, deviceId FROM sent_megolm_sessions WHERE roomId=:roomId AND sessionId=:sessionId")); - query.bindValue(":roomId", room->id()); + query.bindValue(":roomId", roomId); query.bindValue(":sessionId", sessionId); transaction(); execute(query); diff --git a/lib/database.h b/lib/database.h index 00002204..8bef332f 100644 --- a/lib/database.h +++ b/lib/database.h @@ -44,9 +44,9 @@ public: void saveCurrentOutboundMegolmSession(const QString& roomId, const PicklingMode& picklingMode, const QOlmOutboundGroupSessionPtr& data); void updateOlmSession(const QString& senderKey, const QString& sessionId, const QByteArray& pickle); - // Returns a map User -> [Device] that have not received key yet - QHash devicesWithoutKey(Room* room, const QString &sessionId); - void setDevicesReceivedKey(const QString& roomId, QHash devices, const QString& sessionId, int index); + // Returns a map UserId -> [DeviceId] that have not received key yet + QHash devicesWithoutKey(const QString& roomId, QHash& devices, const QString &sessionId); + void setDevicesReceivedKey(const QString& roomId, const QHash>>& devices, const QString& sessionId, int index); private: void migrateTo1(); diff --git a/lib/room.cpp b/lib/room.cpp index d77bf9ef..5d3ae329 100644 --- a/lib/room.cpp +++ b/lib/room.cpp @@ -455,16 +455,16 @@ public: addInboundGroupSession(currentOutboundMegolmSession->sessionId(), *sessionKey, q->localUser()->id(), "SELF"_ls); } - std::unique_ptr payloadForUserDevice(User* user, const QString& device, const QByteArray& sessionId, const QByteArray& sessionKey) + std::unique_ptr payloadForUserDevice(QString user, const QString& device, const QByteArray& sessionId, const QByteArray& sessionKey) { // Noisy but nice for debugging //qCDebug(E2EE) << "Creating the payload for" << user->id() << device << sessionId << sessionKey.toHex(); const auto event = makeEvent("m.megolm.v1.aes-sha2", q->id(), sessionId, sessionKey, q->localUser()->id()); QJsonObject payloadJson = event->fullJson(); - payloadJson["recipient"] = user->id(); + payloadJson["recipient"] = user; payloadJson["sender"] = connection->user()->id(); QJsonObject recipientObject; - recipientObject["ed25519"] = connection->edKeyForUserDevice(user->id(), device); + recipientObject["ed25519"] = connection->edKeyForUserDevice(user, device); payloadJson["recipient_keys"] = recipientObject; QJsonObject senderObject; senderObject["ed25519"] = QString(connection->olmAccount()->identityKeys().ed25519); @@ -472,22 +472,21 @@ public: payloadJson["sender_device"] = connection->deviceId(); auto cipherText = connection->olmEncryptMessage(user, device, QJsonDocument(payloadJson).toJson(QJsonDocument::Compact)); QJsonObject encrypted; - encrypted[connection->curveKeyForUserDevice(user->id(), device)] = QJsonObject{{"type", cipherText.first}, {"body", QString(cipherText.second)}}; + encrypted[connection->curveKeyForUserDevice(user, device)] = QJsonObject{{"type", cipherText.first}, {"body", QString(cipherText.second)}}; return makeEvent(encrypted, connection->olmAccount()->identityKeys().curve25519); } - QHash getDevicesWithoutKey() const + QHash getDevicesWithoutKey() const { - QHash devices; - auto rawDevices = q->connection()->database()->devicesWithoutKey(q, QString(currentOutboundMegolmSession->sessionId())); - for (const auto& user : rawDevices.keys()) { - devices[q->connection()->user(user)] = rawDevices[user]; + QHash devices; + for (const auto& user : q->users()) { + devices[user->id()] = q->connection()->devicesForUser(user->id()); } - return devices; + return q->connection()->database()->devicesWithoutKey(q->id(), devices, QString(currentOutboundMegolmSession->sessionId())); } - void sendRoomKeyToDevices(const QByteArray& sessionId, const QByteArray& sessionKey, const QHash devices, int index) + void sendRoomKeyToDevices(const QByteArray& sessionId, const QByteArray& sessionKey, const QHash devices, int index) { qCDebug(E2EE) << "Sending room key to devices" << sessionId, sessionKey.toHex(); QHash> hash; @@ -500,7 +499,7 @@ public: } } if (!u.isEmpty()) { - hash[user->id()] = u; + hash[user] = u; } } if (hash.isEmpty()) { @@ -512,38 +511,44 @@ public: const auto data = job->jsonData(); for(const auto &user : devices.keys()) { for(const auto &device : devices[user]) { - const auto recipientCurveKey = connection->curveKeyForUserDevice(user->id(), device); + const auto recipientCurveKey = connection->curveKeyForUserDevice(user, device); if (!connection->hasOlmSession(user, device)) { qCDebug(E2EE) << "Creating a new session for" << user << device; - if(data["one_time_keys"][user->id()][device].toObject().isEmpty()) { + if(data["one_time_keys"][user][device].toObject().isEmpty()) { qWarning() << "No one time key for" << user << device; continue; } - const auto keyId = data["one_time_keys"][user->id()][device].toObject().keys()[0]; - const auto oneTimeKey = data["one_time_keys"][user->id()][device][keyId]["key"].toString(); - const auto signature = data["one_time_keys"][user->id()][device][keyId]["signatures"][user->id()][QStringLiteral("ed25519:") + device].toString().toLatin1(); - auto signedData = data["one_time_keys"][user->id()][device][keyId].toObject(); + const auto keyId = data["one_time_keys"][user][device].toObject().keys()[0]; + const auto oneTimeKey = data["one_time_keys"][user][device][keyId]["key"].toString(); + const auto signature = data["one_time_keys"][user][device][keyId]["signatures"][user][QStringLiteral("ed25519:") + device].toString().toLatin1(); + auto signedData = data["one_time_keys"][user][device][keyId].toObject(); signedData.remove("unsigned"); signedData.remove("signatures"); - auto signatureMatch = QOlmUtility().ed25519Verify(connection->edKeyForUserDevice(user->id(), device).toLatin1(), QJsonDocument(signedData).toJson(QJsonDocument::Compact), signature); + auto signatureMatch = QOlmUtility().ed25519Verify(connection->edKeyForUserDevice(user, device).toLatin1(), QJsonDocument(signedData).toJson(QJsonDocument::Compact), signature); if (!signatureMatch) { - qCWarning(E2EE) << "Failed to verify one-time-key signature for" << user->id() << device << ". Skipping this device."; + qCWarning(E2EE) << "Failed to verify one-time-key signature for" << user << device << ". Skipping this device."; continue; } else { } connection->createOlmSession(recipientCurveKey, oneTimeKey); } - usersToDevicesToEvents[user->id()][device] = payloadForUserDevice(user, device, sessionId, sessionKey); + usersToDevicesToEvents[user][device] = payloadForUserDevice(user, device, sessionId, sessionKey); } } if (!usersToDevicesToEvents.empty()) { connection->sendToDevices("m.room.encrypted", usersToDevicesToEvents); - connection->database()->setDevicesReceivedKey(q->id(), devices, sessionId, index); + QHash>> receivedDevices; + for (const auto& user : devices.keys()) { + for (const auto& device : devices[user]) { + receivedDevices[user] += {device, q->connection()->curveKeyForUserDevice(user, device) }; + } + } + connection->database()->setDevicesReceivedKey(q->id(), receivedDevices, sessionId, index); } }); } - void sendMegolmSession(const QHash& devices) { + void sendMegolmSession(const QHash& devices) { // Save the session to this device const auto sessionId = currentOutboundMegolmSession->sessionId(); const auto _sessionKey = currentOutboundMegolmSession->sessionKey(); -- cgit v1.2.3 From 41897df408c1398881bb8cf82ae0dc4503cefef7 Mon Sep 17 00:00:00 2001 From: Tobias Fella Date: Thu, 19 May 2022 14:11:18 +0200 Subject: Use list of 3-tuple instead of map --- lib/database.cpp | 22 ++++++++++------------ lib/database.h | 2 +- lib/room.cpp | 4 ++-- 3 files changed, 13 insertions(+), 15 deletions(-) (limited to 'lib/database.h') diff --git a/lib/database.cpp b/lib/database.cpp index 3255e5e7..0119b35c 100644 --- a/lib/database.cpp +++ b/lib/database.cpp @@ -346,20 +346,18 @@ QOlmOutboundGroupSessionPtr Database::loadCurrentOutboundMegolmSession(const QSt return nullptr; } -void Database::setDevicesReceivedKey(const QString& roomId, const QHash>>& devices, const QString& sessionId, int index) +void Database::setDevicesReceivedKey(const QString& roomId, const QVector>& devices, const QString& sessionId, int index) { transaction(); - for (const auto& user : devices.keys()) { - for (const auto& [device, curveKey] : devices[user]) { - auto query = prepareQuery(QStringLiteral("INSERT INTO sent_megolm_sessions(roomId, userId, deviceId, identityKey, sessionId, i) VALUES(:roomId, :userId, :deviceId, :identityKey, :sessionId, :i);")); - query.bindValue(":roomId", roomId); - query.bindValue(":userId", user); - query.bindValue(":deviceId", device); - query.bindValue(":identityKey", curveKey); - query.bindValue(":sessionId", sessionId); - query.bindValue(":i", index); - execute(query); - } + for (const auto& [user, device, curveKey] : devices) { + auto query = prepareQuery(QStringLiteral("INSERT INTO sent_megolm_sessions(roomId, userId, deviceId, identityKey, sessionId, i) VALUES(:roomId, :userId, :deviceId, :identityKey, :sessionId, :i);")); + query.bindValue(":roomId", roomId); + query.bindValue(":userId", user); + query.bindValue(":deviceId", device); + query.bindValue(":identityKey", curveKey); + query.bindValue(":sessionId", sessionId); + query.bindValue(":i", index); + execute(query); } commit(); } diff --git a/lib/database.h b/lib/database.h index 8bef332f..ef251d66 100644 --- a/lib/database.h +++ b/lib/database.h @@ -46,7 +46,7 @@ public: // Returns a map UserId -> [DeviceId] that have not received key yet QHash devicesWithoutKey(const QString& roomId, QHash& devices, const QString &sessionId); - void setDevicesReceivedKey(const QString& roomId, const QHash>>& devices, const QString& sessionId, int index); + void setDevicesReceivedKey(const QString& roomId, const QVector>& devices, const QString& sessionId, int index); private: void migrateTo1(); diff --git a/lib/room.cpp b/lib/room.cpp index 5d3ae329..3696f808 100644 --- a/lib/room.cpp +++ b/lib/room.cpp @@ -537,10 +537,10 @@ public: } if (!usersToDevicesToEvents.empty()) { connection->sendToDevices("m.room.encrypted", usersToDevicesToEvents); - QHash>> receivedDevices; + QVector> receivedDevices; for (const auto& user : devices.keys()) { for (const auto& device : devices[user]) { - receivedDevices[user] += {device, q->connection()->curveKeyForUserDevice(user, device) }; + receivedDevices += {user, device, q->connection()->curveKeyForUserDevice(user, device) }; } } connection->database()->setDevicesReceivedKey(q->id(), receivedDevices, sessionId, index); -- cgit v1.2.3 From 5df53b8d5c8b21228ecf9938330dd4d85d3de6af Mon Sep 17 00:00:00 2001 From: Tobias Fella Date: Thu, 19 May 2022 16:01:07 +0200 Subject: Document devices tuple --- lib/database.h | 1 + 1 file changed, 1 insertion(+) (limited to 'lib/database.h') diff --git a/lib/database.h b/lib/database.h index ef251d66..45348c8d 100644 --- a/lib/database.h +++ b/lib/database.h @@ -46,6 +46,7 @@ public: // Returns a map UserId -> [DeviceId] that have not received key yet QHash devicesWithoutKey(const QString& roomId, QHash& devices, const QString &sessionId); + // 'devices' contains tuples {userId, deviceId, curveKey} void setDevicesReceivedKey(const QString& roomId, const QVector>& devices, const QString& sessionId, int index); private: -- cgit v1.2.3