From c1929dc22c87ac61e5369cb752e6ddd0ef6a79bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roman=20Pla=CC=81s=CC=8Cil?= Date: Tue, 8 Aug 2017 15:52:52 +0800 Subject: WIP saving intermediate state to JSON --- connection.cpp | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) (limited to 'connection.cpp') diff --git a/connection.cpp b/connection.cpp index f9f1490c..3ecabdc5 100644 --- a/connection.cpp +++ b/connection.cpp @@ -32,6 +32,8 @@ #include "jobs/mediathumbnailjob.h" #include +#include +#include using namespace QMatrixClient; @@ -320,3 +322,44 @@ QByteArray Connection::generateTxnId() { return d->data->generateTxnId(); } + +QFile Connection::getStateSaveFile() const { + return QFile(QStandardPaths::writableLocation(QStandardPaths::AppDataLocation) + "/data.json"); +} + +void Connection::saveState() { + QJsonObject rooms; + + for (auto i : this->roomMap()) { + QJsonObject roomObj; + i->toJson(roomObj); + rooms[i->id()] = roomObj; + } + + QJsonObject rootObj{ + {"next_batch", QJsonValue(d->data->lastEvent())}, + {"presence", QJsonValue(QJsonObject())}, + {"rooms", QJsonValue({ + qMakePair(QString("leave"), QJsonValue(QJsonObject())), + qMakePair(QString("join"), QJsonValue(rooms)), + qMakePair(QString("invite"), QJsonValue(QJsonObject())) + })} + }; + QJsonDocument doc { rootObj }; + QByteArray data = doc.toJson(); + + QFile outfile = getStateSaveFile(); + outfile.open(QFile::WriteOnly); + qInfo() << "Writing state to file=" << outfile.fileName(); + //QFile outfile(path); + outfile.write(data.data(), data.size()); +} + +void Connection::loadState() { + QFile file = getStateSaveFile(); + if (!file.exists()) return; + file.open(QFile::ReadOnly); + QByteArray data = file.readAll(); + + QJsonDocument doc = QJsonDocument.fromJson(data.data(), data.size()); +} -- cgit v1.2.3 From 6ae8e3d78b5c4a75ca7d5ca88af730071047d148 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roman=20Pla=CC=81s=CC=8Cil?= Date: Wed, 16 Aug 2017 13:56:13 +0800 Subject: Implement saving save to enable incremental sync even after shutdown --- connection.cpp | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) (limited to 'connection.cpp') diff --git a/connection.cpp b/connection.cpp index 3ecabdc5..7650b4dd 100644 --- a/connection.cpp +++ b/connection.cpp @@ -159,12 +159,7 @@ void Connection::sync(int timeout) auto job = d->syncJob = callApi(d->data->lastEvent(), filter, timeout); connect( job, &SyncJob::success, [=] () { - d->data->setLastEvent(job->nextBatch()); - for( auto&& roomData: job->takeRoomData() ) - { - if ( auto* r = provideRoom(roomData.roomId) ) - r->updateData(std::move(roomData)); - } + onSyncSuccess(*job->data()); d->syncJob = nullptr; emit syncDone(); }); @@ -178,6 +173,17 @@ void Connection::sync(int timeout) }); } +void Connection::onSyncSuccess(SyncData &data) { + d->data->setLastEvent(data.nextBatch()); + qInfo() << "last event " << d->data->lastEvent(); + for( auto&& roomData: data.takeRoomData() ) + { + if ( auto* r = provideRoom(roomData.roomId) ) + r->updateData(std::move(roomData)); + } + +} + void Connection::stopSync() { if (d->syncJob) @@ -323,8 +329,8 @@ QByteArray Connection::generateTxnId() return d->data->generateTxnId(); } -QFile Connection::getStateSaveFile() const { - return QFile(QStandardPaths::writableLocation(QStandardPaths::AppDataLocation) + "/data.json"); +QString Connection::getStateSaveFile() const { + return QStandardPaths::writableLocation(QStandardPaths::AppDataLocation) + "/state.json"; } void Connection::saveState() { @@ -348,18 +354,20 @@ void Connection::saveState() { QJsonDocument doc { rootObj }; QByteArray data = doc.toJson(); - QFile outfile = getStateSaveFile(); + QFile outfile { getStateSaveFile() }; outfile.open(QFile::WriteOnly); qInfo() << "Writing state to file=" << outfile.fileName(); - //QFile outfile(path); outfile.write(data.data(), data.size()); } void Connection::loadState() { - QFile file = getStateSaveFile(); + QFile file { getStateSaveFile() }; if (!file.exists()) return; file.open(QFile::ReadOnly); QByteArray data = file.readAll(); - QJsonDocument doc = QJsonDocument.fromJson(data.data(), data.size()); + QJsonDocument doc { QJsonDocument::fromJson(data) }; + SyncData sync; + sync.parseJson(doc); + onSyncSuccess(sync); } -- cgit v1.2.3 From f3d893150f3991db2aed7a890d6156a576ae2e42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roman=20Pla=CC=81s=CC=8Cil?= Date: Fri, 18 Aug 2017 14:29:06 +0800 Subject: Fix compilation for Qt 5.2.1 --- connection.cpp | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) (limited to 'connection.cpp') diff --git a/connection.cpp b/connection.cpp index 7650b4dd..4d76f7cd 100644 --- a/connection.cpp +++ b/connection.cpp @@ -59,6 +59,7 @@ class Connection::Private QString userId; SyncJob* syncJob; + QString stateSaveFile; }; Connection::Connection(const QUrl& server, QObject* parent) @@ -175,7 +176,6 @@ void Connection::sync(int timeout) void Connection::onSyncSuccess(SyncData &data) { d->data->setLastEvent(data.nextBatch()); - qInfo() << "last event " << d->data->lastEvent(); for( auto&& roomData: data.takeRoomData() ) { if ( auto* r = provideRoom(roomData.roomId) ) @@ -330,7 +330,11 @@ QByteArray Connection::generateTxnId() } QString Connection::getStateSaveFile() const { - return QStandardPaths::writableLocation(QStandardPaths::AppDataLocation) + "/state.json"; + return d->stateSaveFile; +} + +void Connection::setStateSaveFile(const QString &path) { + d->stateSaveFile = path; } void Connection::saveState() { @@ -342,21 +346,25 @@ void Connection::saveState() { rooms[i->id()] = roomObj; } - QJsonObject rootObj{ - {"next_batch", QJsonValue(d->data->lastEvent())}, - {"presence", QJsonValue(QJsonObject())}, - {"rooms", QJsonValue({ - qMakePair(QString("leave"), QJsonValue(QJsonObject())), - qMakePair(QString("join"), QJsonValue(rooms)), - qMakePair(QString("invite"), QJsonValue(QJsonObject())) - })} - }; + QJsonObject roomObj; + roomObj["leave"] = QJsonObject(); + roomObj["join"] = rooms; + roomObj["invite"] = QJsonObject(); + + QJsonObject rootObj; + rootObj["next_batch"] = d->data->lastEvent(); + rootObj["presence"] = QJsonObject(); + rootObj["rooms"] = roomObj; + QJsonDocument doc { rootObj }; QByteArray data = doc.toJson(); - QFile outfile { getStateSaveFile() }; + QString filepath = getStateSaveFile(); + if (filepath.isEmpty()) return; + + QFile outfile { filepath }; outfile.open(QFile::WriteOnly); - qInfo() << "Writing state to file=" << outfile.fileName(); + qCDebug(MAIN) << "Writing state to file=" << outfile.fileName(); outfile.write(data.data(), data.size()); } -- cgit v1.2.3 From 456604040ce9cf3c22b726e8ef279a1fe4a85f39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roman=20Pl=C3=A1=C5=A1il?= Date: Wed, 23 Aug 2017 14:49:06 +0800 Subject: Use QUrl for state save file, create directory if it doesn't exist --- connection.cpp | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) (limited to 'connection.cpp') diff --git a/connection.cpp b/connection.cpp index 4d76f7cd..136ccda1 100644 --- a/connection.cpp +++ b/connection.cpp @@ -34,6 +34,8 @@ #include #include #include +#include +#include using namespace QMatrixClient; @@ -59,7 +61,7 @@ class Connection::Private QString userId; SyncJob* syncJob; - QString stateSaveFile; + QUrl stateSaveFile; }; Connection::Connection(const QUrl& server, QObject* parent) @@ -329,15 +331,17 @@ QByteArray Connection::generateTxnId() return d->data->generateTxnId(); } -QString Connection::getStateSaveFile() const { +QUrl Connection::getStateSaveFile() const { return d->stateSaveFile; } -void Connection::setStateSaveFile(const QString &path) { +void Connection::setStateSaveFile(const QUrl &path) { d->stateSaveFile = path; } void Connection::saveState() { + if (getStateSaveFile().isEmpty()) return; + QJsonObject rooms; for (auto i : this->roomMap()) { @@ -359,17 +363,21 @@ void Connection::saveState() { QJsonDocument doc { rootObj }; QByteArray data = doc.toJson(); - QString filepath = getStateSaveFile(); - if (filepath.isEmpty()) return; + QFileInfo stateFile { getStateSaveFile().toLocalFile() }; + QFile outfile { stateFile.absoluteFilePath() }; + if (!stateFile.dir().exists()) stateFile.dir().mkpath("."); + + if (outfile.open(QFile::WriteOnly)) { + qCDebug(MAIN) << "Writing state to file=" << outfile.fileName(); + outfile.write(data.data(), data.size()); - QFile outfile { filepath }; - outfile.open(QFile::WriteOnly); - qCDebug(MAIN) << "Writing state to file=" << outfile.fileName(); - outfile.write(data.data(), data.size()); + } else { + qCWarning(MAIN) << outfile.errorString(); + } } void Connection::loadState() { - QFile file { getStateSaveFile() }; + QFile file { getStateSaveFile().toLocalFile() }; if (!file.exists()) return; file.open(QFile::ReadOnly); QByteArray data = file.readAll(); -- cgit v1.2.3 From 1a00b1c38c0957df9758b0edeff4559f2515fe5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roman=20Pl=C3=A1=C5=A1il?= Date: Sun, 27 Aug 2017 12:32:00 +0800 Subject: Use QJsonObject.insert and QStringLiteral --- connection.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'connection.cpp') diff --git a/connection.cpp b/connection.cpp index 136ccda1..89da47cd 100644 --- a/connection.cpp +++ b/connection.cpp @@ -351,14 +351,14 @@ void Connection::saveState() { } QJsonObject roomObj; - roomObj["leave"] = QJsonObject(); - roomObj["join"] = rooms; - roomObj["invite"] = QJsonObject(); + roomObj.insert("leave", QJsonObject()); + roomObj.insert("join", rooms); + roomObj.insert("invite", QJsonObject()); QJsonObject rootObj; - rootObj["next_batch"] = d->data->lastEvent(); - rootObj["presence"] = QJsonObject(); - rootObj["rooms"] = roomObj; + rootObj.insert("next_batch", d->data->lastEvent()); + rootObj.insert("presence", QJsonObject()); + rootObj.insert("rooms", roomObj); QJsonDocument doc { rootObj }; QByteArray data = doc.toJson(); -- cgit v1.2.3 From 1b459a0f32b28b2f3fe8dcc0cd2553c3aea9ccee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roman=20Pl=C3=A1=C5=A1il?= Date: Sun, 27 Aug 2017 14:45:43 +0800 Subject: Remove saveStateFile property and just use an argument. --- connection.cpp | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) (limited to 'connection.cpp') diff --git a/connection.cpp b/connection.cpp index 89da47cd..97594e7d 100644 --- a/connection.cpp +++ b/connection.cpp @@ -61,7 +61,6 @@ class Connection::Private QString userId; SyncJob* syncJob; - QUrl stateSaveFile; }; Connection::Connection(const QUrl& server, QObject* parent) @@ -331,17 +330,7 @@ QByteArray Connection::generateTxnId() return d->data->generateTxnId(); } -QUrl Connection::getStateSaveFile() const { - return d->stateSaveFile; -} - -void Connection::setStateSaveFile(const QUrl &path) { - d->stateSaveFile = path; -} - -void Connection::saveState() { - if (getStateSaveFile().isEmpty()) return; - +void Connection::saveState(const QUrl &toFile) { QJsonObject rooms; for (auto i : this->roomMap()) { @@ -363,7 +352,7 @@ void Connection::saveState() { QJsonDocument doc { rootObj }; QByteArray data = doc.toJson(); - QFileInfo stateFile { getStateSaveFile().toLocalFile() }; + QFileInfo stateFile { toFile.toLocalFile() }; QFile outfile { stateFile.absoluteFilePath() }; if (!stateFile.dir().exists()) stateFile.dir().mkpath("."); @@ -376,8 +365,8 @@ void Connection::saveState() { } } -void Connection::loadState() { - QFile file { getStateSaveFile().toLocalFile() }; +void Connection::loadState(const QUrl &fromFile) { + QFile file { fromFile.toLocalFile() }; if (!file.exists()) return; file.open(QFile::ReadOnly); QByteArray data = file.readAll(); -- cgit v1.2.3 From 9a26c52f8c3e71f803fd38128ecddfd6ce9748f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roman=20Pl=C3=A1=C5=A1il?= Date: Sat, 2 Sep 2017 00:58:58 +0800 Subject: Use status return type for parseJson --- connection.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'connection.cpp') diff --git a/connection.cpp b/connection.cpp index 97594e7d..fe0cb251 100644 --- a/connection.cpp +++ b/connection.cpp @@ -32,10 +32,9 @@ #include "jobs/mediathumbnailjob.h" #include -#include -#include -#include -#include +#include +#include +#include using namespace QMatrixClient; -- cgit v1.2.3 From a7ee0dfacc2c571572240191b3cf0846a9e32998 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roman=20Pla=CC=81s=CC=8Cil?= Date: Sun, 3 Sep 2017 14:43:05 +0800 Subject: More fixes --- connection.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'connection.cpp') diff --git a/connection.cpp b/connection.cpp index fe0cb251..d2acf928 100644 --- a/connection.cpp +++ b/connection.cpp @@ -333,8 +333,7 @@ void Connection::saveState(const QUrl &toFile) { QJsonObject rooms; for (auto i : this->roomMap()) { - QJsonObject roomObj; - i->toJson(roomObj); + QJsonObject roomObj = i->toJson(); rooms[i->id()] = roomObj; } -- cgit v1.2.3 From 56d34ecab5eb35c426a6e64b034bf2507761dd09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roman=20Pla=CC=81s=CC=8Cil?= Date: Mon, 4 Sep 2017 10:35:45 +0800 Subject: Use SyncJob::SyncData as a plain member --- connection.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'connection.cpp') diff --git a/connection.cpp b/connection.cpp index d2acf928..9fc2f85b 100644 --- a/connection.cpp +++ b/connection.cpp @@ -160,7 +160,7 @@ void Connection::sync(int timeout) auto job = d->syncJob = callApi(d->data->lastEvent(), filter, timeout); connect( job, &SyncJob::success, [=] () { - onSyncSuccess(*job->data()); + onSyncSuccess(job->data()); d->syncJob = nullptr; emit syncDone(); }); -- cgit v1.2.3 From 071856d31995f665ee9219b3e05510ab83f9f4d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roman=20Pla=CC=81s=CC=8Cil?= Date: Mon, 4 Sep 2017 19:23:50 +0800 Subject: Use move on SyncData --- connection.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'connection.cpp') diff --git a/connection.cpp b/connection.cpp index 9fc2f85b..27f0a86f 100644 --- a/connection.cpp +++ b/connection.cpp @@ -160,7 +160,7 @@ void Connection::sync(int timeout) auto job = d->syncJob = callApi(d->data->lastEvent(), filter, timeout); connect( job, &SyncJob::success, [=] () { - onSyncSuccess(job->data()); + onSyncSuccess(job->takeData()); d->syncJob = nullptr; emit syncDone(); }); @@ -174,7 +174,7 @@ void Connection::sync(int timeout) }); } -void Connection::onSyncSuccess(SyncData &data) { +void Connection::onSyncSuccess(SyncData &&data) { d->data->setLastEvent(data.nextBatch()); for( auto&& roomData: data.takeRoomData() ) { @@ -333,8 +333,7 @@ void Connection::saveState(const QUrl &toFile) { QJsonObject rooms; for (auto i : this->roomMap()) { - QJsonObject roomObj = i->toJson(); - rooms[i->id()] = roomObj; + rooms[i->id()] = i->toJson(); } QJsonObject roomObj; @@ -372,5 +371,5 @@ void Connection::loadState(const QUrl &fromFile) { QJsonDocument doc { QJsonDocument::fromJson(data) }; SyncData sync; sync.parseJson(doc); - onSyncSuccess(sync); + onSyncSuccess(std::move(sync)); } -- cgit v1.2.3 From 95e6ae003e3f5ed806bc7adf4e10713cd4e35d1f Mon Sep 17 00:00:00 2001 From: Kitsune Ral Date: Tue, 19 Sep 2017 10:40:31 +0900 Subject: Connection::cacheState property, load/saveState() tweaks and fixes The property controls whether or not the rooms state is cached: if it's off, loadState() and saveState() become no-ops. Other changes: * loadState/saveState properly deal with rooms in Invite state (this is not quite relevant to the current branch but very much is in the light of a concurrent kitsune-invite-kick PR); * Profile loadState/saveState (because dumping and especially parsing JSON takes time); * Use QJsonDocument::Compact layout, it's about 3 times smaller and quicker to parse than Indented, and we really don't care about the cache being human-friendly; * Have a default path for the state cache, based on the connection's user id. --- connection.cpp | 109 ++++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 84 insertions(+), 25 deletions(-) (limited to 'connection.cpp') diff --git a/connection.cpp b/connection.cpp index 27f0a86f..785e0e43 100644 --- a/connection.cpp +++ b/connection.cpp @@ -35,6 +35,9 @@ #include #include #include +#include +#include +#include using namespace QMatrixClient; @@ -60,6 +63,8 @@ class Connection::Private QString userId; SyncJob* syncJob; + + bool cacheState = true; }; Connection::Connection(const QUrl& server, QObject* parent) @@ -329,47 +334,101 @@ QByteArray Connection::generateTxnId() return d->data->generateTxnId(); } -void Connection::saveState(const QUrl &toFile) { - QJsonObject rooms; +void Connection::saveState(const QUrl &toFile) const +{ + if (!d->cacheState) + return; + + QElapsedTimer et; et.start(); - for (auto i : this->roomMap()) { - rooms[i->id()] = i->toJson(); + QFileInfo stateFile { + toFile.isEmpty() ? stateCachePath() : toFile.toLocalFile() + }; + if (!stateFile.dir().exists()) + stateFile.dir().mkpath("."); + + QFile outfile { stateFile.absoluteFilePath() }; + if (!outfile.open(QFile::WriteOnly)) + { + qCWarning(MAIN) << "Error opening" << stateFile.absoluteFilePath() + << ":" << outfile.errorString(); + qCWarning(MAIN) << "Caching the rooms state disabled"; + d->cacheState = false; + return; } QJsonObject roomObj; - roomObj.insert("leave", QJsonObject()); - roomObj.insert("join", rooms); - roomObj.insert("invite", QJsonObject()); + { + QJsonObject rooms; + QJsonObject inviteRooms; + for (auto i : roomMap()) // Pass on rooms in Leave state + { + if (i->joinState() == JoinState::Invite) + inviteRooms.insert(i->id(), i->toJson()); + else + rooms.insert(i->id(), i->toJson()); + } + + if (!rooms.isEmpty()) + roomObj.insert("join", rooms); + if (!inviteRooms.isEmpty()) + roomObj.insert("invite", inviteRooms); + } QJsonObject rootObj; rootObj.insert("next_batch", d->data->lastEvent()); - rootObj.insert("presence", QJsonObject()); rootObj.insert("rooms", roomObj); - QJsonDocument doc { rootObj }; - QByteArray data = doc.toJson(); + QByteArray data = QJsonDocument(rootObj).toJson(QJsonDocument::Compact); - QFileInfo stateFile { toFile.toLocalFile() }; - QFile outfile { stateFile.absoluteFilePath() }; - if (!stateFile.dir().exists()) stateFile.dir().mkpath("."); + qCDebug(MAIN) << "Writing state to file" << outfile.fileName(); + outfile.write(data.data(), data.size()); + qCDebug(PROFILER) << "*** Cached state for" << userId() + << "saved in" << et.elapsed() << "ms"; +} - if (outfile.open(QFile::WriteOnly)) { - qCDebug(MAIN) << "Writing state to file=" << outfile.fileName(); - outfile.write(data.data(), data.size()); +void Connection::loadState(const QUrl &fromFile) +{ + if (!d->cacheState) + return; - } else { - qCWarning(MAIN) << outfile.errorString(); + QElapsedTimer et; et.start(); + QFile file { + fromFile.isEmpty() ? stateCachePath() : fromFile.toLocalFile() + }; + if (!file.exists()) + { + qCDebug(MAIN) << "No state cache file found"; + return; } -} - -void Connection::loadState(const QUrl &fromFile) { - QFile file { fromFile.toLocalFile() }; - if (!file.exists()) return; file.open(QFile::ReadOnly); QByteArray data = file.readAll(); - QJsonDocument doc { QJsonDocument::fromJson(data) }; SyncData sync; - sync.parseJson(doc); + sync.parseJson(QJsonDocument::fromJson(data)); onSyncSuccess(std::move(sync)); + qCDebug(PROFILER) << "*** Cached state for" << userId() + << "loaded in" << et.elapsed() << "ms"; +} + +QString Connection::stateCachePath() const +{ + auto safeUserId = userId(); + safeUserId.replace(':', '_'); + return QStandardPaths::writableLocation(QStandardPaths::CacheLocation) + % '/' % safeUserId % "_state.json"; +} + +bool Connection::cacheState() const +{ + return d->cacheState; +} + +void Connection::setCacheState(bool newValue) +{ + if (d->cacheState != newValue) + { + d->cacheState = newValue; + emit cacheStateChanged(); + } } -- cgit v1.2.3