Make SyncData more self-contained and prepare for cache splitting

SyncData now resides in its own pair of files and is capable to load either from file or from JSON. There is also (yet untested) capability to load rooms from files if a file name stands is the value for a given room id. This allows to store the master cache file separately from cache files for each room, massively easing the problem of bulky accounts that can overflow the poor capacity of Qt's JSON engine.

7 files changed, 277 insertions, 191 deletions

diff --git a/lib/connection.cpp b/lib/connection.cpp
index 6bda932a..8a451a79 100644
--- a/lib/connection.cpp
+++ b/lib/connection.cpp
@@ -39,7 +39,6 @@
 #include <QtNetwork/QDnsLookup>
 #include <QtCore/QFile>
 #include <QtCore/QDir>
-#include <QtCore/QFileInfo>
 #include <QtCore/QStandardPaths>
 #include <QtCore/QStringBuilder>
 #include <QtCore/QElapsedTimer>
@@ -1059,9 +1058,6 @@ void Connection::setHomeserver(const QUrl& url)
     emit homeserverChanged(homeserver());
 }
 
-static constexpr int CACHE_VERSION_MAJOR = 8;
-static constexpr int CACHE_VERSION_MINOR = 0;
-
 void Connection::saveState(const QUrl &toFile) const
 {
     if (!d->cacheState)
@@ -1091,6 +1087,8 @@ void Connection::saveState(const QUrl &toFile) const
         QJsonObject inviteRooms;
         for (const auto* i : roomMap()) // Pass on rooms in Leave state
         {
+            // TODO: instead of adding the room JSON add a file name and save
+            // the JSON to that file.
             if (i->joinState() == JoinState::Invite)
                 inviteRooms.insert(i->id(), i->toJson());
             else
@@ -1123,8 +1121,8 @@ void Connection::saveState(const QUrl &toFile) const
     }
 
     QJsonObject versionObj;
-    versionObj.insert("major", CACHE_VERSION_MAJOR);
-    versionObj.insert("minor", CACHE_VERSION_MINOR);
+    versionObj.insert("major", SyncData::cacheVersion().first);
+    versionObj.insert("minor", SyncData::cacheVersion().second);
     rootObj.insert("cache_version", versionObj);
 
     QJsonDocument json { rootObj };
@@ -1142,42 +1140,20 @@ void Connection::loadState(const QUrl &fromFile)
         return;
 
     QElapsedTimer et; et.start();
-    QFile file {
-        fromFile.isEmpty() ? stateCachePath() : fromFile.toLocalFile()
-    };
-    if (!file.exists())
-    {
-        qCDebug(MAIN) << "No state cache file found";
-        return;
-    }
-    if(!file.open(QFile::ReadOnly))
-    {
-        qCWarning(MAIN) << "file " << file.fileName() << "failed to open for read";
-        return;
-    }
-    QByteArray data = file.readAll();
 
-    auto jsonDoc = d->cacheToBinary ? QJsonDocument::fromBinaryData(data) :
-                                      QJsonDocument::fromJson(data);
-    if (jsonDoc.isNull())
-    {
-        qCWarning(MAIN) << "Cache file broken, discarding";
+    SyncData sync {
+        fromFile.isEmpty() ? stateCachePath() : fromFile.toLocalFile() };
+    if (sync.nextBatch().isEmpty()) // No token means no cache by definition
         return;
-    }
-    auto actualCacheVersionMajor =
-            jsonDoc.object()
-            .value("cache_version").toObject()
-            .value("major").toInt();
-    if (actualCacheVersionMajor < CACHE_VERSION_MAJOR)
+
+    if (!sync.unresolvedRooms().isEmpty())
     {
-        qCWarning(MAIN)
-            << "Major version of the cache file is" << actualCacheVersionMajor
-            << "but" << CACHE_VERSION_MAJOR << "required; discarding the cache";
+        qCWarning(MAIN) << "State cache incomplete, discarding";
         return;
     }
-
-    SyncData sync;
-    sync.parseJson(jsonDoc);
+    // TODO: to handle load failures, instead of the above block:
+    // 1. Do initial sync on failed rooms without saving the nextBatch token
+    // 2. Do the sync across all rooms as normal
     onSyncSuccess(std::move(sync));
     qCDebug(PROFILER) << "*** Cached state for" << userId() << "loaded in" << et;
 }
diff --git a/lib/jobs/syncjob.cpp b/lib/jobs/syncjob.cpp
index 6baf388e..ef9b45dd 100644
--- a/lib/jobs/syncjob.cpp
+++ b/lib/jobs/syncjob.cpp
@@ -18,10 +18,6 @@
 
 #include "syncjob.h"
 
-#include "events/eventloader.h"
-
-#include <QtCore/QElapsedTimer>
-
 using namespace QMatrixClient;
 
 static size_t jobId = 0;
@@ -46,111 +42,15 @@ SyncJob::SyncJob(const QString& since, const QString& filter, int timeout,
     setMaxRetries(std::numeric_limits<int>::max());
 }
 
-QString SyncData::nextBatch() const
-{
-    return nextBatch_;
-}
-
-SyncDataList&& SyncData::takeRoomData()
-{
-    return std::move(roomData);
-}
-
-Events&& SyncData::takePresenceData()
-{
-    return std::move(presenceData);
-}
-
-Events&& SyncData::takeAccountData()
-{
-    return std::move(accountData);
-}
-
-Events&&SyncData::takeToDeviceEvents()
-{
-    return std::move(toDeviceEvents);
-}
-
-template <typename EventsArrayT, typename StrT>
-inline EventsArrayT load(const QJsonObject& batches, StrT keyName)
-{
-    return fromJson<EventsArrayT>(batches[keyName].toObject().value("events"_ls));
-}
-
 BaseJob::Status SyncJob::parseJson(const QJsonDocument& data)
 {
-    return d.parseJson(data);
-}
-
-BaseJob::Status SyncData::parseJson(const QJsonDocument &data)
-{
-    QElapsedTimer et; et.start();
-
-    auto json = data.object();
-    nextBatch_ = json.value("next_batch"_ls).toString();
-    presenceData = load<Events>(json, "presence"_ls);
-    accountData = load<Events>(json, "account_data"_ls);
-    toDeviceEvents = load<Events>(json, "to_device"_ls);
-
-    auto rooms = json.value("rooms"_ls).toObject();
-    JoinStates::Int ii = 1; // ii is used to make a JoinState value
-    auto totalRooms = 0;
-    auto totalEvents = 0;
-    for (size_t i = 0; i < JoinStateStrings.size(); ++i, ii <<= 1)
+    d.parseJson(data.object());
+    if (d.unresolvedRooms().isEmpty())
     {
-        const auto rs = rooms.value(JoinStateStrings[i]).toObject();
-        // We have a Qt container on the right and an STL one on the left
-        roomData.reserve(static_cast<size_t>(rs.size()));
-        for(auto roomIt = rs.begin(); roomIt != rs.end(); ++roomIt)
-        {
-            roomData.emplace_back(roomIt.key(), JoinState(ii),
-                                  roomIt.value().toObject());
-            const auto& r = roomData.back();
-            totalEvents += r.state.size() + r.ephemeral.size() +
-                           r.accountData.size() + r.timeline.size();
-        }
-        totalRooms += rs.size();
+        qCCritical(MAIN) << "Incomplete sync response, missing rooms:"
+                         << d.unresolvedRooms().join(',');
+        return BaseJob::IncorrectResponseError;
     }
-    if (totalRooms > 9 || et.nsecsElapsed() >= profilerMinNsecs())
-        qCDebug(PROFILER) << "*** SyncData::parseJson(): batch with"
-                          << totalRooms << "room(s),"
-                          << totalEvents << "event(s) in" << et;
     return BaseJob::Success;
 }
 
-const QString SyncRoomData::UnreadCountKey =
-        QStringLiteral("x-qmatrixclient.unread_count");
-
-SyncRoomData::SyncRoomData(const QString& roomId_, JoinState joinState_,
-                           const QJsonObject& room_)
-    : roomId(roomId_)
-    , joinState(joinState_)
-    , state(load<StateEvents>(room_,
-                joinState == JoinState::Invite ? "invite_state"_ls : "state"_ls))
-{
-    switch (joinState) {
-        case JoinState::Join:
-            ephemeral = load<Events>(room_, "ephemeral"_ls);
-            FALLTHROUGH;
-        case JoinState::Leave:
-        {
-            accountData = load<Events>(room_, "account_data"_ls);
-            timeline = load<RoomEvents>(room_, "timeline"_ls);
-            const auto timelineJson = room_.value("timeline"_ls).toObject();
-            timelineLimited = timelineJson.value("limited"_ls).toBool();
-            timelinePrevBatch = timelineJson.value("prev_batch"_ls).toString();
-
-            break;
-        }
-        default: /* nothing on top of state */;
-    }
-
-    const auto unreadJson = room_.value("unread_notifications"_ls).toObject();
-    unreadCount = unreadJson.value(UnreadCountKey).toInt(-2);
-    highlightCount = unreadJson.value("highlight_count"_ls).toInt();
-    notificationCount = unreadJson.value("notification_count"_ls).toInt();
-    if (highlightCount > 0 || notificationCount > 0)
-        qCDebug(SYNCJOB) << "Room" << roomId_
-                         << "has highlights:" << highlightCount
-                         << "and notifications:" << notificationCount;
-}
diff --git a/lib/jobs/syncjob.h b/lib/jobs/syncjob.h
index 6b9bedfa..a0a3c026 100644
--- a/lib/jobs/syncjob.h
+++ b/lib/jobs/syncjob.h
@@ -20,56 +20,10 @@
 
 #include "basejob.h"
 
-#include "joinstate.h"
-#include "events/stateevent.h"
-#include "util.h"
+#include "../syncdata.h"
 
 namespace QMatrixClient
 {
-    class SyncRoomData
-    {
-        public:
-            QString roomId;
-            JoinState joinState;
-            StateEvents state;
-            RoomEvents timeline;
-            Events ephemeral;
-            Events accountData;
-
-            bool timelineLimited;
-            QString timelinePrevBatch;
-            int unreadCount;
-            int highlightCount;
-            int notificationCount;
-
-            SyncRoomData(const QString& roomId, JoinState joinState_,
-                         const QJsonObject& room_);
-            SyncRoomData(SyncRoomData&&) = default;
-            SyncRoomData& operator=(SyncRoomData&&) = default;
-
-            static const QString UnreadCountKey;
-    };
-    // QVector cannot work with non-copiable objects, std::vector can.
-    using SyncDataList = std::vector<SyncRoomData>;
-
-    class SyncData
-    {
-        public:
-            BaseJob::Status parseJson(const QJsonDocument &data);
-            Events&& takePresenceData();
-            Events&& takeAccountData();
-            Events&& takeToDeviceEvents();
-            SyncDataList&& takeRoomData();
-            QString nextBatch() const;
-
-        private:
-            QString nextBatch_;
-            Events presenceData;
-            Events accountData;
-            Events toDeviceEvents;
-            SyncDataList roomData;
-    };
-
     class SyncJob: public BaseJob
     {
         public:
diff --git a/lib/room.cpp b/lib/room.cpp
index 656788cb..e5653258 100644
--- a/lib/room.cpp
+++ b/lib/room.cpp
@@ -45,10 +45,10 @@
 #include "connection.h"
 #include "user.h"
 #include "converters.h"
+#include "syncdata.h"
 
 #include <QtCore/QHash>
 #include <QtCore/QStringBuilder> // for efficient string concats (operator%)
-#include <QtCore/QElapsedTimer>
 #include <QtCore/QPointer>
 #include <QtCore/QDir>
 #include <QtCore/QTemporaryFile>
diff --git a/lib/room.h b/lib/room.h
index 633d19dd..b741e229 100644
--- a/lib/room.h
+++ b/lib/room.h
@@ -18,7 +18,6 @@
 
 #pragma once
 
-#include "jobs/syncjob.h"
 #include "csapi/message_pagination.h"
 #include "events/roommessageevent.h"
 #include "events/accountdataevents.h"
@@ -34,6 +33,7 @@
 namespace QMatrixClient
 {
     class Event;
+    class SyncRoomData;
     class RoomMemberEvent;
     class Connection;
     class User;
diff --git a/lib/syncdata.cpp b/lib/syncdata.cpp
new file mode 100644
index 00000000..f0d55fd6
--- /dev/null
+++ b/lib/syncdata.cpp
@@ -0,0 +1,171 @@
+/******************************************************************************
+ * Copyright (C) 2018 Kitsune Ral <kitsune-ral@users.sf.net>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+
+#include "syncdata.h"
+
+#include "events/eventloader.h"
+
+#include <QtCore/QFile>
+
+using namespace QMatrixClient;
+
+const QString SyncRoomData::UnreadCountKey =
+        QStringLiteral("x-qmatrixclient.unread_count");
+
+template <typename EventsArrayT, typename StrT>
+inline EventsArrayT load(const QJsonObject& batches, StrT keyName)
+{
+    return fromJson<EventsArrayT>(batches[keyName].toObject().value("events"_ls));
+}
+
+SyncRoomData::SyncRoomData(const QString& roomId_, JoinState joinState_,
+                           const QJsonObject& room_)
+    : roomId(roomId_)
+    , joinState(joinState_)
+    , state(load<StateEvents>(room_, joinState == JoinState::Invite
+                                     ? "invite_state"_ls : "state"_ls))
+{
+    switch (joinState) {
+        case JoinState::Join:
+            ephemeral = load<Events>(room_, "ephemeral"_ls);
+            FALLTHROUGH;
+        case JoinState::Leave:
+        {
+            accountData = load<Events>(room_, "account_data"_ls);
+            timeline = load<RoomEvents>(room_, "timeline"_ls);
+            const auto timelineJson = room_.value("timeline"_ls).toObject();
+            timelineLimited = timelineJson.value("limited"_ls).toBool();
+            timelinePrevBatch = timelineJson.value("prev_batch"_ls).toString();
+
+            break;
+        }
+        default: /* nothing on top of state */;
+    }
+
+    const auto unreadJson = room_.value("unread_notifications"_ls).toObject();
+    unreadCount = unreadJson.value(UnreadCountKey).toInt(-2);
+    highlightCount = unreadJson.value("highlight_count"_ls).toInt();
+    notificationCount = unreadJson.value("notification_count"_ls).toInt();
+    if (highlightCount > 0 || notificationCount > 0)
+        qCDebug(SYNCJOB) << "Room" << roomId_
+                         << "has highlights:" << highlightCount
+                         << "and notifications:" << notificationCount;
+}
+
+SyncData::SyncData(const QString& cacheFileName)
+{
+    parseJson(loadJson(cacheFileName));
+}
+
+SyncDataList&& SyncData::takeRoomData()
+{
+    return move(roomData);
+}
+
+Events&& SyncData::takePresenceData()
+{
+    return std::move(presenceData);
+}
+
+Events&& SyncData::takeAccountData()
+{
+    return std::move(accountData);
+}
+
+Events&& SyncData::takeToDeviceEvents()
+{
+    return std::move(toDeviceEvents);
+}
+
+QJsonObject SyncData::loadJson(const QString& fileName)
+{
+    QFile roomFile { fileName };
+    if (!roomFile.exists())
+    {
+        qCWarning(MAIN) << "No state cache file" << fileName;
+        return {};
+    }
+    if(!roomFile.open(QIODevice::ReadOnly))
+    {
+        qCWarning(MAIN) << "Failed to open state cache file"
+                        << roomFile.fileName();
+        return {};
+    }
+    auto data = roomFile.readAll();
+
+    const auto json =
+        (data.startsWith('{') ? QJsonDocument::fromJson(data)
+                              : QJsonDocument::fromBinaryData(data)).object();
+    if (json.isEmpty())
+    {
+        qCWarning(MAIN) << "State cache in" << fileName
+                        << "is broken or empty, discarding";
+        return {};
+    }
+    auto requiredVersion = std::get<0>(cacheVersion());
+    auto actualVersion = json.value("cache_version").toObject()
+                             .value("major").toInt();
+    if (actualVersion < requiredVersion)
+    {
+        qCWarning(MAIN)
+            << "Major version of the cache file is" << actualVersion << "but"
+            << requiredVersion << "is required; discarding the cache";
+        return {};
+    }
+    return json;
+}
+
+void SyncData::parseJson(const QJsonObject& json)
+{
+    QElapsedTimer et; et.start();
+
+    nextBatch_ = json.value("next_batch"_ls).toString();
+    presenceData = load<Events>(json, "presence"_ls);
+    accountData = load<Events>(json, "account_data"_ls);
+    toDeviceEvents = load<Events>(json, "to_device"_ls);
+
+    auto rooms = json.value("rooms"_ls).toObject();
+    JoinStates::Int ii = 1; // ii is used to make a JoinState value
+    auto totalRooms = 0;
+    auto totalEvents = 0;
+    for (size_t i = 0; i < JoinStateStrings.size(); ++i, ii <<= 1)
+    {
+        const auto rs = rooms.value(JoinStateStrings[i]).toObject();
+        // We have a Qt container on the right and an STL one on the left
+        roomData.reserve(static_cast<size_t>(rs.size()));
+        for(auto roomIt = rs.begin(); roomIt != rs.end(); ++roomIt)
+        {
+            auto roomJson = roomIt->isString() ? loadJson(roomIt->toString())
+                                               : roomIt->toObject();
+            if (roomJson.isEmpty())
+            {
+                unresolvedRoomIds.push_back(roomIt.key());
+                continue;
+            }
+            roomData.emplace_back(roomIt.key(), JoinState(ii), roomJson);
+            const auto& r = roomData.back();
+            totalEvents += r.state.size() + r.ephemeral.size() +
+                           r.accountData.size() + r.timeline.size();
+        }
+        totalRooms += rs.size();
+    }
+    if (totalRooms > 9 || et.nsecsElapsed() >= profilerMinNsecs())
+        qCDebug(PROFILER) << "*** SyncData::parseJson(): batch with"
+                          << totalRooms << "room(s),"
+                          << totalEvents << "event(s) in" << et;
+}
diff --git a/lib/syncdata.h b/lib/syncdata.h
new file mode 100644
index 00000000..d8007db9
--- /dev/null
+++ b/lib/syncdata.h
@@ -0,0 +1,85 @@
+/******************************************************************************
+ * Copyright (C) 2018 Kitsune Ral <kitsune-ral@users.sf.net>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+
+#pragma once
+
+#include "joinstate.h"
+#include "events/stateevent.h"
+
+namespace QMatrixClient {
+    class SyncRoomData
+    {
+        public:
+            QString roomId;
+            JoinState joinState;
+            StateEvents state;
+            RoomEvents timeline;
+            Events ephemeral;
+            Events accountData;
+
+            bool timelineLimited;
+            QString timelinePrevBatch;
+            int unreadCount;
+            int highlightCount;
+            int notificationCount;
+
+            SyncRoomData(const QString& roomId, JoinState joinState_,
+                         const QJsonObject& room_);
+            SyncRoomData(SyncRoomData&&) = default;
+            SyncRoomData& operator=(SyncRoomData&&) = default;
+
+            static const QString UnreadCountKey;
+    };
+
+    // QVector cannot work with non-copiable objects, std::vector can.
+    using SyncDataList = std::vector<SyncRoomData>;
+
+    class SyncData
+    {
+        public:
+            SyncData() = default;
+            explicit SyncData(const QString& cacheFileName);
+            /** Parse sync response into room events
+             * \param json response from /sync or a room state cache
+             * \return the list of rooms with missing cache files; always
+             *         empty when parsing response from /sync
+             */
+            void parseJson(const QJsonObject& json);
+
+            Events&& takePresenceData();
+            Events&& takeAccountData();
+            Events&& takeToDeviceEvents();
+            SyncDataList&& takeRoomData();
+
+            QString nextBatch() const { return nextBatch_; }
+
+            QStringList unresolvedRooms() const { return unresolvedRoomIds; }
+
+            static std::pair<int, int> cacheVersion() { return { 8, 0 }; }
+
+        private:
+            QString nextBatch_;
+            Events presenceData;
+            Events accountData;
+            Events toDeviceEvents;
+            SyncDataList roomData;
+            QStringList unresolvedRoomIds;
+
+            static QJsonObject loadJson(const QString& fileName);
+    };
+}  // namespace QMatrixClient