Make SyncData more self-contained and prepare for cache splitting

SyncData now resides in its own pair of files and is capable to load either from file or from JSON. There is also (yet untested) capability to load rooms from files if a file name stands is the value for a given room id. This allows to store the master cache file separately from cache files for each room, massively easing the problem of bulky accounts that can overflow the poor capacity of Qt's JSON engine.

2 files changed, 6 insertions, 152 deletions

diff --git a/lib/jobs/syncjob.cpp b/lib/jobs/syncjob.cpp
index 6baf388e..ef9b45dd 100644
--- a/lib/jobs/syncjob.cpp
+++ b/lib/jobs/syncjob.cpp
@@ -18,10 +18,6 @@
 
 #include "syncjob.h"
 
-#include "events/eventloader.h"
-
-#include <QtCore/QElapsedTimer>
-
 using namespace QMatrixClient;
 
 static size_t jobId = 0;
@@ -46,111 +42,15 @@ SyncJob::SyncJob(const QString& since, const QString& filter, int timeout,
     setMaxRetries(std::numeric_limits<int>::max());
 }
 
-QString SyncData::nextBatch() const
-{
-    return nextBatch_;
-}
-
-SyncDataList&& SyncData::takeRoomData()
-{
-    return std::move(roomData);
-}
-
-Events&& SyncData::takePresenceData()
-{
-    return std::move(presenceData);
-}
-
-Events&& SyncData::takeAccountData()
-{
-    return std::move(accountData);
-}
-
-Events&&SyncData::takeToDeviceEvents()
-{
-    return std::move(toDeviceEvents);
-}
-
-template <typename EventsArrayT, typename StrT>
-inline EventsArrayT load(const QJsonObject& batches, StrT keyName)
-{
-    return fromJson<EventsArrayT>(batches[keyName].toObject().value("events"_ls));
-}
-
 BaseJob::Status SyncJob::parseJson(const QJsonDocument& data)
 {
-    return d.parseJson(data);
-}
-
-BaseJob::Status SyncData::parseJson(const QJsonDocument &data)
-{
-    QElapsedTimer et; et.start();
-
-    auto json = data.object();
-    nextBatch_ = json.value("next_batch"_ls).toString();
-    presenceData = load<Events>(json, "presence"_ls);
-    accountData = load<Events>(json, "account_data"_ls);
-    toDeviceEvents = load<Events>(json, "to_device"_ls);
-
-    auto rooms = json.value("rooms"_ls).toObject();
-    JoinStates::Int ii = 1; // ii is used to make a JoinState value
-    auto totalRooms = 0;
-    auto totalEvents = 0;
-    for (size_t i = 0; i < JoinStateStrings.size(); ++i, ii <<= 1)
+    d.parseJson(data.object());
+    if (d.unresolvedRooms().isEmpty())
     {
-        const auto rs = rooms.value(JoinStateStrings[i]).toObject();
-        // We have a Qt container on the right and an STL one on the left
-        roomData.reserve(static_cast<size_t>(rs.size()));
-        for(auto roomIt = rs.begin(); roomIt != rs.end(); ++roomIt)
-        {
-            roomData.emplace_back(roomIt.key(), JoinState(ii),
-                                  roomIt.value().toObject());
-            const auto& r = roomData.back();
-            totalEvents += r.state.size() + r.ephemeral.size() +
-                           r.accountData.size() + r.timeline.size();
-        }
-        totalRooms += rs.size();
+        qCCritical(MAIN) << "Incomplete sync response, missing rooms:"
+                         << d.unresolvedRooms().join(',');
+        return BaseJob::IncorrectResponseError;
     }
-    if (totalRooms > 9 || et.nsecsElapsed() >= profilerMinNsecs())
-        qCDebug(PROFILER) << "*** SyncData::parseJson(): batch with"
-                          << totalRooms << "room(s),"
-                          << totalEvents << "event(s) in" << et;
     return BaseJob::Success;
 }
 
-const QString SyncRoomData::UnreadCountKey =
-        QStringLiteral("x-qmatrixclient.unread_count");
-
-SyncRoomData::SyncRoomData(const QString& roomId_, JoinState joinState_,
-                           const QJsonObject& room_)
-    : roomId(roomId_)
-    , joinState(joinState_)
-    , state(load<StateEvents>(room_,
-                joinState == JoinState::Invite ? "invite_state"_ls : "state"_ls))
-{
-    switch (joinState) {
-        case JoinState::Join:
-            ephemeral = load<Events>(room_, "ephemeral"_ls);
-            FALLTHROUGH;
-        case JoinState::Leave:
-        {
-            accountData = load<Events>(room_, "account_data"_ls);
-            timeline = load<RoomEvents>(room_, "timeline"_ls);
-            const auto timelineJson = room_.value("timeline"_ls).toObject();
-            timelineLimited = timelineJson.value("limited"_ls).toBool();
-            timelinePrevBatch = timelineJson.value("prev_batch"_ls).toString();
-
-            break;
-        }
-        default: /* nothing on top of state */;
-    }
-
-    const auto unreadJson = room_.value("unread_notifications"_ls).toObject();
-    unreadCount = unreadJson.value(UnreadCountKey).toInt(-2);
-    highlightCount = unreadJson.value("highlight_count"_ls).toInt();
-    notificationCount = unreadJson.value("notification_count"_ls).toInt();
-    if (highlightCount > 0 || notificationCount > 0)
-        qCDebug(SYNCJOB) << "Room" << roomId_
-                         << "has highlights:" << highlightCount
-                         << "and notifications:" << notificationCount;
-}
diff --git a/lib/jobs/syncjob.h b/lib/jobs/syncjob.h
index 6b9bedfa..a0a3c026 100644
--- a/lib/jobs/syncjob.h
+++ b/lib/jobs/syncjob.h
@@ -20,56 +20,10 @@
 
 #include "basejob.h"
 
-#include "joinstate.h"
-#include "events/stateevent.h"
-#include "util.h"
+#include "../syncdata.h"
 
 namespace QMatrixClient
 {
-    class SyncRoomData
-    {
-        public:
-            QString roomId;
-            JoinState joinState;
-            StateEvents state;
-            RoomEvents timeline;
-            Events ephemeral;
-            Events accountData;
-
-            bool timelineLimited;
-            QString timelinePrevBatch;
-            int unreadCount;
-            int highlightCount;
-            int notificationCount;
-
-            SyncRoomData(const QString& roomId, JoinState joinState_,
-                         const QJsonObject& room_);
-            SyncRoomData(SyncRoomData&&) = default;
-            SyncRoomData& operator=(SyncRoomData&&) = default;
-
-            static const QString UnreadCountKey;
-    };
-    // QVector cannot work with non-copiable objects, std::vector can.
-    using SyncDataList = std::vector<SyncRoomData>;
-
-    class SyncData
-    {
-        public:
-            BaseJob::Status parseJson(const QJsonDocument &data);
-            Events&& takePresenceData();
-            Events&& takeAccountData();
-            Events&& takeToDeviceEvents();
-            SyncDataList&& takeRoomData();
-            QString nextBatch() const;
-
-        private:
-            QString nextBatch_;
-            Events presenceData;
-            Events accountData;
-            Events toDeviceEvents;
-            SyncDataList roomData;
-    };
-
     class SyncJob: public BaseJob
     {
         public: