Make SyncData more self-contained and prepare for cache splitting

SyncData now resides in its own pair of files and is capable to load either from file or from JSON. There is also (yet untested) capability to load rooms from files if a file name stands is the value for a given room id. This allows to store the master cache file separately from cache files for each room, massively easing the problem of bulky accounts that can overflow the poor capacity of Qt's JSON engine.

1 files changed, 171 insertions, 0 deletions

diff --git a/lib/syncdata.cpp b/lib/syncdata.cpp
new file mode 100644
index 00000000..f0d55fd6
--- /dev/null
+++ b/lib/syncdata.cpp
@@ -0,0 +1,171 @@
+/******************************************************************************
+ * Copyright (C) 2018 Kitsune Ral <kitsune-ral@users.sf.net>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+
+#include "syncdata.h"
+
+#include "events/eventloader.h"
+
+#include <QtCore/QFile>
+
+using namespace QMatrixClient;
+
+const QString SyncRoomData::UnreadCountKey =
+        QStringLiteral("x-qmatrixclient.unread_count");
+
+template <typename EventsArrayT, typename StrT>
+inline EventsArrayT load(const QJsonObject& batches, StrT keyName)
+{
+    return fromJson<EventsArrayT>(batches[keyName].toObject().value("events"_ls));
+}
+
+SyncRoomData::SyncRoomData(const QString& roomId_, JoinState joinState_,
+                           const QJsonObject& room_)
+    : roomId(roomId_)
+    , joinState(joinState_)
+    , state(load<StateEvents>(room_, joinState == JoinState::Invite
+                                     ? "invite_state"_ls : "state"_ls))
+{
+    switch (joinState) {
+        case JoinState::Join:
+            ephemeral = load<Events>(room_, "ephemeral"_ls);
+            FALLTHROUGH;
+        case JoinState::Leave:
+        {
+            accountData = load<Events>(room_, "account_data"_ls);
+            timeline = load<RoomEvents>(room_, "timeline"_ls);
+            const auto timelineJson = room_.value("timeline"_ls).toObject();
+            timelineLimited = timelineJson.value("limited"_ls).toBool();
+            timelinePrevBatch = timelineJson.value("prev_batch"_ls).toString();
+
+            break;
+        }
+        default: /* nothing on top of state */;
+    }
+
+    const auto unreadJson = room_.value("unread_notifications"_ls).toObject();
+    unreadCount = unreadJson.value(UnreadCountKey).toInt(-2);
+    highlightCount = unreadJson.value("highlight_count"_ls).toInt();
+    notificationCount = unreadJson.value("notification_count"_ls).toInt();
+    if (highlightCount > 0 || notificationCount > 0)
+        qCDebug(SYNCJOB) << "Room" << roomId_
+                         << "has highlights:" << highlightCount
+                         << "and notifications:" << notificationCount;
+}
+
+SyncData::SyncData(const QString& cacheFileName)
+{
+    parseJson(loadJson(cacheFileName));
+}
+
+SyncDataList&& SyncData::takeRoomData()
+{
+    return move(roomData);
+}
+
+Events&& SyncData::takePresenceData()
+{
+    return std::move(presenceData);
+}
+
+Events&& SyncData::takeAccountData()
+{
+    return std::move(accountData);
+}
+
+Events&& SyncData::takeToDeviceEvents()
+{
+    return std::move(toDeviceEvents);
+}
+
+QJsonObject SyncData::loadJson(const QString& fileName)
+{
+    QFile roomFile { fileName };
+    if (!roomFile.exists())
+    {
+        qCWarning(MAIN) << "No state cache file" << fileName;
+        return {};
+    }
+    if(!roomFile.open(QIODevice::ReadOnly))
+    {
+        qCWarning(MAIN) << "Failed to open state cache file"
+                        << roomFile.fileName();
+        return {};
+    }
+    auto data = roomFile.readAll();
+
+    const auto json =
+        (data.startsWith('{') ? QJsonDocument::fromJson(data)
+                              : QJsonDocument::fromBinaryData(data)).object();
+    if (json.isEmpty())
+    {
+        qCWarning(MAIN) << "State cache in" << fileName
+                        << "is broken or empty, discarding";
+        return {};
+    }
+    auto requiredVersion = std::get<0>(cacheVersion());
+    auto actualVersion = json.value("cache_version").toObject()
+                             .value("major").toInt();
+    if (actualVersion < requiredVersion)
+    {
+        qCWarning(MAIN)
+            << "Major version of the cache file is" << actualVersion << "but"
+            << requiredVersion << "is required; discarding the cache";
+        return {};
+    }
+    return json;
+}
+
+void SyncData::parseJson(const QJsonObject& json)
+{
+    QElapsedTimer et; et.start();
+
+    nextBatch_ = json.value("next_batch"_ls).toString();
+    presenceData = load<Events>(json, "presence"_ls);
+    accountData = load<Events>(json, "account_data"_ls);
+    toDeviceEvents = load<Events>(json, "to_device"_ls);
+
+    auto rooms = json.value("rooms"_ls).toObject();
+    JoinStates::Int ii = 1; // ii is used to make a JoinState value
+    auto totalRooms = 0;
+    auto totalEvents = 0;
+    for (size_t i = 0; i < JoinStateStrings.size(); ++i, ii <<= 1)
+    {
+        const auto rs = rooms.value(JoinStateStrings[i]).toObject();
+        // We have a Qt container on the right and an STL one on the left
+        roomData.reserve(static_cast<size_t>(rs.size()));
+        for(auto roomIt = rs.begin(); roomIt != rs.end(); ++roomIt)
+        {
+            auto roomJson = roomIt->isString() ? loadJson(roomIt->toString())
+                                               : roomIt->toObject();
+            if (roomJson.isEmpty())
+            {
+                unresolvedRoomIds.push_back(roomIt.key());
+                continue;
+            }
+            roomData.emplace_back(roomIt.key(), JoinState(ii), roomJson);
+            const auto& r = roomData.back();
+            totalEvents += r.state.size() + r.ephemeral.size() +
+                           r.accountData.size() + r.timeline.size();
+        }
+        totalRooms += rs.size();
+    }
+    if (totalRooms > 9 || et.nsecsElapsed() >= profilerMinNsecs())
+        qCDebug(PROFILER) << "*** SyncData::parseJson(): batch with"
+                          << totalRooms << "room(s),"
+                          << totalEvents << "event(s) in" << et;
+}