diff options
| -rw-r--r-- | CMakeLists.txt | 1 | ||||
| -rw-r--r-- | lib/connection.cpp | 50 | ||||
| -rw-r--r-- | lib/jobs/syncjob.cpp | 110 | ||||
| -rw-r--r-- | lib/jobs/syncjob.h | 48 | ||||
| -rw-r--r-- | lib/room.cpp | 2 | ||||
| -rw-r--r-- | lib/room.h | 2 | ||||
| -rw-r--r-- | lib/syncdata.cpp | 171 | ||||
| -rw-r--r-- | lib/syncdata.h | 85 |
8 files changed, 278 insertions, 191 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 7e3eb600..49c5d8b2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -76,6 +76,7 @@ set(libqmatrixclient_SRCS lib/room.cpp lib/user.cpp lib/avatar.cpp + lib/syncdata.cpp lib/settings.cpp lib/networksettings.cpp lib/converters.cpp diff --git a/lib/connection.cpp b/lib/connection.cpp index 6bda932a..8a451a79 100644 --- a/lib/connection.cpp +++ b/lib/connection.cpp @@ -39,7 +39,6 @@ #include <QtNetwork/QDnsLookup> #include <QtCore/QFile> #include <QtCore/QDir> -#include <QtCore/QFileInfo> #include <QtCore/QStandardPaths> #include <QtCore/QStringBuilder> #include <QtCore/QElapsedTimer> @@ -1059,9 +1058,6 @@ void Connection::setHomeserver(const QUrl& url) emit homeserverChanged(homeserver()); } -static constexpr int CACHE_VERSION_MAJOR = 8; -static constexpr int CACHE_VERSION_MINOR = 0; - void Connection::saveState(const QUrl &toFile) const { if (!d->cacheState) @@ -1091,6 +1087,8 @@ void Connection::saveState(const QUrl &toFile) const QJsonObject inviteRooms; for (const auto* i : roomMap()) // Pass on rooms in Leave state { + // TODO: instead of adding the room JSON add a file name and save + // the JSON to that file. if (i->joinState() == JoinState::Invite) inviteRooms.insert(i->id(), i->toJson()); else @@ -1123,8 +1121,8 @@ void Connection::saveState(const QUrl &toFile) const } QJsonObject versionObj; - versionObj.insert("major", CACHE_VERSION_MAJOR); - versionObj.insert("minor", CACHE_VERSION_MINOR); + versionObj.insert("major", SyncData::cacheVersion().first); + versionObj.insert("minor", SyncData::cacheVersion().second); rootObj.insert("cache_version", versionObj); QJsonDocument json { rootObj }; @@ -1142,42 +1140,20 @@ void Connection::loadState(const QUrl &fromFile) return; QElapsedTimer et; et.start(); - QFile file { - fromFile.isEmpty() ? stateCachePath() : fromFile.toLocalFile() - }; - if (!file.exists()) - { - qCDebug(MAIN) << "No state cache file found"; - return; - } - if(!file.open(QFile::ReadOnly)) - { - qCWarning(MAIN) << "file " << file.fileName() << "failed to open for read"; - return; - } - QByteArray data = file.readAll(); - auto jsonDoc = d->cacheToBinary ? QJsonDocument::fromBinaryData(data) : - QJsonDocument::fromJson(data); - if (jsonDoc.isNull()) - { - qCWarning(MAIN) << "Cache file broken, discarding"; + SyncData sync { + fromFile.isEmpty() ? stateCachePath() : fromFile.toLocalFile() }; + if (sync.nextBatch().isEmpty()) // No token means no cache by definition return; - } - auto actualCacheVersionMajor = - jsonDoc.object() - .value("cache_version").toObject() - .value("major").toInt(); - if (actualCacheVersionMajor < CACHE_VERSION_MAJOR) + + if (!sync.unresolvedRooms().isEmpty()) { - qCWarning(MAIN) - << "Major version of the cache file is" << actualCacheVersionMajor - << "but" << CACHE_VERSION_MAJOR << "required; discarding the cache"; + qCWarning(MAIN) << "State cache incomplete, discarding"; return; } - - SyncData sync; - sync.parseJson(jsonDoc); + // TODO: to handle load failures, instead of the above block: + // 1. Do initial sync on failed rooms without saving the nextBatch token + // 2. Do the sync across all rooms as normal onSyncSuccess(std::move(sync)); qCDebug(PROFILER) << "*** Cached state for" << userId() << "loaded in" << et; } diff --git a/lib/jobs/syncjob.cpp b/lib/jobs/syncjob.cpp index 6baf388e..ef9b45dd 100644 --- a/lib/jobs/syncjob.cpp +++ b/lib/jobs/syncjob.cpp @@ -18,10 +18,6 @@ #include "syncjob.h" -#include "events/eventloader.h" - -#include <QtCore/QElapsedTimer> - using namespace QMatrixClient; static size_t jobId = 0; @@ -46,111 +42,15 @@ SyncJob::SyncJob(const QString& since, const QString& filter, int timeout, setMaxRetries(std::numeric_limits<int>::max()); } -QString SyncData::nextBatch() const -{ - return nextBatch_; -} - -SyncDataList&& SyncData::takeRoomData() -{ - return std::move(roomData); -} - -Events&& SyncData::takePresenceData() -{ - return std::move(presenceData); -} - -Events&& SyncData::takeAccountData() -{ - return std::move(accountData); -} - -Events&&SyncData::takeToDeviceEvents() -{ - return std::move(toDeviceEvents); -} - -template <typename EventsArrayT, typename StrT> -inline EventsArrayT load(const QJsonObject& batches, StrT keyName) -{ - return fromJson<EventsArrayT>(batches[keyName].toObject().value("events"_ls)); -} - BaseJob::Status SyncJob::parseJson(const QJsonDocument& data) { - return d.parseJson(data); -} - -BaseJob::Status SyncData::parseJson(const QJsonDocument &data) -{ - QElapsedTimer et; et.start(); - - auto json = data.object(); - nextBatch_ = json.value("next_batch"_ls).toString(); - presenceData = load<Events>(json, "presence"_ls); - accountData = load<Events>(json, "account_data"_ls); - toDeviceEvents = load<Events>(json, "to_device"_ls); - - auto rooms = json.value("rooms"_ls).toObject(); - JoinStates::Int ii = 1; // ii is used to make a JoinState value - auto totalRooms = 0; - auto totalEvents = 0; - for (size_t i = 0; i < JoinStateStrings.size(); ++i, ii <<= 1) + d.parseJson(data.object()); + if (d.unresolvedRooms().isEmpty()) { - const auto rs = rooms.value(JoinStateStrings[i]).toObject(); - // We have a Qt container on the right and an STL one on the left - roomData.reserve(static_cast<size_t>(rs.size())); - for(auto roomIt = rs.begin(); roomIt != rs.end(); ++roomIt) - { - roomData.emplace_back(roomIt.key(), JoinState(ii), - roomIt.value().toObject()); - const auto& r = roomData.back(); - totalEvents += r.state.size() + r.ephemeral.size() + - r.accountData.size() + r.timeline.size(); - } - totalRooms += rs.size(); + qCCritical(MAIN) << "Incomplete sync response, missing rooms:" + << d.unresolvedRooms().join(','); + return BaseJob::IncorrectResponseError; } - if (totalRooms > 9 || et.nsecsElapsed() >= profilerMinNsecs()) - qCDebug(PROFILER) << "*** SyncData::parseJson(): batch with" - << totalRooms << "room(s)," - << totalEvents << "event(s) in" << et; return BaseJob::Success; } -const QString SyncRoomData::UnreadCountKey = - QStringLiteral("x-qmatrixclient.unread_count"); - -SyncRoomData::SyncRoomData(const QString& roomId_, JoinState joinState_, - const QJsonObject& room_) - : roomId(roomId_) - , joinState(joinState_) - , state(load<StateEvents>(room_, - joinState == JoinState::Invite ? "invite_state"_ls : "state"_ls)) -{ - switch (joinState) { - case JoinState::Join: - ephemeral = load<Events>(room_, "ephemeral"_ls); - FALLTHROUGH; - case JoinState::Leave: - { - accountData = load<Events>(room_, "account_data"_ls); - timeline = load<RoomEvents>(room_, "timeline"_ls); - const auto timelineJson = room_.value("timeline"_ls).toObject(); - timelineLimited = timelineJson.value("limited"_ls).toBool(); - timelinePrevBatch = timelineJson.value("prev_batch"_ls).toString(); - - break; - } - default: /* nothing on top of state */; - } - - const auto unreadJson = room_.value("unread_notifications"_ls).toObject(); - unreadCount = unreadJson.value(UnreadCountKey).toInt(-2); - highlightCount = unreadJson.value("highlight_count"_ls).toInt(); - notificationCount = unreadJson.value("notification_count"_ls).toInt(); - if (highlightCount > 0 || notificationCount > 0) - qCDebug(SYNCJOB) << "Room" << roomId_ - << "has highlights:" << highlightCount - << "and notifications:" << notificationCount; -} diff --git a/lib/jobs/syncjob.h b/lib/jobs/syncjob.h index 6b9bedfa..a0a3c026 100644 --- a/lib/jobs/syncjob.h +++ b/lib/jobs/syncjob.h @@ -20,56 +20,10 @@ #include "basejob.h" -#include "joinstate.h" -#include "events/stateevent.h" -#include "util.h" +#include "../syncdata.h" namespace QMatrixClient { - class SyncRoomData - { - public: - QString roomId; - JoinState joinState; - StateEvents state; - RoomEvents timeline; - Events ephemeral; - Events accountData; - - bool timelineLimited; - QString timelinePrevBatch; - int unreadCount; - int highlightCount; - int notificationCount; - - SyncRoomData(const QString& roomId, JoinState joinState_, - const QJsonObject& room_); - SyncRoomData(SyncRoomData&&) = default; - SyncRoomData& operator=(SyncRoomData&&) = default; - - static const QString UnreadCountKey; - }; - // QVector cannot work with non-copiable objects, std::vector can. - using SyncDataList = std::vector<SyncRoomData>; - - class SyncData - { - public: - BaseJob::Status parseJson(const QJsonDocument &data); - Events&& takePresenceData(); - Events&& takeAccountData(); - Events&& takeToDeviceEvents(); - SyncDataList&& takeRoomData(); - QString nextBatch() const; - - private: - QString nextBatch_; - Events presenceData; - Events accountData; - Events toDeviceEvents; - SyncDataList roomData; - }; - class SyncJob: public BaseJob { public: diff --git a/lib/room.cpp b/lib/room.cpp index 656788cb..e5653258 100644 --- a/lib/room.cpp +++ b/lib/room.cpp @@ -45,10 +45,10 @@ #include "connection.h" #include "user.h" #include "converters.h" +#include "syncdata.h" #include <QtCore/QHash> #include <QtCore/QStringBuilder> // for efficient string concats (operator%) -#include <QtCore/QElapsedTimer> #include <QtCore/QPointer> #include <QtCore/QDir> #include <QtCore/QTemporaryFile> @@ -18,7 +18,6 @@ #pragma once -#include "jobs/syncjob.h" #include "csapi/message_pagination.h" #include "events/roommessageevent.h" #include "events/accountdataevents.h" @@ -34,6 +33,7 @@ namespace QMatrixClient { class Event; + class SyncRoomData; class RoomMemberEvent; class Connection; class User; diff --git a/lib/syncdata.cpp b/lib/syncdata.cpp new file mode 100644 index 00000000..f0d55fd6 --- /dev/null +++ b/lib/syncdata.cpp @@ -0,0 +1,171 @@ +/****************************************************************************** + * Copyright (C) 2018 Kitsune Ral <kitsune-ral@users.sf.net> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "syncdata.h" + +#include "events/eventloader.h" + +#include <QtCore/QFile> + +using namespace QMatrixClient; + +const QString SyncRoomData::UnreadCountKey = + QStringLiteral("x-qmatrixclient.unread_count"); + +template <typename EventsArrayT, typename StrT> +inline EventsArrayT load(const QJsonObject& batches, StrT keyName) +{ + return fromJson<EventsArrayT>(batches[keyName].toObject().value("events"_ls)); +} + +SyncRoomData::SyncRoomData(const QString& roomId_, JoinState joinState_, + const QJsonObject& room_) + : roomId(roomId_) + , joinState(joinState_) + , state(load<StateEvents>(room_, joinState == JoinState::Invite + ? "invite_state"_ls : "state"_ls)) +{ + switch (joinState) { + case JoinState::Join: + ephemeral = load<Events>(room_, "ephemeral"_ls); + FALLTHROUGH; + case JoinState::Leave: + { + accountData = load<Events>(room_, "account_data"_ls); + timeline = load<RoomEvents>(room_, "timeline"_ls); + const auto timelineJson = room_.value("timeline"_ls).toObject(); + timelineLimited = timelineJson.value("limited"_ls).toBool(); + timelinePrevBatch = timelineJson.value("prev_batch"_ls).toString(); + + break; + } + default: /* nothing on top of state */; + } + + const auto unreadJson = room_.value("unread_notifications"_ls).toObject(); + unreadCount = unreadJson.value(UnreadCountKey).toInt(-2); + highlightCount = unreadJson.value("highlight_count"_ls).toInt(); + notificationCount = unreadJson.value("notification_count"_ls).toInt(); + if (highlightCount > 0 || notificationCount > 0) + qCDebug(SYNCJOB) << "Room" << roomId_ + << "has highlights:" << highlightCount + << "and notifications:" << notificationCount; +} + +SyncData::SyncData(const QString& cacheFileName) +{ + parseJson(loadJson(cacheFileName)); +} + +SyncDataList&& SyncData::takeRoomData() +{ + return move(roomData); +} + +Events&& SyncData::takePresenceData() +{ + return std::move(presenceData); +} + +Events&& SyncData::takeAccountData() +{ + return std::move(accountData); +} + +Events&& SyncData::takeToDeviceEvents() +{ + return std::move(toDeviceEvents); +} + +QJsonObject SyncData::loadJson(const QString& fileName) +{ + QFile roomFile { fileName }; + if (!roomFile.exists()) + { + qCWarning(MAIN) << "No state cache file" << fileName; + return {}; + } + if(!roomFile.open(QIODevice::ReadOnly)) + { + qCWarning(MAIN) << "Failed to open state cache file" + << roomFile.fileName(); + return {}; + } + auto data = roomFile.readAll(); + + const auto json = + (data.startsWith('{') ? QJsonDocument::fromJson(data) + : QJsonDocument::fromBinaryData(data)).object(); + if (json.isEmpty()) + { + qCWarning(MAIN) << "State cache in" << fileName + << "is broken or empty, discarding"; + return {}; + } + auto requiredVersion = std::get<0>(cacheVersion()); + auto actualVersion = json.value("cache_version").toObject() + .value("major").toInt(); + if (actualVersion < requiredVersion) + { + qCWarning(MAIN) + << "Major version of the cache file is" << actualVersion << "but" + << requiredVersion << "is required; discarding the cache"; + return {}; + } + return json; +} + +void SyncData::parseJson(const QJsonObject& json) +{ + QElapsedTimer et; et.start(); + + nextBatch_ = json.value("next_batch"_ls).toString(); + presenceData = load<Events>(json, "presence"_ls); + accountData = load<Events>(json, "account_data"_ls); + toDeviceEvents = load<Events>(json, "to_device"_ls); + + auto rooms = json.value("rooms"_ls).toObject(); + JoinStates::Int ii = 1; // ii is used to make a JoinState value + auto totalRooms = 0; + auto totalEvents = 0; + for (size_t i = 0; i < JoinStateStrings.size(); ++i, ii <<= 1) + { + const auto rs = rooms.value(JoinStateStrings[i]).toObject(); + // We have a Qt container on the right and an STL one on the left + roomData.reserve(static_cast<size_t>(rs.size())); + for(auto roomIt = rs.begin(); roomIt != rs.end(); ++roomIt) + { + auto roomJson = roomIt->isString() ? loadJson(roomIt->toString()) + : roomIt->toObject(); + if (roomJson.isEmpty()) + { + unresolvedRoomIds.push_back(roomIt.key()); + continue; + } + roomData.emplace_back(roomIt.key(), JoinState(ii), roomJson); + const auto& r = roomData.back(); + totalEvents += r.state.size() + r.ephemeral.size() + + r.accountData.size() + r.timeline.size(); + } + totalRooms += rs.size(); + } + if (totalRooms > 9 || et.nsecsElapsed() >= profilerMinNsecs()) + qCDebug(PROFILER) << "*** SyncData::parseJson(): batch with" + << totalRooms << "room(s)," + << totalEvents << "event(s) in" << et; +} diff --git a/lib/syncdata.h b/lib/syncdata.h new file mode 100644 index 00000000..d8007db9 --- /dev/null +++ b/lib/syncdata.h @@ -0,0 +1,85 @@ +/****************************************************************************** + * Copyright (C) 2018 Kitsune Ral <kitsune-ral@users.sf.net> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#pragma once + +#include "joinstate.h" +#include "events/stateevent.h" + +namespace QMatrixClient { + class SyncRoomData + { + public: + QString roomId; + JoinState joinState; + StateEvents state; + RoomEvents timeline; + Events ephemeral; + Events accountData; + + bool timelineLimited; + QString timelinePrevBatch; + int unreadCount; + int highlightCount; + int notificationCount; + + SyncRoomData(const QString& roomId, JoinState joinState_, + const QJsonObject& room_); + SyncRoomData(SyncRoomData&&) = default; + SyncRoomData& operator=(SyncRoomData&&) = default; + + static const QString UnreadCountKey; + }; + + // QVector cannot work with non-copiable objects, std::vector can. + using SyncDataList = std::vector<SyncRoomData>; + + class SyncData + { + public: + SyncData() = default; + explicit SyncData(const QString& cacheFileName); + /** Parse sync response into room events + * \param json response from /sync or a room state cache + * \return the list of rooms with missing cache files; always + * empty when parsing response from /sync + */ + void parseJson(const QJsonObject& json); + + Events&& takePresenceData(); + Events&& takeAccountData(); + Events&& takeToDeviceEvents(); + SyncDataList&& takeRoomData(); + + QString nextBatch() const { return nextBatch_; } + + QStringList unresolvedRooms() const { return unresolvedRoomIds; } + + static std::pair<int, int> cacheVersion() { return { 8, 0 }; } + + private: + QString nextBatch_; + Events presenceData; + Events accountData; + Events toDeviceEvents; + SyncDataList roomData; + QStringList unresolvedRoomIds; + + static QJsonObject loadJson(const QString& fileName); + }; +} // namespace QMatrixClient |