From fefa95b86c31f3b7e46fc28327b23050cbadedb9 Mon Sep 17 00:00:00 2001 From: Alexey Rusakov Date: Wed, 23 Jun 2021 13:41:44 +0200 Subject: Update to the new matrix-doc layout and tooling Among things affecting Quotient, the update involved moving API files from api/ to data/api/, adding extensions to event schema files, and switching from ReStructured Text to Markdown as a lightweight markup language. This commit updates the build system and GTAD configuration to accommodate for these. The build system is also more robust now in choosing whether the update-api target should be provided. Previously the target was provided whenever GTAD_PATH and MATRIX_DOC_PATH were specified, even if they did not point to anything valid. CMake now checks that MATRIX_DOC_PATH is an actual directory and that GTAD_PATH points to an actual file. # Conflicts: # CMakeLists.txt --- gtad/gtad.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'gtad') diff --git a/gtad/gtad.yaml b/gtad/gtad.yaml index adf5024a..d68cc8a0 100644 --- a/gtad/gtad.yaml +++ b/gtad/gtad.yaml @@ -85,7 +85,7 @@ analyzer: { type: RoomEventPtr, imports: "events/eventloader.h" } - /event.yaml$/: { type: EventPtr, imports: "events/eventloader.h" } - - /m\.room\.member$/: void # Skip resolving; see EventsArray<> below + - /m\.room\.member/: void # Skip resolving; see EventsArray<> below - '/^(\./)?definitions/request_email_validation.yaml$/': title: EmailValidationData - '/^(\./)?definitions/request_msisdn_validation.yaml$/': @@ -109,7 +109,7 @@ analyzer: - /^Notification|Result$/: type: "std::vector<{{1}}>" imports: "events/eventloader.h" - - /m\.room\.member$/: # Only used in an array (see also above) + - /m\.room\.member/: # Only used in an array (see also above) type: "EventsArray" imports: "events/roommemberevent.h" - /state_event.yaml$/: StateEvents -- cgit v1.2.3 From 65e6ecd9711372e1e2afde769967ee46b3920307 Mon Sep 17 00:00:00 2001 From: Alexey Rusakov Date: Wed, 23 Jun 2021 19:12:26 +0200 Subject: *.mustache: Drop a stray leading end-of-line An SPDX comment in the source code did not collapse entirely. --- gtad/data.h.mustache | 3 +-- gtad/operation.cpp.mustache | 3 +-- gtad/operation.h.mustache | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) (limited to 'gtad') diff --git a/gtad/data.h.mustache b/gtad/data.h.mustache index a2193380..1b511262 100644 --- a/gtad/data.h.mustache +++ b/gtad/data.h.mustache @@ -1,8 +1,7 @@ {{! SPDX-FileCopyrightText: 2020 Kitsune Ral SPDX-License-Identifier: LGPL-2.1-or-later -}} -{{>preamble}} +}}{{>preamble}} #pragma once #include "converters.h" diff --git a/gtad/operation.cpp.mustache b/gtad/operation.cpp.mustache index 1d0ae476..5bbc45ec 100644 --- a/gtad/operation.cpp.mustache +++ b/gtad/operation.cpp.mustache @@ -1,8 +1,7 @@ {{! SPDX-FileCopyrightText: 2020 Kitsune Ral SPDX-License-Identifier: LGPL-2.1-or-later -}} -{{>preamble}} +}}{{>preamble}} #include "{{filenameBase}}.h" #include diff --git a/gtad/operation.h.mustache b/gtad/operation.h.mustache index 135eee55..f91dc66c 100644 --- a/gtad/operation.h.mustache +++ b/gtad/operation.h.mustache @@ -1,8 +1,7 @@ {{! SPDX-FileCopyrightText: 2020 Kitsune Ral SPDX-License-Identifier: LGPL-2.1-or-later -}} -{{>preamble}} +}}{{>preamble}} #pragma once #include "jobs/basejob.h" -- cgit v1.2.3 From ac8a4487e0254386462c43bfc0aeef8412854117 Mon Sep 17 00:00:00 2001 From: Alexey Rusakov Date: Mon, 28 Jun 2021 20:32:14 +0200 Subject: gtad.yaml: update for use with GTAD pre-0.8 --- gtad/gtad.yaml | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) (limited to 'gtad') diff --git a/gtad/gtad.yaml b/gtad/gtad.yaml index d68cc8a0..e56c394d 100644 --- a/gtad/gtad.yaml +++ b/gtad/gtad.yaml @@ -77,28 +77,31 @@ analyzer: +on: - object: &QJsonObject { type: QJsonObject } - $ref: - - +set: { moveOnly: } + - +set: + moveOnly: + imports: '"events/eventloader.h"' +on: - - /state_event.yaml$/: - { type: StateEventPtr, imports: "events/eventloader.h" } - - /room_event.yaml$/: - { type: RoomEventPtr, imports: "events/eventloader.h" } - - /event.yaml$/: - { type: EventPtr, imports: "events/eventloader.h" } + - /state_event.yaml$/: StateEventPtr + - /room_event.yaml$/: RoomEventPtr + - /event.yaml$/: EventPtr - /m\.room\.member/: void # Skip resolving; see EventsArray<> below - - '/^(\./)?definitions/request_email_validation.yaml$/': - title: EmailValidationData - - '/^(\./)?definitions/request_msisdn_validation.yaml$/': - title: MsisdnValidationData - - /_filter.yaml$/: # Event/RoomEventFilters do NOT need Omittable<> - - /public_rooms_response.yaml$/: { _inline: true } - - //: *UseOmittable # Also apply "avoidCopy" to all other ref'ed types + - +set: + # This renderer actually applies to all $ref things + _importRenderer: '"{{#segments}}{{_}}{{#_join}}/{{/_join}}{{/segments}}.h"' + +on: + - '/^(\./)?definitions/request_email_validation.yaml$/': + title: EmailValidationData + - '/^(\./)?definitions/request_msisdn_validation.yaml$/': + title: MsisdnValidationData + - /_filter.yaml$/: # Event/RoomEventFilters do NOT need Omittable<> + - /public_rooms_response.yaml$/: { _inline: true } + - //: *UseOmittable # Also apply "avoidCopy" to all other ref'ed types - schema: - getTurnServer<: *QJsonObject # It's used as an opaque JSON object - PublicRoomResponse: { _inline: true } # - defineFilter>: &Filter # Force folding into a structure # type: Filter -# imports: "csapi/definitions/sync_filter.h" +# imports: '"csapi/definitions/sync_filter.h"' # - getFilter<: *Filter - RoomFilter: # A structure inside Filter, same story as with *_filter.yaml - //: *UseOmittable @@ -108,10 +111,10 @@ analyzer: +on: - /^Notification|Result$/: type: "std::vector<{{1}}>" - imports: "events/eventloader.h" + imports: '"events/eventloader.h"' - /m\.room\.member/: # Only used in an array (see also above) type: "EventsArray" - imports: "events/roommemberevent.h" + imports: '"events/roommemberevent.h"' - /state_event.yaml$/: StateEvents - /room_event.yaml$/: RoomEvents - /event.yaml$/: Events @@ -134,7 +137,6 @@ mustache: # _quote: '"' # Common quote for left and right # _leftQuote: '"' # _rightQuote: '"' -# _joinChar: ',' # The character used by {{_join}} - not working yet _comment: '//' copyrightName: Kitsune Ral copyrightEmail: -- cgit v1.2.3 From 7f4ef8b303c36831a9172a1fc60888e906fc9b93 Mon Sep 17 00:00:00 2001 From: Alexey Rusakov Date: Wed, 30 Jun 2021 20:10:40 +0200 Subject: Add a commented out delimiter override example --- gtad/gtad.yaml | 1 + 1 file changed, 1 insertion(+) (limited to 'gtad') diff --git a/gtad/gtad.yaml b/gtad/gtad.yaml index e56c394d..928a1495 100644 --- a/gtad/gtad.yaml +++ b/gtad/gtad.yaml @@ -132,6 +132,7 @@ analyzer: #operations: mustache: +# delimiter: '%| |%' # or something else instead of '{{ }}' constants: # Syntax elements used by GTAD # _quote: '"' # Common quote for left and right -- cgit v1.2.3 From b64bfa8f72084d9d9397001a735e985a4bf94e56 Mon Sep 17 00:00:00 2001 From: Alexey Rusakov Date: Fri, 2 Jul 2021 10:33:59 +0200 Subject: Abandon BaseJob::Query - Mustache template --- gtad/operation.cpp.mustache | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'gtad') diff --git a/gtad/operation.cpp.mustache b/gtad/operation.cpp.mustache index 5bbc45ec..f34c9280 100644 --- a/gtad/operation.cpp.mustache +++ b/gtad/operation.cpp.mustache @@ -13,7 +13,7 @@ using namespace Quotient; auto queryTo{{camelCaseOperationId}}( {{#queryParams}}{{>joinedParamDef}}{{/queryParams}}) { - BaseJob::Query _q;{{#queryParams}} + QUrlQuery _q;{{#queryParams}} addParam<{{^required?}}IfNotEmpty{{/required?}}>(_q, QStringLiteral("{{baseName}}"), {{paramName}});{{/queryParams}} return _q; -- cgit v1.2.3 From 08d5cef8bbbef65c961370d4a9e2c48a6d7281f1 Mon Sep 17 00:00:00 2001 From: Alexey Rusakov Date: Sat, 7 Aug 2021 22:08:11 +0200 Subject: gtad.yaml: use QUrl where API uses 'format: uri' --- gtad/gtad.yaml | 3 +++ 1 file changed, 3 insertions(+) (limited to 'gtad') diff --git a/gtad/gtad.yaml b/gtad/gtad.yaml index 928a1495..58e1909c 100644 --- a/gtad/gtad.yaml +++ b/gtad/gtad.yaml @@ -68,6 +68,9 @@ analyzer: - dateTime: type: QDateTime initializer: QDateTime::fromString("{{defaultValue}}") + - uri: + type: QUrl + initializer: QUrl::fromEncoded("{{defaultValue}}") - //: &QString type: QString initializer: QStringLiteral("{{defaultValue}}") -- cgit v1.2.3 From 71384a49c3a053e715241172d9d9893bb1742e6b Mon Sep 17 00:00:00 2001 From: Alexey Rusakov Date: Sun, 22 Aug 2021 19:37:09 +0200 Subject: Mustache: avoid BaseJob::Data It's about to be deprecated in the next commits. --- gtad/operation.cpp.mustache | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'gtad') diff --git a/gtad/operation.cpp.mustache b/gtad/operation.cpp.mustache index f34c9280..7f692e4a 100644 --- a/gtad/operation.cpp.mustache +++ b/gtad/operation.cpp.mustache @@ -36,7 +36,7 @@ QUrl {{camelCaseOperationId}}Job::makeRequestUrl(QUrl baseUrl{{#allParams?}}, { {{#headerParams}} setRequestHeader("{{baseName}}", {{paramName}}.toLatin1()); {{/headerParams}}{{#inlineBody}}{{^propertyMap}}{{^bodyParams?}} - setRequestData(Data({{#consumesNonJson?}}{{nameCamelCase}}{{/consumesNonJson? + setRequestData(RequestData({{#consumesNonJson?}}{{nameCamelCase}}{{/consumesNonJson? }}{{^consumesNonJson?}}toJson({{nameCamelCase}}){{/consumesNonJson?}})); {{/bodyParams?}}{{/propertyMap}}{{/inlineBody }}{{^consumesNonJson?}}{{#bodyParams?}} -- cgit v1.2.3 From 7c11f7fddbcc98e4b3b92060c475799d7518624c Mon Sep 17 00:00:00 2001 From: Alexey Rusakov Date: Mon, 4 Oct 2021 18:26:47 +0200 Subject: gtad.yaml: make _rightQuote example less trivial --- gtad/gtad.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'gtad') diff --git a/gtad/gtad.yaml b/gtad/gtad.yaml index 58e1909c..ee8a43fe 100644 --- a/gtad/gtad.yaml +++ b/gtad/gtad.yaml @@ -140,7 +140,7 @@ mustache: # Syntax elements used by GTAD # _quote: '"' # Common quote for left and right # _leftQuote: '"' -# _rightQuote: '"' +# _rightQuote: '"_ls' _comment: '//' copyrightName: Kitsune Ral copyrightEmail: -- cgit v1.2.3 From 96f31d7d8ed1c9ab905c24ac039079aea622f4dc Mon Sep 17 00:00:00 2001 From: Alexey Rusakov Date: Mon, 4 Oct 2021 18:39:21 +0200 Subject: BaseJob: percent-encode variable path parts This is meant to spare clients from having to percent-encode room aliases, v3 event ids etc. that happen to hit the endpoint path. It is unfair to expect clients to do that since they are not supposed to care about the shape of CS API, which parameter should be encoded in which way. The trick (together with the slightly updated GTAD configuration) is to percent-encode parts that happen to be QStrings and not `const char[]`'s while passing all constant parts as plain C character literals. This also allows to make it more certain that the path is correctly encoded by passing and storing QByteArray's wherever the path is already encoded, and only use QStrings (next to const char arrays) before that. Since the change alters the API contract (even if that contract was crappy), some crude detection of percent-encoded stuff on input is inserted; if input is already percent-encoded, a warning is put to the logs, alerting developers about the change. --- gtad/gtad.yaml | 4 ++-- gtad/operation.cpp.mustache | 2 -- lib/jobs/basejob.cpp | 51 +++++++++++++++++++++++++++++++-------------- lib/jobs/basejob.h | 21 ++++++++++++++++--- lib/jobs/syncjob.cpp | 2 +- 5 files changed, 56 insertions(+), 24 deletions(-) (limited to 'gtad') diff --git a/gtad/gtad.yaml b/gtad/gtad.yaml index ee8a43fe..943ac013 100644 --- a/gtad/gtad.yaml +++ b/gtad/gtad.yaml @@ -190,8 +190,8 @@ mustache: joinedParamDef: "{{>maybeCrefType}} {{paramName}}{{>cjoin}}" passPathAndMaybeQuery: >- - QStringLiteral("{{basePathWithoutHost}}") - {{#pathParts}} % {{_}}{{/pathParts}}{{#queryParams?}}, + makePath("{{basePathWithoutHost}}"{{#pathParts}}, + {{_}}{{/pathParts}}){{#queryParams?}}, queryTo{{camelCaseOperationId}}( {{#queryParams}}{{paramName}}{{>cjoin}}{{/queryParams}}){{/queryParams?}} diff --git a/gtad/operation.cpp.mustache b/gtad/operation.cpp.mustache index 7f692e4a..3d26ec73 100644 --- a/gtad/operation.cpp.mustache +++ b/gtad/operation.cpp.mustache @@ -4,8 +4,6 @@ SPDX-License-Identifier: LGPL-2.1-or-later }}{{>preamble}} #include "{{filenameBase}}.h" -#include - using namespace Quotient; {{#operations}}{{#operation}} {{#queryParams?}} diff --git a/lib/jobs/basejob.cpp b/lib/jobs/basejob.cpp index 85066024..73762e4f 100644 --- a/lib/jobs/basejob.cpp +++ b/lib/jobs/basejob.cpp @@ -71,7 +71,7 @@ public: // Using an idiom from clang-tidy: // http://clang.llvm.org/extra/clang-tidy/checks/modernize-pass-by-value.html - Private(HttpVerb v, QString endpoint, const QUrlQuery& q, + Private(HttpVerb v, QByteArray endpoint, const QUrlQuery& q, RequestData&& data, bool nt) : verb(v) , apiEndpoint(std::move(endpoint)) @@ -106,7 +106,7 @@ public: // Contents for the network request HttpVerb verb; - QString apiEndpoint; + QByteArray apiEndpoint; QHash requestHeaders; QUrlQuery requestQuery; RequestData requestData; @@ -166,14 +166,36 @@ public: } }; -BaseJob::BaseJob(HttpVerb verb, const QString& name, const QString& endpoint, +inline bool isHex(QChar c) +{ + return c.isDigit() || (c >= u'A' && c <= u'F') || (c >= u'a' && c <= u'f'); +} + +QByteArray BaseJob::encodeIfParam(const QString& paramPart) +{ + const auto percentIndex = paramPart.indexOf('%'); + if (percentIndex != -1 && paramPart.size() > percentIndex + 2 + && isHex(paramPart[percentIndex + 1]) + && isHex(paramPart[percentIndex + 2])) { + qCWarning(JOBS) + << "Developers, upfront percent-encoding of job parameters is " + "deprecated since libQuotient 0.7; the string involved is" + << paramPart; + return QUrl(paramPart, QUrl::TolerantMode).toEncoded(); + } + return QUrl::toPercentEncoding(paramPart); +} + +BaseJob::BaseJob(HttpVerb verb, const QString& name, QByteArray endpoint, bool needsToken) - : BaseJob(verb, name, endpoint, QUrlQuery {}, RequestData {}, needsToken) + : BaseJob(verb, name, std::move(endpoint), QUrlQuery {}, RequestData {}, + needsToken) {} -BaseJob::BaseJob(HttpVerb verb, const QString& name, const QString& endpoint, +BaseJob::BaseJob(HttpVerb verb, const QString& name, QByteArray endpoint, const QUrlQuery& query, RequestData&& data, bool needsToken) - : d(new Private(verb, endpoint, query, std::move(data), needsToken)) + : d(new Private(verb, std::move(endpoint), query, std::move(data), + needsToken)) { setObjectName(name); connect(&d->timer, &QTimer::timeout, this, &BaseJob::timeout); @@ -194,13 +216,6 @@ QUrl BaseJob::requestUrl() const { return d->reply ? d->reply->url() : QUrl(); } bool BaseJob::isBackground() const { return d->inBackground; } -//const QString& BaseJob::apiEndpoint() const { return d->apiUrl.path(); } - -//void BaseJob::setApiEndpoint(const QString& apiEndpoint) -//{ -// d->apiEndpoint = apiEndpoint; -//} - const BaseJob::headers_t& BaseJob::requestHeaders() const { return d->requestHeaders; @@ -259,13 +274,17 @@ const QNetworkReply* BaseJob::reply() const { return d->reply.data(); } QNetworkReply* BaseJob::reply() { return d->reply.data(); } -QUrl BaseJob::makeRequestUrl(QUrl baseUrl, const QString& path, +QUrl BaseJob::makeRequestUrl(QUrl baseUrl, const QByteArray& encodedPath, const QUrlQuery& query) { // Make sure the added path is relative even if it's not (the official // API definitions have the leading slash though it's not really correct). - baseUrl = baseUrl.resolved( - QUrl(path.mid(path.startsWith('/')), QUrl::TolerantMode)); + const auto pathUrl = + QUrl::fromEncoded(encodedPath.mid(encodedPath.startsWith('/')), + QUrl::StrictMode); + Q_ASSERT_X(pathUrl.isValid(), __FUNCTION__, + qPrintable(pathUrl.errorString())); + baseUrl = baseUrl.resolved(pathUrl); baseUrl.setQuery(query); return baseUrl; } diff --git a/lib/jobs/basejob.h b/lib/jobs/basejob.h index 663c121c..81455307 100644 --- a/lib/jobs/basejob.h +++ b/lib/jobs/basejob.h @@ -9,6 +9,7 @@ #include "../converters.h" #include +#include class QNetworkReply; class QSslError; @@ -23,6 +24,14 @@ class BaseJob : public QObject { Q_PROPERTY(QUrl requestUrl READ requestUrl CONSTANT) Q_PROPERTY(int maxRetries READ maxRetries WRITE setMaxRetries) Q_PROPERTY(int statusCode READ error NOTIFY statusChanged) + + static QByteArray encodeIfParam(const QString& paramPart); + template + static inline auto encodeIfParam(const char (&constPart)[N]) + { + return constPart; + } + public: /*! The status code of a job * @@ -70,6 +79,12 @@ public: }; Q_ENUM(StatusCode) + template + static QByteArray makePath(StrTs&&... parts) + { + return (QByteArray() % ... % encodeIfParam(parts)); + } + using Data #ifndef Q_CC_MSVC Q_DECL_DEPRECATED_X("Use Quotient::RequestData instead") @@ -124,9 +139,9 @@ public: }; public: - BaseJob(HttpVerb verb, const QString& name, const QString& endpoint, + BaseJob(HttpVerb verb, const QString& name, QByteArray endpoint, bool needsToken = true); - BaseJob(HttpVerb verb, const QString& name, const QString& endpoint, + BaseJob(HttpVerb verb, const QString& name, QByteArray endpoint, const QUrlQuery& query, RequestData&& data = {}, bool needsToken = true); @@ -352,7 +367,7 @@ protected: * The function ensures exactly one '/' between the path component of * \p baseUrl and \p path. The query component of \p baseUrl is ignored. */ - static QUrl makeRequestUrl(QUrl baseUrl, const QString& path, + static QUrl makeRequestUrl(QUrl baseUrl, const QByteArray &encodedPath, const QUrlQuery& query = {}); /*! Prepares the job for execution diff --git a/lib/jobs/syncjob.cpp b/lib/jobs/syncjob.cpp index 59a34ef3..9b1b46f0 100644 --- a/lib/jobs/syncjob.cpp +++ b/lib/jobs/syncjob.cpp @@ -10,7 +10,7 @@ static size_t jobId = 0; SyncJob::SyncJob(const QString& since, const QString& filter, int timeout, const QString& presence) : BaseJob(HttpVerb::Get, QStringLiteral("SyncJob-%1").arg(++jobId), - QStringLiteral("_matrix/client/r0/sync")) + "_matrix/client/r0/sync") { setLoggingCategory(SYNCJOB); QUrlQuery query; -- cgit v1.2.3