aboutsummaryrefslogtreecommitdiff
path: root/lib/jobs/basejob.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'lib/jobs/basejob.cpp')
-rw-r--r--lib/jobs/basejob.cpp896
1 files changed, 522 insertions, 374 deletions
diff --git a/lib/jobs/basejob.cpp b/lib/jobs/basejob.cpp
index 0e6a8403..5960203d 100644
--- a/lib/jobs/basejob.cpp
+++ b/lib/jobs/basejob.cpp
@@ -13,92 +13,195 @@
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "basejob.h"
#include "connectiondata.h"
-#include "util.h"
+#include <QtCore/QRegularExpression>
+#include <QtCore/QTimer>
+#include <QtCore/QStringBuilder>
+#include <QtCore/QMetaEnum>
+#include <QtCore/QPointer>
#include <QtNetwork/QNetworkAccessManager>
-#include <QtNetwork/QNetworkRequest>
#include <QtNetwork/QNetworkReply>
-#include <QtCore/QTimer>
-#include <QtCore/QRegularExpression>
-#include <QtCore/QJsonObject>
+#include <QtNetwork/QNetworkRequest>
#include <array>
-using namespace QMatrixClient;
+using namespace Quotient;
+using std::chrono::seconds, std::chrono::milliseconds;
+using namespace std::chrono_literals;
+
+BaseJob::StatusCode BaseJob::Status::fromHttpCode(int httpCode)
+{
+ // Based on https://en.wikipedia.org/wiki/List_of_HTTP_status_codes
+ if (httpCode / 10 == 41) // 41x errors
+ return httpCode == 410 ? IncorrectRequestError : NotFoundError;
+ switch (httpCode) {
+ case 401:
+ return Unauthorised;
+ // clang-format off
+ case 403: case 407: // clang-format on
+ return ContentAccessError;
+ case 404:
+ return NotFoundError;
+ // clang-format off
+ case 400: case 405: case 406: case 426: case 428: case 505: // clang-format on
+ case 494: // Unofficial nginx "Request header too large"
+ case 497: // Unofficial nginx "HTTP request sent to HTTPS port"
+ return IncorrectRequestError;
+ case 429:
+ return TooManyRequestsError;
+ case 501:
+ case 510:
+ return RequestNotImplementedError;
+ case 511:
+ return NetworkAuthRequiredError;
+ default:
+ return NetworkError;
+ }
+}
+
+QDebug BaseJob::Status::dumpToLog(QDebug dbg) const
+{
+ QDebugStateSaver _s(dbg);
+ dbg.noquote().nospace();
+ if (auto* const k = QMetaEnum::fromType<StatusCode>().valueToKey(code)) {
+ const QByteArray b = k;
+ dbg << b.mid(b.lastIndexOf(':'));
+ } else
+ dbg << code;
+ return dbg << ": " << message;
+}
-struct NetworkReplyDeleter : public QScopedPointerDeleteLater
+template <typename... Ts>
+constexpr auto make_array(Ts&&... items)
{
- static inline void cleanup(QNetworkReply* reply)
+ return std::array<std::common_type_t<Ts...>, sizeof...(Ts)>({items...});
+}
+
+class BaseJob::Private {
+public:
+ struct JobTimeoutConfig {
+ seconds jobTimeout;
+ seconds nextRetryInterval;
+ };
+
+ // Using an idiom from clang-tidy:
+ // http://clang.llvm.org/extra/clang-tidy/checks/modernize-pass-by-value.html
+ Private(HttpVerb v, QString endpoint, const QUrlQuery& q, Data&& data,
+ bool nt)
+ : verb(v)
+ , apiEndpoint(std::move(endpoint))
+ , requestQuery(q)
+ , requestData(std::move(data))
+ , needsToken(nt)
{
- if (reply && reply->isRunning())
- reply->abort();
- QScopedPointerDeleteLater::cleanup(reply);
+ timer.setSingleShot(true);
+ retryTimer.setSingleShot(true);
+ }
+
+ ~Private()
+ {
+ if (reply) {
+ if (reply->isRunning()) {
+ reply->abort();
+ }
+ delete reply;
+ }
+ }
+
+ void sendRequest();
+ /*! \brief Parse the response byte array into JSON
+ *
+ * This calls QJsonDocument::fromJson() on rawResponse, converts
+ * the QJsonParseError result to BaseJob::Status and stores the resulting
+ * JSON in jsonResponse.
+ */
+ Status parseJson();
+
+ ConnectionData* connection = nullptr;
+
+ // Contents for the network request
+ HttpVerb verb;
+ QString apiEndpoint;
+ QHash<QByteArray, QByteArray> requestHeaders;
+ QUrlQuery requestQuery;
+ Data requestData;
+ bool needsToken;
+
+ bool inBackground = false;
+
+ // There's no use of QMimeType here because we don't want to match
+ // content types against the known MIME type hierarchy; and at the same
+ // type QMimeType is of little help with MIME type globs (`text/*` etc.)
+ QByteArrayList expectedContentTypes { "application/json" };
+
+ QByteArrayList expectedKeys;
+
+ // When the QNetworkAccessManager is destroyed it destroys all pending replies.
+ // Using QPointer allows us to know when that happend.
+ QPointer<QNetworkReply> reply;
+
+ Status status = Unprepared;
+ QByteArray rawResponse;
+ /// Contains a null document in case of non-JSON body (for a successful
+ /// or unsuccessful response); a document with QJsonObject or QJsonArray
+ /// in case of a successful response with JSON payload, as per the API
+ /// definition (including an empty JSON object - QJsonObject{});
+ /// and QJsonObject in case of an API error.
+ QJsonDocument jsonResponse;
+ QUrl errorUrl; //< May contain a URL to help with some errors
+
+ LoggingCategory logCat = JOBS;
+
+ QTimer timer;
+ QTimer retryTimer;
+
+ static constexpr std::array<const JobTimeoutConfig, 3> errorStrategy {
+ { { 90s, 5s }, { 90s, 10s }, { 120s, 30s } }
+ };
+ int maxRetries = int(errorStrategy.size());
+ int retriesTaken = 0;
+
+ [[nodiscard]] const JobTimeoutConfig& getCurrentTimeoutConfig() const
+ {
+ return errorStrategy[std::min(size_t(retriesTaken),
+ errorStrategy.size() - 1)];
}
-};
-class BaseJob::Private
-{
- public:
- // Using an idiom from clang-tidy:
- // http://clang.llvm.org/extra/clang-tidy/checks/modernize-pass-by-value.html
- Private(HttpVerb v, QString endpoint, const QUrlQuery& q,
- Data&& data, bool nt)
- : verb(v), apiEndpoint(std::move(endpoint)), requestQuery(q)
- , requestData(std::move(data)), needsToken(nt)
- { }
-
- void sendRequest(bool inBackground);
- const JobTimeoutConfig& getCurrentTimeoutConfig() const;
-
- const ConnectionData* connection = nullptr;
-
- // Contents for the network request
- HttpVerb verb;
- QString apiEndpoint;
- QHash<QByteArray, QByteArray> requestHeaders;
- QUrlQuery requestQuery;
- Data requestData;
- bool needsToken;
-
- // There's no use of QMimeType here because we don't want to match
- // content types against the known MIME type hierarchy; and at the same
- // type QMimeType is of little help with MIME type globs (`text/*` etc.)
- QByteArrayList expectedContentTypes;
-
- QScopedPointer<QNetworkReply, NetworkReplyDeleter> reply;
- Status status = Pending;
- QByteArray rawResponse;
- QUrl errorUrl; //< May contain a URL to help with some errors
-
- QTimer timer;
- QTimer retryTimer;
-
- QVector<JobTimeoutConfig> errorStrategy =
- { { 90, 5 }, { 90, 10 }, { 120, 30 } };
- int maxRetries = errorStrategy.size();
- int retriesTaken = 0;
-
- LoggingCategory logCat = JOBS;
+ [[nodiscard]] QString dumpRequest() const
+ {
+ // FIXME: use std::array {} when Apple stdlib gets deduction guides for it
+ static const auto verbs =
+ make_array(QStringLiteral("GET"), QStringLiteral("PUT"),
+ QStringLiteral("POST"), QStringLiteral("DELETE"));
+ const auto verbWord = verbs.at(size_t(verb));
+ return verbWord % ' '
+ % (reply ? reply->url().toString(QUrl::RemoveQuery)
+ : makeRequestUrl(connection->baseUrl(), apiEndpoint)
+ .toString());
+ }
};
-BaseJob::BaseJob(HttpVerb verb, const QString& name, const QString& endpoint, bool needsToken)
- : BaseJob(verb, name, endpoint, Query { }, Data { }, needsToken)
-{ }
+BaseJob::BaseJob(HttpVerb verb, const QString& name, const QString& endpoint,
+ bool needsToken)
+ : BaseJob(verb, name, endpoint, Query {}, Data {}, needsToken)
+{}
BaseJob::BaseJob(HttpVerb verb, const QString& name, const QString& endpoint,
const Query& query, Data&& data, bool needsToken)
: d(new Private(verb, endpoint, query, std::move(data), needsToken))
{
setObjectName(name);
- setExpectedContentTypes({ "application/json" });
- d->timer.setSingleShot(true);
- connect (&d->timer, &QTimer::timeout, this, &BaseJob::timeout);
+ connect(&d->timer, &QTimer::timeout, this, &BaseJob::timeout);
+ connect(&d->retryTimer, &QTimer::timeout, this, [this] {
+ qCDebug(d->logCat) << "Retrying" << this;
+ d->connection->submit(this);
+ });
}
BaseJob::~BaseJob()
@@ -108,28 +211,18 @@ BaseJob::~BaseJob()
qCDebug(d->logCat) << this << "destroyed";
}
-QUrl BaseJob::requestUrl() const
-{
- return d->reply ? d->reply->request().url() : QUrl();
-}
+QUrl BaseJob::requestUrl() const { return d->reply ? d->reply->url() : QUrl(); }
-bool BaseJob::isBackground() const
-{
- return d->reply && d->reply->request().attribute(
- QNetworkRequest::BackgroundRequestAttribute).toBool();
-}
+bool BaseJob::isBackground() const { return d->inBackground; }
-const QString& BaseJob::apiEndpoint() const
-{
- return d->apiEndpoint;
-}
+const QString& BaseJob::apiEndpoint() const { return d->apiEndpoint; }
void BaseJob::setApiEndpoint(const QString& apiEndpoint)
{
d->apiEndpoint = apiEndpoint;
}
-const BaseJob::headers_t&BaseJob::requestHeaders() const
+const BaseJob::headers_t& BaseJob::requestHeaders() const
{
return d->requestHeaders;
}
@@ -145,25 +238,16 @@ void BaseJob::setRequestHeaders(const BaseJob::headers_t& headers)
d->requestHeaders = headers;
}
-const QUrlQuery& BaseJob::query() const
-{
- return d->requestQuery;
-}
+const QUrlQuery& BaseJob::query() const { return d->requestQuery; }
void BaseJob::setRequestQuery(const QUrlQuery& query)
{
d->requestQuery = query;
}
-const BaseJob::Data& BaseJob::requestData() const
-{
- return d->requestData;
-}
+const BaseJob::Data& BaseJob::requestData() const { return d->requestData; }
-void BaseJob::setRequestData(Data&& data)
-{
- std::swap(d->requestData, data);
-}
+void BaseJob::setRequestData(Data&& data) { std::swap(d->requestData, data); }
const QByteArrayList& BaseJob::expectedContentTypes() const
{
@@ -180,182 +264,195 @@ void BaseJob::setExpectedContentTypes(const QByteArrayList& contentTypes)
d->expectedContentTypes = contentTypes;
}
-QUrl BaseJob::makeRequestUrl(QUrl baseUrl,
- const QString& path, const QUrlQuery& query)
+const QByteArrayList BaseJob::expectedKeys() const { return d->expectedKeys; }
+
+void BaseJob::addExpectedKey(const QByteArray& key) { d->expectedKeys << key; }
+
+void BaseJob::setExpectedKeys(const QByteArrayList& keys)
+{
+ d->expectedKeys = keys;
+}
+
+const QNetworkReply* BaseJob::reply() const { return d->reply.data(); }
+
+QNetworkReply* BaseJob::reply() { return d->reply.data(); }
+
+QUrl BaseJob::makeRequestUrl(QUrl baseUrl, const QString& path,
+ const QUrlQuery& query)
{
auto pathBase = baseUrl.path();
- if (!pathBase.endsWith('/') && !path.startsWith('/'))
- pathBase.push_back('/');
+ // QUrl::adjusted(QUrl::StripTrailingSlashes) doesn't help with root '/'
+ while (pathBase.endsWith('/'))
+ pathBase.chop(1);
+ if (!path.startsWith('/')) // Normally API files do start with '/'
+ pathBase.push_back('/'); // so this shouldn't be needed these days
baseUrl.setPath(pathBase + path, QUrl::TolerantMode);
baseUrl.setQuery(query);
return baseUrl;
}
-void BaseJob::Private::sendRequest(bool inBackground)
+void BaseJob::Private::sendRequest()
{
- QNetworkRequest req
- { makeRequestUrl(connection->baseUrl(), apiEndpoint, requestQuery) };
+ QNetworkRequest req { makeRequestUrl(connection->baseUrl(), apiEndpoint,
+ requestQuery) };
if (!requestHeaders.contains("Content-Type"))
req.setHeader(QNetworkRequest::ContentTypeHeader, "application/json");
if (needsToken)
req.setRawHeader("Authorization",
QByteArray("Bearer ") + connection->accessToken());
req.setAttribute(QNetworkRequest::BackgroundRequestAttribute, inBackground);
-#if (QT_VERSION >= QT_VERSION_CHECK(5, 6, 0))
req.setAttribute(QNetworkRequest::FollowRedirectsAttribute, true);
req.setMaximumRedirectsAllowed(10);
-#endif
req.setAttribute(QNetworkRequest::HttpPipeliningAllowedAttribute, true);
-#if QT_VERSION >= QT_VERSION_CHECK(5, 9, 0)
- // some sources claim that there are issues with QT 5.8
- req.setAttribute(QNetworkRequest::HTTP2AllowedAttribute, true);
+ req.setAttribute(
+#if (QT_VERSION >= QT_VERSION_CHECK(5, 15, 0))
+ QNetworkRequest::Http2AllowedAttribute
+#else
+ QNetworkRequest::HTTP2AllowedAttribute
#endif
+ // Qt doesn't combine HTTP2 with SSL quite right, occasionally crashing at
+ // what seems like an attempt to write to a closed channel. If/when that
+ // changes, false should be turned to true below.
+ , false);
Q_ASSERT(req.url().isValid());
for (auto it = requestHeaders.cbegin(); it != requestHeaders.cend(); ++it)
req.setRawHeader(it.key(), it.value());
- switch( verb )
- {
- case HttpVerb::Get:
- reply.reset( connection->nam()->get(req) );
- break;
- case HttpVerb::Post:
- reply.reset( connection->nam()->post(req, requestData.source()) );
- break;
- case HttpVerb::Put:
- reply.reset( connection->nam()->put(req, requestData.source()) );
- break;
- case HttpVerb::Delete:
- reply.reset( connection->nam()->deleteResource(req) );
- break;
+
+ switch (verb) {
+ case HttpVerb::Get:
+ reply = connection->nam()->get(req);
+ break;
+ case HttpVerb::Post:
+ reply = connection->nam()->post(req, requestData.source());
+ break;
+ case HttpVerb::Put:
+ reply = connection->nam()->put(req, requestData.source());
+ break;
+ case HttpVerb::Delete:
+ reply = connection->nam()->sendCustomRequest(req, "DELETE", requestData.source());
+ break;
}
}
-void BaseJob::beforeStart(const ConnectionData*)
-{ }
+void BaseJob::doPrepare() { }
-void BaseJob::afterStart(const ConnectionData*, QNetworkReply*)
-{ }
+void BaseJob::onSentRequest(QNetworkReply*) { }
-void BaseJob::beforeAbandon(QNetworkReply*)
-{ }
+void BaseJob::beforeAbandon() { }
-void BaseJob::start(const ConnectionData* connData, bool inBackground)
+void BaseJob::initiate(ConnectionData* connData, bool inBackground)
{
- if (connData && connData->baseUrl().isValid()) {
+ if (Q_LIKELY(connData && connData->baseUrl().isValid())) {
+ d->inBackground = inBackground;
d->connection = connData;
- d->retryTimer.setSingleShot(true);
- connect(&d->retryTimer, &QTimer::timeout, this,
- [this, inBackground] { sendRequest(inBackground); });
-
- beforeStart(connData);
- if (status().good())
- sendRequest(inBackground);
- if (status().good())
- afterStart(connData, d->reply.data());
+ doPrepare();
+
+ if (d->needsToken && d->connection->accessToken().isEmpty())
+ setStatus(Unauthorised);
+ else if ((d->verb == HttpVerb::Post || d->verb == HttpVerb::Put)
+ && d->requestData.source()
+ && !d->requestData.source()->isReadable()) {
+ setStatus(FileError, "Request data not ready");
+ }
+ Q_ASSERT(status().code != Pending); // doPrepare() must NOT set this
+ if (Q_LIKELY(status().code == Unprepared)) {
+ d->connection->submit(this);
+ return;
+ }
+ qCWarning(d->logCat).noquote()
+ << "Request failed preparation and won't be sent:"
+ << d->dumpRequest();
} else {
qCCritical(d->logCat)
<< "Developers, ensure the Connection is valid before using it";
Q_ASSERT(false);
setStatus(IncorrectRequestError, tr("Invalid server connection"));
}
- if (!status().good())
- QTimer::singleShot(0, this, &BaseJob::finishJob);
+ // The status is no good, finalise
+ QTimer::singleShot(0, this, &BaseJob::finishJob);
}
-void BaseJob::sendRequest(bool inBackground)
+void BaseJob::sendRequest()
{
- emit aboutToStart();
- d->retryTimer.stop(); // In case we were counting down at the moment
- qCDebug(d->logCat) << this << "sending request to" << d->apiEndpoint;
- if (!d->requestQuery.isEmpty())
- qCDebug(d->logCat) << " query:" << d->requestQuery.toString();
- d->sendRequest(inBackground);
- connect( d->reply.data(), &QNetworkReply::finished, this, &BaseJob::gotReply );
- if (d->reply->isRunning())
- {
- connect( d->reply.data(), &QNetworkReply::metaDataChanged,
- this, &BaseJob::checkReply);
- connect( d->reply.data(), &QNetworkReply::uploadProgress,
- this, &BaseJob::uploadProgress);
- connect( d->reply.data(), &QNetworkReply::downloadProgress,
- this, &BaseJob::downloadProgress);
- d->timer.start(getCurrentTimeout());
- qCDebug(d->logCat) << this << "request has been sent";
- emit started();
+ if (status().code == Abandoned) {
+ qCDebug(d->logCat) << "Won't proceed with the abandoned request:"
+ << d->dumpRequest();
+ return;
}
- else
- qCWarning(d->logCat) << this << "request could not start";
+ Q_ASSERT(d->connection && status().code == Pending);
+ qCDebug(d->logCat).noquote() << "Making" << d->dumpRequest();
+ d->needsToken |= d->connection->needsToken(objectName());
+ emit aboutToSendRequest();
+ d->sendRequest();
+ Q_ASSERT(d->reply);
+ connect(reply(), &QNetworkReply::finished, this, [this] {
+ gotReply();
+ finishJob();
+ });
+ if (d->reply->isRunning()) {
+ connect(reply(), &QNetworkReply::metaDataChanged, this,
+ [this] { checkReply(reply()); });
+ connect(reply(), &QNetworkReply::uploadProgress, this,
+ &BaseJob::uploadProgress);
+ connect(reply(), &QNetworkReply::downloadProgress, this,
+ &BaseJob::downloadProgress);
+ d->timer.start(getCurrentTimeout());
+ qCInfo(d->logCat).noquote() << "Sent" << d->dumpRequest();
+ onSentRequest(reply());
+ emit sentRequest();
+ } else
+ qCCritical(d->logCat).noquote()
+ << "Request could not start:" << d->dumpRequest();
}
-void BaseJob::checkReply()
+BaseJob::Status BaseJob::Private::parseJson()
{
- setStatus(doCheckReply(d->reply.data()));
+ QJsonParseError error { 0, QJsonParseError::MissingObject };
+ jsonResponse = QJsonDocument::fromJson(rawResponse, &error);
+ return { error.error == QJsonParseError::NoError ? NoError
+ : IncorrectResponse,
+ error.errorString() };
}
void BaseJob::gotReply()
{
- checkReply();
+ setStatus(checkReply(reply()));
+
+ if (status().good()
+ && d->expectedContentTypes == QByteArrayList { "application/json" }) {
+ d->rawResponse = reply()->readAll();
+ setStatus(d->parseJson());
+ if (status().good() && !expectedKeys().empty()) {
+ const auto& responseObject = jsonData();
+ QByteArrayList missingKeys;
+ for (const auto& k: expectedKeys())
+ if (!responseObject.contains(k))
+ missingKeys.push_back(k);
+ if (!missingKeys.empty())
+ setStatus(IncorrectResponse, tr("Required JSON keys missing: ")
+ + missingKeys.join());
+ }
+ if (!status().good()) // Bad JSON in a "good" reply: bail out
+ return;
+ } // else {
+ // If the endpoint expects anything else than just (API-related) JSON
+ // reply()->readAll() is not performed and the whole reply processing
+ // is left to derived job classes: they may read it piecemeal or customise
+ // per content type in prepareResult(), or even have read it already
+ // (see, e.g., DownloadFileJob).
+ // }
+
if (status().good())
- setStatus(parseReply(d->reply.data()));
+ setStatus(prepareResult());
else {
- // FIXME: Factor out to smth like BaseJob::handleError()
- d->rawResponse = d->reply->readAll();
- const auto jsonBody =
- d->reply->rawHeader("Content-Type") == "application/json";
+ d->rawResponse = reply()->readAll();
qCDebug(d->logCat).noquote()
- << "Error body (truncated if long):" << d->rawResponse.left(500);
- if (jsonBody)
- {
- auto json = QJsonDocument::fromJson(d->rawResponse).object();
- const auto errCode = json.value("errcode"_ls).toString();
- if (error() == TooManyRequestsError ||
- errCode == "M_LIMIT_EXCEEDED")
- {
- QString msg = tr("Too many requests");
- auto retryInterval = json.value("retry_after_ms"_ls).toInt(-1);
- if (retryInterval != -1)
- msg += tr(", next retry advised after %1 ms")
- .arg(retryInterval);
- else // We still have to figure some reasonable interval
- retryInterval = getNextRetryInterval();
-
- setStatus(TooManyRequestsError, msg);
-
- // Shortcut to retry instead of executing finishJob()
- stop();
- qCWarning(d->logCat)
- << this << "will retry in" << retryInterval << "ms";
- d->retryTimer.start(retryInterval);
- emit retryScheduled(d->retriesTaken, retryInterval);
- return;
- }
- if (errCode == "M_CONSENT_NOT_GIVEN")
- {
- d->status.code = UserConsentRequiredError;
- d->errorUrl = json.value("consent_uri"_ls).toString();
- }
- else if (errCode == "M_UNSUPPORTED_ROOM_VERSION" ||
- errCode == "M_INCOMPATIBLE_ROOM_VERSION")
- {
- d->status.code = UnsupportedRoomVersionError;
- if (json.contains("room_version"))
- d->status.message =
- tr("Requested room version: %1")
- .arg(json.value("room_version").toString());
- }
- else if (errCode == "M_CANNOT_LEAVE_SERVER_NOTICE_ROOM")
- setStatus(IncorrectRequestError,
- tr("It's not allowed to leave a server notices room"));
- else if (errCode == "M_USER_DEACTIVATED")
- setStatus(ContentAccessError,
- tr("The user has been deactivated"));
- else if (!json.isEmpty()) // Not localisable on the client side
- setStatus(d->status.code, json.value("error"_ls).toString());
- }
+ << "Error body (truncated if long):" << rawDataSample(500);
+ // Parse the error payload and update the status if needed
+ if (const auto newStatus = prepareError(); !newStatus.good())
+ setStatus(newStatus);
}
-
- finishJob();
}
bool checkContentType(const QByteArray& type, const QByteArrayList& patterns)
@@ -366,138 +463,183 @@ bool checkContentType(const QByteArray& type, const QByteArrayList& patterns)
// ignore possible appendixes of the content type
const auto ctype = type.split(';').front();
- for (const auto& pattern: patterns)
- {
+ for (const auto& pattern: patterns) {
if (pattern.startsWith('*') || ctype == pattern) // Fast lane
return true;
auto patternParts = pattern.split('/');
Q_ASSERT_X(patternParts.size() <= 2, __FUNCTION__,
- "BaseJob: Expected content type should have up to two"
- " /-separated parts; violating pattern: " + pattern);
+ "BaseJob: Expected content type should have up to two"
+ " /-separated parts; violating pattern: "
+ + pattern);
- if (ctype.split('/').front() == patternParts.front() &&
- patternParts.back() == "*")
+ if (ctype.split('/').front() == patternParts.front()
+ && patternParts.back() == "*")
return true; // Exact match already went on fast lane
}
return false;
}
-BaseJob::Status BaseJob::doCheckReply(QNetworkReply* reply) const
+BaseJob::Status BaseJob::checkReply(const QNetworkReply* reply) const
{
- // QNetworkReply error codes seem to be flawed when it comes to HTTP;
- // see, e.g., https://github.com/QMatrixClient/libqmatrixclient/issues/200
- // so check genuine HTTP codes. The below processing is based on
- // https://en.wikipedia.org/wiki/List_of_HTTP_status_codes
+ // QNetworkReply error codes are insufficient for our purposes (e.g. they
+ // don't allow to discern HTTP code 429) so check the original code instead
const auto httpCodeHeader =
- reply->attribute(QNetworkRequest::HttpStatusCodeAttribute);
- if (!httpCodeHeader.isValid())
- {
- qCWarning(d->logCat) << this << "didn't get valid HTTP headers";
+ reply->attribute(QNetworkRequest::HttpStatusCodeAttribute);
+ if (!httpCodeHeader.isValid()) {
+ qCWarning(d->logCat).noquote()
+ << "No valid HTTP headers from" << d->dumpRequest();
return { NetworkError, reply->errorString() };
}
- const QString replyState = reply->isRunning() ?
- QStringLiteral("(tentative)") : QStringLiteral("(final)");
- const auto urlString = '|' + d->reply->url().toDisplayString();
const auto httpCode = httpCodeHeader.toInt();
- const auto reason =
- reply->attribute(QNetworkRequest::HttpReasonPhraseAttribute).toString();
if (httpCode / 100 == 2) // 2xx
{
- qCDebug(d->logCat).noquote().nospace() << this << urlString;
- qCDebug(d->logCat).noquote() << " " << httpCode << reason << replyState;
+ if (reply->isFinished())
+ qCInfo(d->logCat).noquote() << httpCode << "<-" << d->dumpRequest();
if (!checkContentType(reply->rawHeader("Content-Type"),
- d->expectedContentTypes))
+ d->expectedContentTypes))
return { UnexpectedResponseTypeWarning,
"Unexpected content type of the response" };
return NoError;
}
+ if (reply->isFinished())
+ qCWarning(d->logCat).noquote() << httpCode << "<-" << d->dumpRequest();
- qCWarning(d->logCat).noquote().nospace() << this << urlString;
- qCWarning(d->logCat).noquote() << " " << httpCode << reason << replyState;
- return { [httpCode]() -> StatusCode {
- if (httpCode / 10 == 41)
- return httpCode == 410 ? IncorrectRequestError : NotFoundError;
- switch (httpCode)
- {
- case 401: case 403: case 407:
- return ContentAccessError;
- case 404:
- return NotFoundError;
- case 400: case 405: case 406: case 426: case 428:
- case 505:
- case 494: // Unofficial nginx "Request header too large"
- case 497: // Unofficial nginx "HTTP request sent to HTTPS port"
- return IncorrectRequestError;
- case 429:
- return TooManyRequestsError;
- case 501: case 510:
- return RequestNotImplementedError;
- case 511:
- return NetworkAuthRequiredError;
- default:
- return NetworkError;
- }
- }(), reply->errorString() };
+ auto message = reply->errorString();
+ if (message.isEmpty())
+ message = reply->attribute(QNetworkRequest::HttpReasonPhraseAttribute)
+ .toString();
+
+ return Status::fromHttpCode(httpCode, message);
}
-BaseJob::Status BaseJob::parseReply(QNetworkReply* reply)
+BaseJob::Status BaseJob::prepareResult() { return Success; }
+
+BaseJob::Status BaseJob::prepareError()
{
- d->rawResponse = reply->readAll();
- QJsonParseError error { 0, QJsonParseError::MissingObject };
- const auto& json = QJsonDocument::fromJson(d->rawResponse, &error);
- if( error.error == QJsonParseError::NoError )
- return parseJson(json);
+ // Try to make sense of the error payload but be prepared for all kinds
+ // of unexpected stuff (raw HTML, plain text, foreign JSON among those)
+ if (!d->rawResponse.isEmpty()
+ && reply()->rawHeader("Content-Type") == "application/json")
+ d->parseJson();
+
+ // By now, if d->parseJson() above succeeded then jsonData() will return
+ // a valid JSON object - or an empty object otherwise (in which case most
+ // of if's below will fall through to `return NoError` at the end
+ const auto& errorJson = jsonData();
+ const auto errCode = errorJson.value("errcode"_ls).toString();
+ if (error() == TooManyRequestsError || errCode == "M_LIMIT_EXCEEDED") {
+ QString msg = tr("Too many requests");
+ int64_t retryAfterMs = errorJson.value("retry_after_ms"_ls).toInt(-1);
+ if (retryAfterMs >= 0)
+ msg += tr(", next retry advised after %1 ms").arg(retryAfterMs);
+ else // We still have to figure some reasonable interval
+ retryAfterMs = getNextRetryMs();
+
+ d->connection->limitRate(milliseconds(retryAfterMs));
+
+ return { TooManyRequestsError, msg };
+ }
+
+ if (errCode == "M_CONSENT_NOT_GIVEN") {
+ d->errorUrl = errorJson.value("consent_uri"_ls).toString();
+ return { UserConsentRequiredError };
+ }
+ if (errCode == "M_UNSUPPORTED_ROOM_VERSION"
+ || errCode == "M_INCOMPATIBLE_ROOM_VERSION")
+ return { UnsupportedRoomVersionError,
+ errorJson.contains("room_version"_ls)
+ ? tr("Requested room version: %1")
+ .arg(errorJson.value("room_version"_ls).toString())
+ : errorJson.value("error"_ls).toString() };
+ if (errCode == "M_CANNOT_LEAVE_SERVER_NOTICE_ROOM")
+ return { CannotLeaveRoom,
+ tr("It's not allowed to leave a server notices room") };
+ if (errCode == "M_USER_DEACTIVATED")
+ return { UserDeactivated };
- return { IncorrectResponseError, error.errorString() };
+ // Not localisable on the client side
+ if (errorJson.contains("error"_ls)) // Keep the code, update the message
+ return { d->status.code, errorJson.value("error"_ls).toString() };
+
+ return NoError; // Retain the status if the error payload is not recognised
}
-BaseJob::Status BaseJob::parseJson(const QJsonDocument&)
+QJsonValue BaseJob::takeValueFromJson(const QString& key)
{
- return Success;
+ if (!d->jsonResponse.isObject())
+ return QJsonValue::Undefined;
+ auto o = d->jsonResponse.object();
+ auto v = o.take(key);
+ d->jsonResponse.setObject(o);
+ return v;
}
void BaseJob::stop()
{
+ // This method is (also) used to semi-finalise the job before retrying; so
+ // stop the timeout timer but keep the retry timer running.
d->timer.stop();
- if (d->reply)
- {
+ if (d->reply) {
d->reply->disconnect(this); // Ignore whatever comes from the reply
- if (d->reply->isRunning())
- {
- qCWarning(d->logCat) << this << "stopped without ready network reply";
- d->reply->abort();
+ if (d->reply->isRunning()) {
+ qCWarning(d->logCat)
+ << this << "stopped without ready network reply";
+ d->reply->abort(); // Keep the reply object in case clients need it
}
- }
- else
+ } else
qCWarning(d->logCat) << this << "stopped with empty network reply";
}
void BaseJob::finishJob()
{
stop();
- if ((error() == NetworkError || error() == TimeoutError)
- && d->retriesTaken < d->maxRetries)
- {
- // TODO: The whole retrying thing should be put to ConnectionManager
- // otherwise independently retrying jobs make a bit of notification
- // storm towards the UI.
- const auto retryInterval =
- error() == TimeoutError ? 0 : getNextRetryInterval();
- ++d->retriesTaken;
- qCWarning(d->logCat).nospace() << this << ": retry #" << d->retriesTaken
- << " in " << retryInterval/1000 << " s";
- d->retryTimer.start(retryInterval);
- emit retryScheduled(d->retriesTaken, retryInterval);
+ switch(error()) {
+ case TooManyRequests:
+ emit rateLimited();
+ d->connection->submit(this);
return;
+ case Unauthorised:
+ if (!d->needsToken && !d->connection->accessToken().isEmpty()) {
+ // Rerun with access token (extension of the spec while
+ // https://github.com/matrix-org/matrix-doc/issues/701 is pending)
+ d->connection->setNeedsToken(objectName());
+ qCWarning(d->logCat) << this << "re-running with authentication";
+ emit retryScheduled(d->retriesTaken, 0);
+ d->connection->submit(this);
+ return;
+ }
+ break;
+ case NetworkError:
+ case IncorrectResponse:
+ case Timeout:
+ if (d->retriesTaken < d->maxRetries) {
+ // TODO: The whole retrying thing should be put to
+ // Connection(Manager) otherwise independently retrying jobs make a
+ // bit of notification storm towards the UI.
+ const seconds retryIn = error() == Timeout ? 0s
+ : getNextRetryInterval();
+ ++d->retriesTaken;
+ qCWarning(d->logCat).nospace()
+ << this << ": retry #" << d->retriesTaken << " in "
+ << retryIn.count() << " s";
+ setStatus(Pending, "Pending retry");
+ d->retryTimer.start(retryIn);
+ emit retryScheduled(d->retriesTaken, milliseconds(retryIn).count());
+ return;
+ }
+ [[fallthrough]];
+ default:;
}
- // Notify those interested in any completion of the job (including killing)
+ Q_ASSERT(status().code != Pending);
+
+ // Notify those interested in any completion of the job including abandon()
emit finished(this);
- emit result(this);
+ emit result(this); // abandon() doesn't emit this
if (error())
emit failure(this);
else
@@ -506,135 +648,144 @@ void BaseJob::finishJob()
deleteLater();
}
-const JobTimeoutConfig& BaseJob::Private::getCurrentTimeoutConfig() const
+seconds BaseJob::getCurrentTimeout() const
{
- return errorStrategy[std::min(retriesTaken, errorStrategy.size() - 1)];
+ return d->getCurrentTimeoutConfig().jobTimeout;
}
-BaseJob::duration_t BaseJob::getCurrentTimeout() const
+BaseJob::duration_ms_t BaseJob::getCurrentTimeoutMs() const
{
- return d->getCurrentTimeoutConfig().jobTimeout * 1000;
+ return milliseconds(getCurrentTimeout()).count();
}
-BaseJob::duration_t BaseJob::getNextRetryInterval() const
+seconds BaseJob::getNextRetryInterval() const
{
- return d->getCurrentTimeoutConfig().nextRetryInterval * 1000;
+ return d->getCurrentTimeoutConfig().nextRetryInterval;
}
-BaseJob::duration_t BaseJob::millisToRetry() const
+BaseJob::duration_ms_t BaseJob::getNextRetryMs() const
{
- return d->retryTimer.isActive() ? d->retryTimer.remainingTime() : 0;
+ return milliseconds(getNextRetryInterval()).count();
}
-int BaseJob::maxRetries() const
+milliseconds BaseJob::timeToRetry() const
{
- return d->maxRetries;
+ return d->retryTimer.isActive() ? d->retryTimer.remainingTimeAsDuration()
+ : 0s;
}
-void BaseJob::setMaxRetries(int newMaxRetries)
+BaseJob::duration_ms_t BaseJob::millisToRetry() const
{
- d->maxRetries = newMaxRetries;
+ return timeToRetry().count();
}
-BaseJob::Status BaseJob::status() const
+int BaseJob::maxRetries() const { return d->maxRetries; }
+
+void BaseJob::setMaxRetries(int newMaxRetries)
{
- return d->status;
+ d->maxRetries = newMaxRetries;
}
+BaseJob::Status BaseJob::status() const { return d->status; }
+
QByteArray BaseJob::rawData(int bytesAtMost) const
{
return bytesAtMost > 0 && d->rawResponse.size() > bytesAtMost
- ? d->rawResponse.left(bytesAtMost) : d->rawResponse;
+ ? d->rawResponse.left(bytesAtMost)
+ : d->rawResponse;
}
+const QByteArray& BaseJob::rawData() const { return d->rawResponse; }
+
QString BaseJob::rawDataSample(int bytesAtMost) const
{
auto data = rawData(bytesAtMost);
Q_ASSERT(data.size() <= d->rawResponse.size());
return data.size() == d->rawResponse.size()
- ? data : data + tr("...(truncated, %Ln bytes in total)",
- "Comes after trimmed raw network response",
- d->rawResponse.size());
-
+ ? data
+ : data + tr("...(truncated, %Ln bytes in total)",
+ "Comes after trimmed raw network response",
+ d->rawResponse.size());
}
-QString BaseJob::statusCaption() const
+QJsonObject BaseJob::jsonData() const
{
- switch (d->status.code)
- {
- case Success:
- return tr("Success");
- case Pending:
- return tr("Request still pending response");
- case UnexpectedResponseTypeWarning:
- return tr("Warning: Unexpected response type");
- case Abandoned:
- return tr("Request was abandoned");
- case NetworkError:
- return tr("Network problems");
- case JsonParseError:
- return tr("Response could not be parsed");
- case TimeoutError:
- return tr("Request timed out");
- case ContentAccessError:
- return tr("Access error");
- case NotFoundError:
- return tr("Not found");
- case IncorrectRequestError:
- return tr("Invalid request");
- case IncorrectResponseError:
- return tr("Response could not be parsed");
- case TooManyRequestsError:
- return tr("Too many requests");
- case RequestNotImplementedError:
- return tr("Function not implemented by the server");
- case NetworkAuthRequiredError:
- return tr("Network authentication required");
- case UserConsentRequiredError:
- return tr("User consent required");
- case UnsupportedRoomVersionError:
- return tr("The server does not support the needed room version");
- default:
- return tr("Request failed");
- }
+ return d->jsonResponse.object();
}
-int BaseJob::error() const
+QJsonArray BaseJob::jsonItems() const
{
- return d->status.code;
+ return d->jsonResponse.array();
}
-QString BaseJob::errorString() const
+QString BaseJob::statusCaption() const
{
- return d->status.message;
+ switch (d->status.code) {
+ case Success:
+ return tr("Success");
+ case Pending:
+ return tr("Request still pending response");
+ case UnexpectedResponseTypeWarning:
+ return tr("Warning: Unexpected response type");
+ case Abandoned:
+ return tr("Request was abandoned");
+ case NetworkError:
+ return tr("Network problems");
+ case TimeoutError:
+ return tr("Request timed out");
+ case Unauthorised:
+ return tr("Unauthorised request");
+ case ContentAccessError:
+ return tr("Access error");
+ case NotFoundError:
+ return tr("Not found");
+ case IncorrectRequestError:
+ return tr("Invalid request");
+ case IncorrectResponseError:
+ return tr("Response could not be parsed");
+ case TooManyRequestsError:
+ return tr("Too many requests");
+ case RequestNotImplementedError:
+ return tr("Function not implemented by the server");
+ case NetworkAuthRequiredError:
+ return tr("Network authentication required");
+ case UserConsentRequiredError:
+ return tr("User consent required");
+ case UnsupportedRoomVersionError:
+ return tr("The server does not support the needed room version");
+ default:
+ return tr("Request failed");
+ }
}
-QUrl BaseJob::errorUrl() const
-{
- return d->errorUrl;
-}
+int BaseJob::error() const { return d->status.code; }
+
+QString BaseJob::errorString() const { return d->status.message; }
+
+QUrl BaseJob::errorUrl() const { return d->errorUrl; }
void BaseJob::setStatus(Status s)
{
// The crash that led to this code has been reported in
- // https://github.com/QMatrixClient/Quaternion/issues/566 - basically,
- // when cleaning up childrent of a deleted Connection, there's a chance
+ // https://github.com/quotient-im/Quaternion/issues/566 - basically,
+ // when cleaning up children of a deleted Connection, there's a chance
// of pending jobs being abandoned, calling setStatus(Abandoned).
// There's nothing wrong with this; however, the safety check for
// cleartext access tokens below uses d->connection - which is a dangling
// pointer.
// To alleviate that, a stricter condition is applied, that for Abandoned
// and to-be-Abandoned jobs the status message will be disregarded entirely.
- // For 0.6 we might rectify the situation by making d->connection
- // a QPointer<> (and derive ConnectionData from QObject, respectively).
- if (d->status.code == Abandoned || s.code == Abandoned)
- s.message.clear();
-
+ // We could rectify the situation by making d->connection a QPointer<>
+ // (and deriving ConnectionData from QObject, respectively) but it's
+ // a too edge case for the hassle.
if (d->status == s)
return;
- if (!s.message.isEmpty()
- && d->connection && !d->connection->accessToken().isEmpty())
+ if (d->status.code == Abandoned || s.code == Abandoned)
+ s.message.clear();
+
+ if (!s.message.isEmpty() && d->connection
+ && !d->connection->accessToken().isEmpty())
s.message.replace(d->connection->accessToken(), "(REDACTED)");
if (!s.good())
qCWarning(d->logCat) << this << "status" << s;
@@ -649,7 +800,7 @@ void BaseJob::setStatus(int code, QString message)
void BaseJob::abandon()
{
- beforeAbandon(d->reply ? d->reply.data() : nullptr);
+ beforeAbandon();
d->timer.stop();
d->retryTimer.stop(); // In case abandon() was called between retries
setStatus(Abandoned);
@@ -662,11 +813,8 @@ void BaseJob::abandon()
void BaseJob::timeout()
{
- setStatus( TimeoutError, "The job has timed out" );
+ setStatus(TimeoutError, "The job has timed out");
finishJob();
}
-void BaseJob::setLoggingCategory(LoggingCategory lcf)
-{
- d->logCat = lcf;
-}
+void BaseJob::setLoggingCategory(LoggingCategory lcf) { d->logCat = lcf; }
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-16 07:43:34 +0000