Merge pull request #56 from Fxrh/kitsune-retry-on-some-errors

Jobs retry functionality for recoverable errors

1 files changed, 70 insertions, 15 deletions

diff --git a/jobs/basejob.cpp b/jobs/basejob.cpp
index 5c1be0f0..fd70312e 100644
--- a/jobs/basejob.cpp
+++ b/jobs/basejob.cpp
@@ -18,6 +18,8 @@
 
 #include "basejob.h"
 
+#include <array>
+
 #include <QtNetwork/QNetworkAccessManager>
 #include <QtNetwork/QNetworkRequest>
 #include <QtNetwork/QNetworkReply>
@@ -63,6 +65,10 @@ class BaseJob::Private
         Status status;
 
         QTimer timer;
+        QTimer retryTimer;
+
+        size_t maxRetries = 3;
+        size_t retriesTaken = 0;
 };
 
 inline QDebug operator<<(QDebug dbg, const BaseJob* j)
@@ -76,12 +82,16 @@ BaseJob::BaseJob(ConnectionData* connection, JobHttpType type, QString name,
     : d(new Private(connection, type, endpoint, query, data, needsToken))
 {
     setObjectName(name);
+    d->timer.setSingleShot(true);
     connect (&d->timer, &QTimer::timeout, this, &BaseJob::timeout);
+    d->retryTimer.setSingleShot(true);
+    connect (&d->retryTimer, &QTimer::timeout, this, &BaseJob::start);
     qDebug() << this << "created";
 }
 
 BaseJob::~BaseJob()
 {
+    stop();
     qDebug() << this << "destroyed";
 }
 
@@ -143,10 +153,13 @@ void BaseJob::Private::sendRequest()
 
 void BaseJob::start()
 {
+    emit aboutToStart();
+    d->retryTimer.stop(); // In case we were counting down at the moment
     d->sendRequest();
     connect( d->reply.data(), &QNetworkReply::sslErrors, this, &BaseJob::sslErrors );
     connect( d->reply.data(), &QNetworkReply::finished, this, &BaseJob::gotReply );
-    d->timer.start( 120*1000 );
+    d->timer.start(getCurrentTimeout());
+    emit started();
 }
 
 void BaseJob::gotReply()
@@ -155,7 +168,7 @@ void BaseJob::gotReply()
     if (status().good())
         setStatus(parseReply(d->reply->readAll()));
 
-    finishJob(true);
+    finishJob();
 }
 
 BaseJob::Status BaseJob::checkReply(QNetworkReply* reply) const
@@ -198,33 +211,75 @@ BaseJob::Status BaseJob::parseJson(const QJsonDocument&)
     return Success;
 }
 
-void BaseJob::finishJob(bool emitResult)
+void BaseJob::stop()
 {
     d->timer.stop();
     if (!d->reply)
     {
-        qWarning() << this << "finishes with empty network reply";
+        qWarning() << this << "stopped with empty network reply";
     }
     else if (d->reply->isRunning())
     {
-        qWarning() << this << "finishes without ready network reply";
+        qWarning() << this << "stopped without ready network reply";
         d->reply->disconnect(this); // Ignore whatever comes from the reply
+        d->reply->abort();
+    }
+}
+
+void BaseJob::finishJob()
+{
+    stop();
+    if ((error() == NetworkError || error() == TimeoutError)
+            && d->retriesTaken < d->maxRetries)
+    {
+        const auto retryInterval = getNextRetryInterval();
+        ++d->retriesTaken;
+        qWarning() << this << "will take retry" << d->retriesTaken
+                   << "in" << retryInterval/1000 << "s";
+        d->retryTimer.start(retryInterval);
+        emit retryScheduled(d->retriesTaken, retryInterval);
+        return;
     }
 
-    // Notify those that are interested in any completion of the job (including killing)
+    // Notify those interested in any completion of the job (including killing)
     emit finished(this);
 
-    if (emitResult) {
-        emit result(this);
-        if (error())
-            emit failure(this);
-        else
-            emit success(this);
-    }
+    emit result(this);
+    if (error())
+        emit failure(this);
+    else
+        emit success(this);
 
     deleteLater();
 }
 
+BaseJob::duration_t BaseJob::getCurrentTimeout() const
+{
+    static const std::array<int, 4> timeouts = { 90, 90, 120, 120 };
+    return timeouts[std::min(d->retriesTaken, timeouts.size() - 1)] * 1000;
+}
+
+BaseJob::duration_t BaseJob::getNextRetryInterval() const
+{
+    static const std::array<int, 3> intervals = { 5, 10, 30 };
+    return intervals[std::min(d->retriesTaken, intervals.size() - 1)] * 1000;
+}
+
+BaseJob::duration_t BaseJob::millisToRetry() const
+{
+    return d->retryTimer.isActive() ? d->retryTimer.remainingTime() : 0;
+}
+
+size_t BaseJob::maxRetries() const
+{
+    return d->maxRetries;
+}
+
+void BaseJob::setMaxRetries(size_t newMaxRetries)
+{
+    d->maxRetries = newMaxRetries;
+}
+
 BaseJob::Status BaseJob::status() const
 {
     return d->status;
@@ -256,13 +311,13 @@ void BaseJob::setStatus(int code, QString message)
 
 void BaseJob::abandon()
 {
-    finishJob(false);
+    deleteLater();
 }
 
 void BaseJob::timeout()
 {
     setStatus( TimeoutError, "The job has timed out" );
-    finishJob(true);
+    finishJob();
 }
 
 void BaseJob::sslErrors(const QList<QSslError>& errors)