Merge pull request #211 from flant/feat_exponential_backoff

feat: exponential backoff for hook retries

Author: diafour

.github/workflows/build.yaml

@@ -7,6 +7,8 @@ jobs:
   build_binary:
     name: Build shell-operator binary
     runs-on: ubuntu-latest
+    env:
+      ACTIONS_ALLOW_UNSECURE_COMMANDS: true
     steps:
       - name: Set up Go 1.15
         uses: actions/setup-go@v2

.github/workflows/dev-build.yaml

@@ -54,6 +54,8 @@ jobs:
           - ubuntu
           - alpine
     runs-on: ubuntu-latest
+    env:
+      ACTIONS_ALLOW_UNSECURE_COMMANDS: true
     if: github.event_name == 'pull_request' && github.event.label.id == 1838515600 # ':robot: build dev images' label
     steps:
       - uses: actions/checkout@v2

.github/workflows/lint.yaml

@@ -8,6 +8,8 @@ jobs:
   run_linter:
     name: Run linter
     runs-on: ubuntu-latest
+    env:
+      ACTIONS_ALLOW_UNSECURE_COMMANDS: true
     steps:
       - name: Set up Go 1.15
         uses: actions/setup-go@v2

.github/workflows/release-build.yaml

@@ -20,6 +20,8 @@ jobs:
           - alpine3.11
           - alpine3.12
     runs-on: [ubuntu-latest]
+    env:
+      ACTIONS_ALLOW_UNSECURE_COMMANDS: true
     steps:
       - uses: actions/checkout@v2
 

.github/workflows/tests.yaml

@@ -46,6 +46,8 @@ jobs:
     name: Build and unit tests
     if: github.event_name == 'push' && github.event.ref != 'refs/heads/master'
     runs-on: ubuntu-latest
+    env:
+      ACTIONS_ALLOW_UNSECURE_COMMANDS: true
     steps:
       - name: Set up Go 1.15
         uses: actions/setup-go@v2
@@ -132,6 +134,8 @@ jobs:
     name: Download modules and build libjq
     if: (github.event_name == 'push' && github.event.ref == 'refs/heads/master') || (github.event_name == 'pull_request' && github.event.label.id == 1838578615)
     runs-on: ubuntu-latest
+    env:
+      ACTIONS_ALLOW_UNSECURE_COMMANDS: true
     steps:
       - name: Set up Go 1.15
         uses: actions/setup-go@v2
@@ -208,6 +212,8 @@ jobs:
     if: (github.event_name == 'push' && github.event.ref == 'refs/heads/master') || (github.event_name == 'pull_request' && github.event.label.id == 1838578615)
     needs: prepare_build_dependencies
     runs-on: ubuntu-latest
+    env:
+      ACTIONS_ALLOW_UNSECURE_COMMANDS: true
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
@@ -277,6 +283,8 @@ jobs:
         os: [ubuntu-latest]
         k8s_version: [1.16, 1.17]
     runs-on: ${{ matrix.os }}
+    env:
+      ACTIONS_ALLOW_UNSECURE_COMMANDS: true
     steps:
       - name: Checkout code
         uses: actions/checkout@v2

cmd/shell-operator/main.go

@@ -2,7 +2,9 @@ package main
 
 import (
 	"fmt"
+	"math/rand"
 	"os"
+	"time"
 
 	"github.com/flant/shell-operator/pkg/debug"
 	log "github.com/sirupsen/logrus"
@@ -29,8 +31,11 @@ func main() {
 	startCmd := kpApp.Command("start", "Start shell-operator.").
 		Default().
 		Action(func(c *kingpin.ParseContext) error {
+			// Init logging subsystem.
 			app.SetupLogging()
 			log.Infof("%s %s", app.AppName, app.Version)
+			// Init rand generator.
+			rand.Seed(time.Now().UnixNano())
 
 			defaultOperator := shell_operator.DefaultOperator()
 			err := shell_operator.InitAndStart(defaultOperator)

pkg/task/queue/task_queue.go

@@ -3,7 +3,6 @@ package queue
 import (
 	"context"
 	"fmt"
-	"github.com/flant/shell-operator/pkg/utils/measure"
 	"os"
 	"strings"
 	"sync"
@@ -13,6 +12,8 @@ import (
 
 	"github.com/flant/shell-operator/pkg/metric_storage"
 	"github.com/flant/shell-operator/pkg/task"
+	"github.com/flant/shell-operator/pkg/utils/exponential_backoff"
+	"github.com/flant/shell-operator/pkg/utils/measure"
 )
 
 /*
@@ -351,9 +352,9 @@ func (q *TaskQueue) Start() {
 
 			switch taskRes.Status {
 			case "Fail":
+				// Exponential backoff delay before retry.
+				nextSleepDelay = exponential_backoff.CalculateDelay(DelayOnFailedTask, t.GetFailureCount())
 				t.IncrementFailureCount()
-				// delay before retry
-				nextSleepDelay = DelayOnFailedTask
 				q.Status = fmt.Sprintf("sleep after fail for %s", nextSleepDelay.String())
 			case "Success":
 				// add tasks after current task in reverse order

pkg/utils/exponential_backoff/delay.go

@@ -0,0 +1,40 @@
+package exponential_backoff
+
+import (
+	"math"
+	"math/rand"
+	"time"
+)
+
+const MaxExponentialBackoffDelay = time.Duration(32 * time.Second)
+const ExponentialDelayFactor float64 = 2.0 // Each delay is twice longer.
+const ExponentialDelayRandomMs = 1000      // Each delay has random additional milliseconds.
+
+// CalculateDelay returns delay distributed from initialDelay to default maxDelay (32s)
+//
+// Example:
+//   Retry 0: 5s
+//   Retry 1: 6s
+//   Retry 2: 7.8s
+//   Retry 3: 9.8s
+//   Retry 4: 13s
+//   Retry 5: 21s
+//   Retry 6: 32s
+//   Retry 7: 32s
+func CalculateDelay(initialDelay time.Duration, retryCount int) time.Duration {
+	return CalculateDelayWithMax(initialDelay, MaxExponentialBackoffDelay, retryCount)
+}
+
+func CalculateDelayWithMax(initialDelay time.Duration, maxDelay time.Duration, retryCount int) time.Duration {
+	if retryCount == 0 {
+		return initialDelay
+	}
+	delayNs := int64(float64(time.Second) * math.Pow(ExponentialDelayFactor, float64(retryCount-1)))
+	rndDelayMs := rand.Intn(ExponentialDelayRandomMs)
+	delay := initialDelay + time.Duration(delayNs) + time.Duration(rndDelayMs)*time.Millisecond
+	delay = delay.Truncate(100 * time.Millisecond)
+	if delay.Nanoseconds() > maxDelay.Nanoseconds() {
+		return maxDelay
+	}
+	return delay
+}

pkg/utils/exponential_backoff/delay_test.go

@@ -0,0 +1,18 @@
+package exponential_backoff
+
+import (
+	"math/rand"
+	"testing"
+	"time"
+)
+
+func Test_Delay(t *testing.T) {
+	t.SkipNow()
+	rand.Seed(time.Now().UnixNano())
+	for i := 0; i < 16; i++ {
+		t.Logf("Delay for %02d retry: %s\n",
+			i,
+			CalculateDelay(5*time.Second, i),
+		)
+	}
+}