Add artificial delay

This adds an artificial delay to the VSO. Some services (such as AWS IAM)
are eventually consistent and require some time between generating
the secret, and using the secret.

Without this delay our services can't access AWS for a short while
immediately after the secret rotation happens.

.github/workflows/salesloft_build.yaml

@@ -0,0 +1,56 @@
+name: Docker Build and Push
+
+on:
+  push:
+    branches:
+      - 'salesloft/*'
+
+jobs:
+  build-pre-checks:
+    runs-on:
+      group: org/techops
+      labels: [self-hosted]
+    outputs:
+      go-version: ${{ steps.setup-go.outputs.go-version }}
+    steps:
+      - uses: actions/[email protected]
+      - id: setup-go
+        uses: actions/[email protected]
+        with:
+          go-version-file: .go-version
+
+  vso-docker-build-push:
+    runs-on:
+      group: org/techops
+      labels: [self-hosted]
+    needs: [build-pre-checks]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Extract branch name
+        run: echo "BRANCH_NAME=${GITHUB_REF#refs/heads/salesloft/}" >> $GITHUB_ENV
+
+      - name: Setup Docker BuildX
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ secrets.SL_QA_DOCKER_REGISTRY }}
+          username: ${{ secrets.SL_QA_DOCKER_USER }}
+          password: ${{ secrets.SL_QA_DOCKER_TOKEN }}
+
+      - name: Build & Push
+        uses: docker/build-push-action@v5
+        with:
+          platforms: linux/amd64
+          push: true
+          file: Dockerfile
+          build-args: |
+            GO_VERSION: ${{ needs.build-pre-checks.outputs.go-version }}
+          tags: ${{ secrets.SL_QA_DOCKER_REGISTRY }}/vault-secrets-operator:${{ env.BRANCH_NAME }}-salesloft
+
+      - name: Image digest
+        run: echo "Image - vault-secrets-operator:${{ env.BRANCH_NAME }}-salesloft"

controllers/vaultdynamicsecret_controller.go

@@ -438,6 +438,17 @@ func (r *VaultDynamicSecretReconciler) syncSecret(ctx context.Context, c vault.C
 		}
 	}
 
+	if ad := os.Getenv("ARTIFICIAL_DELAY"); ad != "" {
+		r.Recorder.Eventf(o, corev1.EventTypeNormal, "SyncSecretArtificialDelay",
+			"Introducing ARTIFICIAL_DELAY during syncSecret for %s", ad)
+		d, err := time.ParseDuration(ad)
+		if err != nil {
+			logger.Error(err, "Invalid ARTIFICIAL_DELAY value, sleeping for 60 seconds")
+			time.Sleep(60 * time.Second)
+		}
+		time.Sleep(d)
+	}
+
 	if err := helpers.SyncSecret(ctx, r.Client, o, data); err != nil {
 		logger.Error(err, "Destination sync failed")
 		return nil, false, err

controllers/vaultdynamicsecret_controller_test.go

@@ -8,13 +8,15 @@ import (
 	"fmt"
 	"math"
 	"net/http"
+	"os"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/record"
 	"k8s.io/client-go/util/workqueue"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
@@ -734,6 +736,144 @@ func Test_computeRotationTime(t *testing.T) {
 	}
 }
 
+func Test_artificialDelay(t *testing.T) {
+	type fields struct {
+		Client        client.Client
+		runtimePodUID types.UID
+	}
+	type args struct {
+		ctx     context.Context
+		vClient *vault.MockRecordingVaultClient
+		o       *secretsv1beta1.VaultDynamicSecret
+	}
+	tests := []struct {
+		name    string
+		envs    map[string]string
+		fields  fields
+		args    args
+		want    int
+		wantErr assert.ErrorAssertionFunc
+	}{
+		{
+			name: "no-delay",
+			envs: map[string]string{},
+			fields: fields{
+				Client:        fake.NewClientBuilder().Build(),
+				runtimePodUID: "",
+			},
+			args: args{
+				ctx:     nil,
+				vClient: &vault.MockRecordingVaultClient{},
+				o: &secretsv1beta1.VaultDynamicSecret{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "baz",
+						Namespace: "default",
+					},
+					Spec: secretsv1beta1.VaultDynamicSecretSpec{
+						Mount: "baz",
+						Path:  "foo",
+						Destination: secretsv1beta1.Destination{
+							Name:   "baz",
+							Create: true,
+						},
+					},
+					Status: secretsv1beta1.VaultDynamicSecretStatus{},
+				},
+			},
+			wantErr: assert.NoError,
+			want:    0,
+		},
+		{
+			name: "with-two-second-delay",
+			envs: map[string]string{
+				"ARTIFICIAL_DELAY": "2s",
+			},
+			fields: fields{
+				Client:        fake.NewClientBuilder().Build(),
+				runtimePodUID: "",
+			},
+			args: args{
+				ctx:     nil,
+				vClient: &vault.MockRecordingVaultClient{},
+				o: &secretsv1beta1.VaultDynamicSecret{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "baz",
+						Namespace: "default",
+					},
+					Spec: secretsv1beta1.VaultDynamicSecretSpec{
+						Mount: "baz",
+						Path:  "foo",
+						Destination: secretsv1beta1.Destination{
+							Name:   "baz",
+							Create: true,
+						},
+					},
+
+					Status: secretsv1beta1.VaultDynamicSecretStatus{},
+				},
+			},
+			want:    2,
+			wantErr: assert.NoError,
+		},
+		{
+			name: "with-broken-env",
+			envs: map[string]string{
+				"ARTIFICIAL_DELAY": "rubbish",
+			},
+			fields: fields{
+				Client:        fake.NewClientBuilder().Build(),
+				runtimePodUID: "",
+			},
+			args: args{
+				ctx:     nil,
+				vClient: &vault.MockRecordingVaultClient{},
+				o: &secretsv1beta1.VaultDynamicSecret{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "baz",
+						Namespace: "default",
+					},
+					Spec: secretsv1beta1.VaultDynamicSecretSpec{
+						Mount: "baz",
+						Path:  "foo",
+						Destination: secretsv1beta1.Destination{
+							Name:   "baz",
+							Create: true,
+						},
+					},
+
+					Status: secretsv1beta1.VaultDynamicSecretStatus{},
+				},
+			},
+			want:    60,
+			wantErr: assert.NoError,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			for env, val := range tt.envs {
+				require.NoError(t, os.Setenv(env, val))
+			}
+			r := &VaultDynamicSecretReconciler{
+				Client:   tt.fields.Client,
+				Recorder: &record.FakeRecorder{},
+			}
+			start := time.Now()
+			_, _, err := r.syncSecret(tt.args.ctx, tt.args.vClient, tt.args.o, nil)
+			if !tt.wantErr(t, err, fmt.Sprintf("syncSecret(%v, %v, %v, %v)", tt.args.ctx, tt.args.vClient, tt.args.o, nil)) {
+				return
+			}
+
+			duration := time.Since(start).Round(time.Second).Seconds()
+
+			fmt.Println(duration)
+			if duration != float64(tt.want) {
+				fmt.Println(duration != float64(tt.want))
+				t.Error("Sleep length not correct")
+			}
+		})
+	}
+}
+
 func Test_computeRelativeHorizonWithJitter(t *testing.T) {
 	staticNow := time.Unix(nowFunc().Unix(), 0)
 	defaultNowFunc := func() time.Time { return staticNow }