Merge pull request #73 from robotdana/dana/retry-docker-login

Retry docker login

hooks/environment

@@ -53,11 +53,20 @@ function retry() {
   local -i max_attempts=$((retries + 1))
   local -i attempt_num=1
   local exit_code
+  local stdin_value
 
-  until "$@" ; do
+  if [[ "$1" == "--with-stdin" ]]; then
+    read -sr stdin_value
+    shift
+  fi
+
+  while (( attempt_num <= max_attempts )); do
+    set +e
+    "$@" <<< "${stdin_value:-}"
     exit_code=$?
-    echo "$attempt_num == $retries" >&2
-    if [[ $retries -eq 0 ]] ; then
+    set -e
+
+    if [[ $retries -eq 0 ]] || [[ $exit_code -eq 0 ]]; then
       return $exit_code
     elif (( attempt_num == max_attempts )) ; then
       echo "Login failed after $attempt_num attempts" >&2
@@ -146,7 +155,7 @@ function login_using_aws_ecr_get_login_password() {
   echo "^^^ Authenticating with AWS ECR in $region for ${account_ids[*]} :ecr: :docker:"
   local password; password="$(retry "${BUILDKITE_PLUGIN_ECR_RETRIES:-0}" aws --region "$region" ecr get-login-password)"
   for account_id in "${account_ids[@]}"; do
-    docker login --username AWS --password-stdin "$account_id.dkr.ecr.$region.amazonaws.com" <<< "$password"
+    retry "${BUILDKITE_PLUGIN_ECR_RETRIES:-0}" --with-stdin docker login --username AWS --password-stdin "$account_id.dkr.ecr.$region.amazonaws.com" <<< "$password"
   done
 }
 

tests/run.bats

@@ -191,6 +191,32 @@ load '/usr/local/lib/bats/load.bash'
   unstub docker
 }
 
+@test "ECR login; discovered account ID, with error in docker login, and then retry until success" {
+  [[ -z $SKIP_SLOW ]] || skip "skipping slow test"
+  export BUILDKITE_PLUGIN_ECR_LOGIN=true
+  export BUILDKITE_PLUGIN_ECR_RETRIES=1
+  export AWS_DEFAULT_REGION=us-east-1
+
+  stub aws \
+    "--version : echo aws-cli/2.0.0 Python/3.8.1 Linux/5.5.6-arch1-1 botocore/1.15.3" \
+    "sts get-caller-identity --query Account --output text : echo 888888888888" \
+    "--region us-east-1 ecr get-login-password : echo hunter2"
+
+  stub docker \
+    "login --username AWS --password-stdin 888888888888.dkr.ecr.us-east-1.amazonaws.com : exit 1" \
+    "login --username AWS --password-stdin 888888888888.dkr.ecr.us-east-1.amazonaws.com : cat > /tmp/password-stdin ; echo logging in to docker"
+
+  run "$PWD/hooks/environment"
+
+  assert_success
+  assert_output --partial "Login failed on attempt 1 of 2. Trying again in 1 seconds.."
+  assert_output --partial "logging in to docker"
+  [[ $(cat /tmp/password-stdin) == "hunter2" ]]
+
+  unstub aws
+  unstub docker
+}
+
 @test "ECR login; discovered account ID, with error, and then retry until failure" {
   [[ -z $SKIP_SLOW ]] || skip "skipping slow test"
   export BUILDKITE_PLUGIN_ECR_LOGIN=true
@@ -211,6 +237,32 @@ load '/usr/local/lib/bats/load.bash'
 
   unstub aws
 }
+
+@test "ECR login; discovered account ID, with error in docker login, and then retry until failure" {
+  [[ -z $SKIP_SLOW ]] || skip "skipping slow test"
+  export BUILDKITE_PLUGIN_ECR_LOGIN=true
+  export BUILDKITE_PLUGIN_ECR_RETRIES=1
+  export AWS_DEFAULT_REGION=us-east-1
+
+  stub aws \
+    "--version : echo aws-cli/2.0.0 Python/3.8.1 Linux/5.5.6-arch1-1 botocore/1.15.3" \
+    "sts get-caller-identity --query Account --output text : echo 888888888888" \
+    "--region us-east-1 ecr get-login-password : echo hunter2"
+
+  stub docker \
+    "login --username AWS --password-stdin 888888888888.dkr.ecr.us-east-1.amazonaws.com : exit 1" \
+    "login --username AWS --password-stdin 888888888888.dkr.ecr.us-east-1.amazonaws.com : exit 1"
+
+  run "$PWD/hooks/environment"
+
+  assert_failure
+  assert_output --partial "Login failed on attempt 1 of 2. Trying again in 1 seconds.."
+  assert_output --partial "Login failed after 2 attempts"
+
+  unstub aws
+  unstub docker
+}
+
 @test "ECR login (v1.17.10; after get-login-password was added, before get-login was removed)" {
   export BUILDKITE_PLUGIN_ECR_LOGIN=true
   export BUILDKITE_PLUGIN_ECR_NO_INCLUDE_EMAIL=true