Merge pull request #52 from arkahna/features/addRetryWhenGettingResou…

…rceNamesFromState Add retry when getting resource names from terraform
arkahna · Sep 2, 2022 · b144b49 · b144b49
2 parents 87827f4 + bd45e9a
commit b144b49
Show file tree

Hide file tree

Showing 14 changed files with 112 additions and 59 deletions.
diff --git a/.changeset/honest-tigers-hunt.md b/.changeset/honest-tigers-hunt.md
@@ -0,0 +1,5 @@
+---
+'@arkahna/nx-terraform': minor
+---
+
+Add retry when getting resource names from terraform
diff --git a/...-terraform/src/common/addFirewallRules.ts → ...m/src/common/addFirewallRulesWithRetry.ts b/...-terraform/src/common/addFirewallRules.ts → ...m/src/common/addFirewallRulesWithRetry.ts
@@ -1,20 +1,25 @@
 import execa from 'execa'
-import { getTfResourceName } from './getTfResourceName'
+import { getTfResourceNameWithRetry } from './getTfResourceNameWithRetry'
 
-export async function addFirewallRules({
+export async function addFirewallRulesWithRetry({
     resourceGroupName,
     addIpToKeyVaults,
     addIpToStorageAccounts,
     publicIpv4,
     projectRoot,
     terragruntConfigFile,
+    retryAttempts,
+    retryDelay,
 }: {
     resourceGroupName: string
     addIpToKeyVaults: string[]
     addIpToStorageAccounts: string[]
     publicIpv4: string
     projectRoot: string
     terragruntConfigFile: string
+    retryAttempts: number
+    /** Retry delay, in seconds */
+    retryDelay: number
 }): Promise<{
     keyVaultsToRemoveFirewallRules: string[]
     storageAccountsToRemoveFirewallRules: string[]
@@ -26,10 +31,12 @@ export async function addFirewallRules({
         let storageAccountName = storageAccount
         if (storageAccountName.includes('azurerm_storage_account')) {
             console.log(`Finding resource for tf resource ${storageAccountName}`)
-            const result = await getTfResourceName(
+            const result = await getTfResourceNameWithRetry(
                 terragruntConfigFile,
                 storageAccountName,
                 projectRoot,
+                retryAttempts,
+                retryDelay,
             )
             if (!result) {
                 continue
@@ -63,7 +70,13 @@ export async function addFirewallRules({
     for (const keyVault of addIpToKeyVaults) {
         let keyVaultName = keyVault
         if (keyVaultName.includes('azurerm_key_vault')) {
-            const result = await getTfResourceName(terragruntConfigFile, keyVaultName, projectRoot)
+            const result = await getTfResourceNameWithRetry(
+                terragruntConfigFile,
+                keyVaultName,
+                projectRoot,
+                retryAttempts,
+                retryDelay,
+            )
             if (!result) {
                 continue
             }

diff --git a/libs/nx-terraform/src/common/getTfResourceName.ts b/libs/nx-terraform/src/common/getTfResourceName.ts
diff --git a/libs/nx-terraform/src/common/getTfResourceNameWithRetry.ts b/libs/nx-terraform/src/common/getTfResourceNameWithRetry.ts
@@ -0,0 +1,47 @@
+import execa from 'execa'
+import { retryOnFirewallError } from './retryOnFirewallError'
+
+export async function getTfResourceNameWithRetry(
+    terragruntConfigFile: string,
+    resourceAddress: string,
+    projectRoot: string,
+    retryAttempts: number,
+    /** Retry delay, in seconds */
+    retryDelay: number,
+) {
+    try {
+        const outputHcl = await retryOnFirewallError(
+            async () =>
+                await execa(
+                    'terragrunt',
+                    [
+                        'state',
+                        'show',
+                        '--terragrunt-config',
+                        terragruntConfigFile,
+                        resourceAddress,
+                        '-no-color',
+                    ],
+                    {
+                        stdio: [process.stdin, 'pipe', 'pipe'],
+                        cwd: projectRoot,
+                    },
+                ),
+            {
+                retryAttempts,
+                retryDelay,
+            },
+        )
+
+        const match = outputHcl.stdout.match(/.{4}name\s+=\s"([a-z]+)"/)
+        if (!match || !match[1]) {
+            throw 'could not find match in returned hcl'
+        }
+
+        resourceAddress = match[1]
+        return resourceAddress
+    } catch (err) {
+        console.log('Failed to look up resource name', err)
+        return
+    }
+}
diff --git a/libs/nx-terraform/src/executors/apply/executor.ts b/libs/nx-terraform/src/executors/apply/executor.ts
@@ -2,7 +2,7 @@ import { ExecutorContext } from '@nrwl/devkit'
 import execa from 'execa'
 import { getEscapedCommand } from 'execa/lib/command'
 import publicIp from 'public-ip'
-import { addFirewallRules } from '../../common/addFirewallRules'
+import { addFirewallRulesWithRetry } from '../../common/addFirewallRulesWithRetry'
 import { createTerragruntCliArgs } from '../../common/createTerragruntCliArgs'
 import { getCurrentAzAccount } from '../../common/getCurrentAzAccount'
 import { getTfEnvVars } from '../../common/getEnvTfVars'
@@ -62,7 +62,7 @@ export default async function runExecutor(options: ApplyExecutorSchema, context:
     const kvOptions = options.addIpToKeyVaults || []
     const storageOptions = options.addIpToStorage || []
     const { keyVaultsToRemoveFirewallRules, storageAccountsToRemoveFirewallRules } =
-        await addFirewallRules({
+        await addFirewallRulesWithRetry({
             resourceGroupName,
             addIpToKeyVaults: options.addIpToDefaultKeyVault
                 ? [keyVaultName, ...kvOptions]
@@ -74,6 +74,8 @@ export default async function runExecutor(options: ApplyExecutorSchema, context:
             publicIpv4,
             terragruntConfigFile,
             projectRoot,
+            retryAttempts: options.firewallRetryAttempts,
+            retryDelay: options.firewallRetryDelay,
         })
 
     const terragruntCliArgs = createTerragruntCliArgs([

diff --git a/libs/nx-terraform/src/executors/destroy/executor.ts b/libs/nx-terraform/src/executors/destroy/executor.ts
@@ -2,7 +2,7 @@ import { ExecutorContext } from '@nrwl/devkit'
 import execa from 'execa'
 import { getEscapedCommand } from 'execa/lib/command'
 import publicIp from 'public-ip'
-import { addFirewallRules } from '../../common/addFirewallRules'
+import { addFirewallRulesWithRetry } from '../../common/addFirewallRulesWithRetry'
 import { createTerragruntCliArgs } from '../../common/createTerragruntCliArgs'
 import { getCurrentAzAccount } from '../../common/getCurrentAzAccount'
 import { getTfEnvVars } from '../../common/getEnvTfVars'
@@ -59,7 +59,7 @@ export default async function runExecutor(options: ApplyExecutorSchema, context:
     const kvOptions = options.addIpToKeyVaults || []
     const storageOptions = options.addIpToStorage || []
     const { keyVaultsToRemoveFirewallRules, storageAccountsToRemoveFirewallRules } =
-        await addFirewallRules({
+        await addFirewallRulesWithRetry({
             resourceGroupName,
             addIpToKeyVaults: options.addIpToDefaultKeyVault
                 ? [keyVaultName, ...kvOptions]
@@ -71,6 +71,8 @@ export default async function runExecutor(options: ApplyExecutorSchema, context:
             publicIpv4,
             terragruntConfigFile,
             projectRoot,
+            retryAttempts: options.firewallRetryAttempts,
+            retryDelay: options.firewallRetryDelay,
         })
 
     const terragruntCliArgs = createTerragruntCliArgs([

diff --git a/libs/nx-terraform/src/executors/output/executor.ts b/libs/nx-terraform/src/executors/output/executor.ts
@@ -1,7 +1,7 @@
 import { ExecutorContext } from '@nrwl/devkit'
 import execa from 'execa'
 import publicIp from 'public-ip'
-import { addFirewallRules } from '../../common/addFirewallRules'
+import { addFirewallRulesWithRetry } from '../../common/addFirewallRulesWithRetry'
 import { createTerragruntCliArgs } from '../../common/createTerragruntCliArgs'
 import { getCurrentAzAccount } from '../../common/getCurrentAzAccount'
 import { getTfEnvVars } from '../../common/getEnvTfVars'
@@ -50,7 +50,7 @@ export default async function runExecutor(options: OutputExecutorSchema, context
     }
 
     const { keyVaultsToRemoveFirewallRules, storageAccountsToRemoveFirewallRules } =
-        await addFirewallRules({
+        await addFirewallRulesWithRetry({
             resourceGroupName,
             addIpToKeyVaults: [],
             addIpToStorageAccounts:
@@ -60,6 +60,8 @@ export default async function runExecutor(options: OutputExecutorSchema, context
             publicIpv4,
             terragruntConfigFile,
             projectRoot,
+            retryAttempts: options.firewallRetryAttempts,
+            retryDelay: options.firewallRetryDelay,
         })
 
     try {

diff --git a/libs/nx-terraform/src/executors/plan/executor.ts b/libs/nx-terraform/src/executors/plan/executor.ts
@@ -2,7 +2,7 @@ import { ExecutorContext } from '@nrwl/devkit'
 import execa from 'execa'
 import { getEscapedCommand } from 'execa/lib/command'
 import publicIp from 'public-ip'
-import { addFirewallRules } from '../../common/addFirewallRules'
+import { addFirewallRulesWithRetry } from '../../common/addFirewallRulesWithRetry'
 import { createTerragruntCliArgs } from '../../common/createTerragruntCliArgs'
 import { getCurrentAzAccount } from '../../common/getCurrentAzAccount'
 import { getTfEnvVars } from '../../common/getEnvTfVars'
@@ -58,7 +58,7 @@ export default async function runExecutor(options: PlanExecutorSchema, context:
     const kvOptions = options.addIpToKeyVaults || []
     const storageOptions = options.addIpToStorage || []
     const { keyVaultsToRemoveFirewallRules, storageAccountsToRemoveFirewallRules } =
-        await addFirewallRules({
+        await addFirewallRulesWithRetry({
             resourceGroupName,
             addIpToKeyVaults: options.addIpToDefaultKeyVault
                 ? [keyVaultName, ...kvOptions]
@@ -70,6 +70,8 @@ export default async function runExecutor(options: PlanExecutorSchema, context:
             publicIpv4,
             terragruntConfigFile,
             projectRoot,
+            retryAttempts: options.firewallRetryAttempts,
+            retryDelay: options.firewallRetryDelay,
         })
 
     const terragruntCliArgs = createTerragruntCliArgs([

diff --git a/libs/nx-terraform/src/executors/state/executor.ts b/libs/nx-terraform/src/executors/state/executor.ts
@@ -2,7 +2,7 @@ import { ExecutorContext } from '@nrwl/devkit'
 import execa from 'execa'
 import { getEscapedCommand } from 'execa/lib/command'
 import publicIp from 'public-ip'
-import { addFirewallRules } from '../../common/addFirewallRules'
+import { addFirewallRulesWithRetry } from '../../common/addFirewallRulesWithRetry'
 import { createTerragruntCliArgs } from '../../common/createTerragruntCliArgs'
 import { getCurrentAzAccount } from '../../common/getCurrentAzAccount'
 import { getTfEnvVars } from '../../common/getEnvTfVars'
@@ -58,7 +58,7 @@ export default async function runExecutor(options: StateExecutorSchema, context:
     const kvOptions = options.addIpToKeyVaults || []
     const storageOptions = options.addIpToStorage || []
     const { keyVaultsToRemoveFirewallRules, storageAccountsToRemoveFirewallRules } =
-        await addFirewallRules({
+        await addFirewallRulesWithRetry({
             resourceGroupName,
             addIpToKeyVaults: options.addIpToDefaultKeyVault
                 ? [keyVaultName, ...kvOptions]
@@ -70,6 +70,8 @@ export default async function runExecutor(options: StateExecutorSchema, context:
             publicIpv4,
             terragruntConfigFile,
             projectRoot,
+            retryAttempts: options.firewallRetryAttempts,
+            retryDelay: options.firewallRetryDelay,
         })
 
     try {

diff --git a/libs/nx-terraform/src/executors/tf-init/executor.ts b/libs/nx-terraform/src/executors/tf-init/executor.ts
@@ -1,6 +1,6 @@
 import { ExecutorContext } from '@nrwl/devkit'
 import publicIp from 'public-ip'
-import { addFirewallRules } from '../../common/addFirewallRules'
+import { addFirewallRulesWithRetry } from '../../common/addFirewallRulesWithRetry'
 import { ensureNoBackendFile } from '../../common/ensure-no-backend-file'
 import { initEnvironmentWorkspaceWithFirewallRuleRetry } from '../../common/initEnvironmentWorkspaceWithRetry'
 import { readRepoSettings } from '../../common/read-repo-settings'
@@ -52,7 +52,7 @@ export default async function runExecutor(options: InitExecutorSchema, context:
         const { resourceGroupName, terraformStorageAccount, terragruntConfigFile } = config
 
         const { keyVaultsToRemoveFirewallRules, storageAccountsToRemoveFirewallRules } =
-            await addFirewallRules({
+            await addFirewallRulesWithRetry({
                 resourceGroupName,
                 addIpToKeyVaults: [],
                 addIpToStorageAccounts:
@@ -62,6 +62,8 @@ export default async function runExecutor(options: InitExecutorSchema, context:
                 publicIpv4,
                 terragruntConfigFile,
                 projectRoot,
+                retryAttempts: options.firewallRetryAttempts,
+                retryDelay: options.firewallRetryDelay,
             })
 
         try {

diff --git a/libs/nx-terraform/src/generators/add-firewall-exceptions/generator.ts b/libs/nx-terraform/src/generators/add-firewall-exceptions/generator.ts
@@ -1,6 +1,6 @@
 import { Tree } from '@nrwl/devkit'
 import publicIp from 'public-ip'
-import { addFirewallRules } from '../../common/addFirewallRules'
+import { addFirewallRulesWithRetry } from '../../common/addFirewallRulesWithRetry'
 import { isDryRun } from '../../common/isDryRun'
 import { readRepoSettings } from '../../common/read-repo-settings'
 import { readConfigFromEnvFile } from '../../common/readConfigFromEnvFile'
@@ -32,7 +32,7 @@ export default async function (tree: Tree, options: NxTerraformAddFirewallExcept
 
         const kvOptions = options.addIpToKeyVaults || []
         const storageOptions = options.addIpToStorage || []
-        await addFirewallRules({
+        await addFirewallRulesWithRetry({
             resourceGroupName,
             addIpToKeyVaults: options.addIpToDefaultKeyVault
                 ? [keyVaultName, ...kvOptions]
@@ -45,6 +45,8 @@ export default async function (tree: Tree, options: NxTerraformAddFirewallExcept
             terragruntConfigFile,
             // This generator isn't tied to a project, so it doesn't support looking up tf resources
             projectRoot: process.cwd(),
+            retryAttempts: options.firewallRetryAttempts,
+            retryDelay: options.firewallRetryDelay,
         })
         console.log('🎉 Success 🎉')
     }

diff --git a/libs/nx-terraform/src/generators/add-firewall-exceptions/schema.d.ts b/libs/nx-terraform/src/generators/add-firewall-exceptions/schema.d.ts
@@ -4,4 +4,6 @@ export interface NxTerraformAddFirewallExceptionsSchema {
     addIpToDefaultStorage: boolean
     addIpToKeyVaults?: string[]
     addIpToStorage?: string[]
+    firewallRetryAttempts: number
+    firewallRetryDelay: number
 }
diff --git a/libs/nx-terraform/src/generators/add-firewall-exceptions/schema.json b/libs/nx-terraform/src/generators/add-firewall-exceptions/schema.json
@@ -34,9 +34,17 @@
                 "type": "string"
             },
             "description": "Adds a firewall exception for the current ip to specified storage accounts. Specify as a semicolon-delimited list of resource names, e.g. 'arkdevsta001;arkdevsta001'."
+        },
+        "firewallRetryAttempts": {
+            "type": "integer",
+            "description": "Number of times to retry adding the firewall exception for the current ip to the environment keyvault",
+            "default": 5
+        },
+        "firewallRetryDelay": {
+            "type": "integer",
+            "description": "Delay in seconds between firewall retry attempts",
+            "default": 5
         }
     },
-    "required": [
-        "environmentName"
-    ]
+    "required": ["environmentName"]
 }
diff --git a/libs/nx-terraform/src/index.ts b/libs/nx-terraform/src/index.ts
@@ -1,5 +1,5 @@
-export { addFirewallRules } from './common/addFirewallRules'
-export { getTfResourceName } from './common/getTfResourceName'
+export { addFirewallRulesWithRetry } from './common/addFirewallRulesWithRetry'
+export { getTfResourceNameWithRetry } from './common/getTfResourceNameWithRetry'
 export { readRepoSettings } from './common/read-repo-settings'
 export { readConfigFromEnvFile } from './common/readConfigFromEnvFile'
 export { removeFirewallRules } from './common/removeFirewallRules'