v1.2.0

Makefile

@@ -4,8 +4,9 @@ init:
 test:
 	# Run unit tests
 	pytest src/lambda_codebase/initial_commit/bootstrap_repository -vvv -s -c src/lambda_codebase/initial_commit/bootstrap_repository/pytest.ini
-	pytest src/lambda_codebase/initial_commit/bootstrap_repository/deployment/lambda_codebase -vvv -s -c src/lambda_codebase/initial_commit/bootstrap_repository/deployment/lambda_codebase/pytest.ini
-	pytest src/lambda_codebase/initial_commit/bootstrap_repository/deployment/lambda_codebase/initial_commit/pipelines_repository -vvv -s -c src/lambda_codebase/initial_commit/bootstrap_repository/deployment/lambda_codebase/initial_commit/pipelines_repository/pytest.ini
+	pytest src/lambda_codebase/initial_commit/bootstrap_repository/deployment/lambda_codebase -vvv -s -c src/lambda_codebase/initial_commit/bootstrap_repository/deployment/lambda_codebase/pytest.ini																																
+	pytest src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python -vvv -s -c src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/pytest.ini
+	pytest src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared -vvv -s -c src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/pytest.ini
 lint:
 	# Linter performs static analysis to catch latent bugs
 	find src/ -iname "*.py" | xargs pylint --rcfile .pylintrc 

samples/sample-ec2-java-app-codedeploy/README.md

@@ -11,6 +11,7 @@ This example is coupled with the `sample-ec2-with-codedeploy` repository and is
     type: cc-codedeploy
     params:
       - SourceAccountId: 111111111111
+      - Image: "aws/codebuild/standard:2.0"
     targets:
       - path: 222222222222
         regions: eu-west-1

samples/sample-vpc/README.md

@@ -1,5 +1,4 @@
 ## Sample VPC to showcase ADF Pipelines
-## Sample VPC to showcase ADF Pipelines
 
 This pipeline is expecting *(in the example case)* a AWS CodeCommit repository on the account `111111111111` in your main deployment region named *sample-vpc*.
 

src/lambda_codebase/account/handler.py

@@ -5,12 +5,17 @@
 The Account handler that is called when ADF is installed to initially create the deployment account if required
 """
 
+import os
+
 try:
     from main import lambda_handler # pylint: disable=unused-import
 except Exception as err:  # pylint: disable=broad-except
     from urllib.request import Request, urlopen
     import json
 
+    if os.environ["AWS_REGION"] != 'us-east-1':
+        raise Exception("ADF must be installed in US-EAST-1 region to function as expected.")
+
     def lambda_handler(event, _context, prior_error=err):
         response = dict(
             LogicalResourceId=event["LogicalResourceId"],

src/lambda_codebase/account/main.py

@@ -106,10 +106,7 @@ def delete_(event, _context):
         return
 
     if physical_resource.created:
-        raise NotImplementedError(
-            "Cannot delete account %s (%s). This is a manual process"
-            % (physical_resource.account_id, physical_resource.account_name)
-        )
+        return
 
 
 # pylint: disable=bad-continuation # https://github.com/PyCQA/pylint/issues/747

src/lambda_codebase/event.py

@@ -46,7 +46,7 @@ def __init__(self, event, parameter_store, organizations, account_id):
         self.moved_to_protected = 1 if self.destination_ou_id in self.protected_ou_list else 0
         self.regions = ast.literal_eval(
             parameter_store.fetch_parameter('target_regions')
-        )
+        ) or []
         self.deployment_account_region = parameter_store.fetch_parameter(
             'deployment_account_region')
         self.cross_account_access_role = parameter_store.fetch_parameter(

src/lambda_codebase/initial_commit/adfconfig.yml.j2

@@ -17,5 +17,4 @@ config:
   scp:
     keep-default-scp: enabled  # determines if the default AWSFullAccess SCP stays attached to all OU's
   scm:
-    auto-create-repositories: true
-
+    auto-create-repositories: enabled

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/__init__.py

@@ -4,14 +4,3 @@
 """
 __init__ for adf-build
 """
-
-
-import sys
-import os
-
-sys.path.append(
-    os.path.abspath(
-        os.path.join(
-            os.path.dirname(__file__),
-            'shared',
-            'python')))

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/config.py

@@ -36,7 +36,7 @@ def __init__(self, parameter_store=None, config_path=None):
         self.deployment_account_region = None
         self.notification_channel = None
         self.protected = None
-        self.target_regions = None
+        self.target_regions = []
         self.cross_account_access_role = None
         self._load_config_file()
 
@@ -62,12 +62,6 @@ def _validate(self):
                 'Please see the documentation.'
             )
 
-        if not len(self.target_regions) >= 1:
-            raise InvalidConfigError(
-                'ADF requires you to have at least 1 target region '
-                'for deployments'
-            )
-
         try:
             if self.config.get('scp'):
                 assert self.config.get('scp').get('keep-default-scp') in ['enabled', 'disabled']
@@ -99,11 +93,11 @@ def _parse_config(self):
         """
         Parses the adfconfig.yml file and executes _validate
         """
-
+        regions = self.config_contents.get(
+            'regions', {}).get('targets', [])
         self.deployment_account_region = self.config_contents.get(
             'regions', None).get('deployment-account', None)
-        self.target_regions = self.config_contents.get(
-            'regions', None).get('targets', None)
+        self.target_regions = [] if regions[0] is None else regions
         self.cross_account_access_role = self.config_contents.get(
             'roles', None).get('cross-account-access', None)
         self.config = self.config_contents.get('config', None)

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/main.py

@@ -129,9 +129,6 @@ def prepare_deployment_account(sts, deployment_account_id, config):
     deployment_account_parameter_store.put_parameter(
         'deployment_account_bucket', DEPLOYMENT_ACCOUNT_S3_BUCKET_NAME
     )
-    deployment_account_parameter_store.put_parameter(
-        'adf_version', os.environ["ADF_VERSION"]
-    )
     auto_create_repositories = config.config.get('scm', {}).get('auto-create-repositories')
     if auto_create_repositories is not None:
         deployment_account_parameter_store.put_parameter(

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/requirements.txt

@@ -1,4 +1,4 @@
-# Install libs here that you might want in AWS CodeBuild
+# Install libs here that you might want in AWS CodeBuild (On Master Account)
 boto3~=1.9.89
 pylint~=2.2.2
 pytest~=3.0.7

src/lambda_codebase/initial_commit/bootstrap_repository/deployment/lambda_codebase/initial_commit/pipelines_repository/adf-build/deployment_map.py → src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/deployment_map.py

@@ -35,7 +35,6 @@ def update_deployment_parameters(self, pipeline):
                 {item['name']: item['path'] for item in account if item['name'] != 'approval'}
             )
 
-        # TODO Ensure this doesn't grow to reach max parameter store size (4092)
         self.parameter_store.put_parameter(
             "/deployment/{0}/account_ous".format(
                 pipeline.name
@@ -58,10 +57,8 @@ def _get_deployment_map(self, file_path=None):
             with open(file_path, 'r') as stream:
                 return yaml.load(stream, Loader=yaml.FullLoader)
         except FileNotFoundError:
-            LOGGER.error('Cannot Create Deployment Pipelines as there '
-                         'is no deployment_map.yml file in the repository. '
-                         'If this is your first time using ADF please see read the user guide'
-                         , exc_info=True)
+            LOGGER.debug('Nothing found at %s', file_path)
+            return None
 
     def _get_deployment_apps_from_dir(self):
         if os.path.isdir(self.map_dir_path):
@@ -88,3 +85,9 @@ def _validate_deployment_map(self):
             raise InvalidDeploymentMapError(
                 "Deployment Map target or regions specification is invalid"
             )
+        except TypeError:
+            LOGGER.error(
+                "No Deployment Map files found, create a deployment_map.yml file in the root of the repository to create pipelines. "
+                "You can create additional deployment maps if required in a folder named deployment_maps with any name (ending in .yml)"
+            )
+            raise Exception from None

src/lambda_codebase/initial_commit/bootstrap_repository/deployment/lambda_codebase/initial_commit/pipelines_repository/adf-build/generate_pipelines.py → src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/generate_pipelines.py

@@ -12,6 +12,7 @@
 from repo import Repo
 from target import Target, TargetStructure
 from logger import configure_logger
+from errors import ParameterNotFoundError
 from deployment_map import DeploymentMap
 from cloudformation import CloudFormation
 from organizations import Organizations
@@ -21,12 +22,11 @@
 LOGGER = configure_logger(__name__)
 DEPLOYMENT_ACCOUNT_REGION = os.environ.get("AWS_REGION", 'us-east-1')
 DEPLOYMENT_ACCOUNT_ID = os.environ["ACCOUNT_ID"]
-MASTER_ACCOUNT_ID = os.environ.get("MASTER_ACCOUNT_ID", 'us-east-1')
+MASTER_ACCOUNT_ID = os.environ["MASTER_ACCOUNT_ID"]
 S3_BUCKET_NAME = os.environ["S3_BUCKET_NAME"]
 ADF_PIPELINE_PREFIX = os.environ["ADF_PIPELINE_PREFIX"]
 ADF_VERSION = os.environ["ADF_VERSION"]
 ADF_LOG_LEVEL = os.environ["ADF_LOG_LEVEL"]
-TARGET_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
 
 def clean(parameter_store, deployment_map):
     """
@@ -86,8 +86,7 @@ def upload_pipeline(s3, pipeline):
     """
     s3_object_path = s3.put_object(
         "pipelines/{0}/global.yml".format(
-            pipeline.name), "{0}/{1}/{2}/global.yml".format(
-                TARGET_DIR,
+            pipeline.name), "{0}/{1}/global.yml".format(
                 'pipelines',
                 pipeline.name
             )
@@ -122,11 +121,15 @@ def main(): #pylint: disable=R0915
     organizations = Organizations(role)
     clean(parameter_store, deployment_map)
 
+    try:
+        auto_create_repositories = parameter_store.fetch_parameter('auto_create_repositories')
+    except ParameterNotFoundError:
+        auto_create_repositories = 'enabled'
+
     for p in deployment_map.map_contents.get('pipelines'):
         pipeline = Pipeline(p)
 
-        auto_create_repositories = parameter_store.fetch_parameter('auto_create_repositories')
-        if auto_create_repositories:
+        if auto_create_repositories == 'enabled':
             code_account_id = next(param['SourceAccountId'] for param in p['params'] if 'SourceAccountId' in param)
             if auto_create_repositories and code_account_id and str(code_account_id).isdigit():
                 repo = Repo(code_account_id, p.get('name'), p.get('description'))

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/pytest.ini

@@ -0,0 +1,3 @@
+[pytest]
+testpaths = tests
+norecursedirs = python

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+testpaths = tests

src/lambda_codebase/initial_commit/bootstrap_repository/deployment/lambda_codebase/initial_commit/pipelines_repository/adf-build/repo.py → src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/repo.py

@@ -7,46 +7,34 @@
 """
 
 import os
-import boto3
-from boto3.session import Session
 from cloudformation import CloudFormation
 from s3 import S3
+from sts import STS
 from logger import configure_logger
 
 LOGGER = configure_logger(__name__)
 TARGET_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
 DEPLOYMENT_ACCOUNT_ID = os.environ["ACCOUNT_ID"]
 DEPLOYMENT_ACCOUNT_REGION = os.environ.get("AWS_REGION", 'us-east-1')
 CODE_ACCOUNT_REGION = os.environ.get("AWS_REGION", 'us-east-1')
-sts = boto3.client('sts')
 S3_BUCKET_NAME = os.environ["S3_BUCKET_NAME"]
+sts = STS()
 s3 = S3(
     DEPLOYMENT_ACCOUNT_REGION,
     S3_BUCKET_NAME
 )
 class Repo:
     def __init__(self, account_id, name, description=''):
         self.name = name
-
         if not description:
             description = 'Created by ADF'
 
         self.description = description
         self.stack_name = "{0}-{1}".format('adf-codecommit', self.name)
-
         self.account_id = account_id
-
-        arn = 'arn:aws:iam::{0}:role/adf-cloudformation-deployment-role'.format(account_id)
-
-        response = sts.assume_role(
-            RoleArn=arn,
-            RoleSessionName='create_repo_{0}'.format(account_id)
-        )
-        creds = response['Credentials']
-        self.session = Session(
-            aws_access_key_id=creds['AccessKeyId'],
-            aws_secret_access_key=creds['SecretAccessKey'],
-            aws_session_token=creds['SessionToken']
+        self.session = sts.assume_cross_account_role(
+            'arn:aws:iam::{0}:role/adf-cloudformation-deployment-role'.format(account_id),
+            'create_repo_{0}'.format(account_id)
         )
 
     def repo_exists(self):
@@ -62,8 +50,8 @@ def repo_exists(self):
 
     def create_update(self):
         s3_object_path = s3.put_object(
-            "repo_templates/codecommit.yml",
-            "{0}/repo_templates/codecommit.yml".format(TARGET_DIR)
+            "adf-build/repo_templates/codecommit.yml",
+            "{0}/adf-build/repo_templates/codecommit.yml".format(TARGET_DIR)
         )
         parameters = [{
             'ParameterKey': 'RepoName',
@@ -85,10 +73,11 @@ def create_update(self):
             account_id=DEPLOYMENT_ACCOUNT_ID,
         )
 
-        # Create the repo stack if the repo is missing and
+        # Create the repo stack if the repo is missing
         create_stack = not self.repo_exists()
+
         # Update the stack if the repo and the adf contolled stack exist
         update_stack = (self.repo_exists() and cloudformation.get_stack_status())
         if create_stack or update_stack:
-            LOGGER.info('Creating Stack for Codecommit Repository %s', self.name)
+            LOGGER.info('Creating Stack for Codecommit Repository %s on Account %s', self.name, self.account_id)
             cloudformation.create_stack()

src/lambda_codebase/initial_commit/bootstrap_repository/deployment/lambda_codebase/initial_commit/pipelines_repository/repo_templates/codecommit.yml → src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/repo_templates/codecommit.yml

@@ -5,7 +5,7 @@ Parameters:
     Type: String
     Default: Created by ADF
 Resources:
-  MyRepo:
+  Repo:
     Type: AWS::CodeCommit::Repository
     Properties:
       RepositoryName: !Ref RepoName