Mcdaniel 202301 remotes (#42)

* add cookiecutter params for remote mysql, redis * add post hooks to remove unrequired modules * lint * run hooks * remove the terragrunt stuff too * tests
cookiecutter-openedx · Jan 29, 2023 · 35eb33b · 35eb33b
1 parent 24b7845
commit 35eb33b
Show file tree

Hide file tree

Showing 12 changed files with 132 additions and 113 deletions.
diff --git a/cookiecutter.json b/cookiecutter.json
@@ -14,12 +14,18 @@
     "global_aws_route53_hosted_zone_id": "Z1234567ABCDE1U23DEF",
     "global_aws_region": "us-east-1",
     "global_account_id": "123456789012",
-    "stack_install_k8s_dashboard": ["Y", "N"],
-    "stack_install_k8s_kubeapps": ["Y", "N"],
-    "stack_install_k8s_karpenter": ["Y", "N"],
-    "stack_install_k8s_prometheus": ["Y", "N"],
+    "stack_add_bastion": ["Y", "N"],
+    "stack_add_bastion_openedx_dev_environment": ["N", "Y"],
+    "stack_add_k8s_dashboard": ["Y", "N"],
+    "stack_add_k8s_kubeapps": ["Y", "N"],
+    "stack_add_k8s_karpenter": ["Y", "N"],
+    "stack_add_k8s_prometheus": ["Y", "N"],
+    "stack_add_remote_mysql":  ["Y", "N"],
+    "stack_add_remote_mongodb": ["Y", "N"],
+    "stack_add_remote_redis":  ["Y", "N"],
     "ci_build_tutor_version": "15.2.0",
-    "ci_deploy_open_edx_version": "olive.1",
+    "ci_build_kubectl_version": "1.24/stable",
+    "kubernetes_cluster_version": "1.24",
     "ci_build_theme_repository": "edx-theme-example",
     "ci_build_theme_repository_organization": "lpm0073",
     "ci_build_theme_ref": "main",
@@ -29,7 +35,7 @@
     "ci_build_xblock_org": "openedx",
     "ci_build_xblock_repository": "edx-ora2",
     "ci_build_xblock_ref": "master",
-    "ci_build_kubectl_version": "1.24/stable",
+    "ci_deploy_open_edx_version": "olive.1",
     "ci_deploy_install_backup_plugin": ["N", "Y"],
     "ci_deploy_install_credentials_server": ["N", "Y"],
     "ci_deploy_install_discovery_service": ["Y", "N"],
@@ -85,12 +91,8 @@
     "eks_karpenter_group_min_size": 3,
     "eks_karpenter_group_max_size": 10,
     "eks_karpenter_group_desired_size": 3,
-    "kubernetes_cluster_version": "1.24",
-    "stack_add_remote_mongodb": ["Y", "N"],
     "mongodb_instance_type": "t3.medium",
     "mongodb_allocated_storage": 10,
-    "stack_add_bastion": ["Y", "N"],
-    "stack_add_bastion_openedx_dev_environment": ["N", "Y"],
     "bastion_instance_type": "t3.micro",
     "bastion_allocated_storage": 50,
     "mysql_username": "root",

diff --git a/doc/README.rst b/doc/README.rst
@@ -153,7 +153,7 @@ See this example screen shot of a typical AWS resource. In this case, an AWS RDS
 Cookiecutter AWS Services Stack Installation Options
 ----------------------------------------------------
 
-- **stack_install_k8s_dashboard:**
+- **stack_add_k8s_dashboard:**
 
   'Y' to install `Kubernetes Dashboard <https://kubernetes.io/docs/tasks/access-application-cluster/web-ui-dashboard/>`_
   in the AWS EKS cluster and add an ingress, ssl-tls cert and url endpoint to global_services_subdomain.
@@ -162,7 +162,7 @@ Cookiecutter AWS Services Stack Installation Options
 
   *default value: Y*
 
-- **stack_install_k8s_kubeapps:**
+- **stack_add_k8s_kubeapps:**
 
   'Y' to install `VMWare Bitnami Kubeapps <https://kubeapps.dev/>`_
   in the AWS EKS cluster and add an ingress, ssl-tls cert and url endpoint to global_services_subdomain.
@@ -171,7 +171,7 @@ Cookiecutter AWS Services Stack Installation Options
 
   *default value: Y*
 
-- **stack_install_k8s_karpenter:**
+- **stack_add_k8s_karpenter:**
 
   'Y' to install `Karpenter <https://karpenter.sh/>`_ in the AWS EKS cluster.
 
@@ -181,7 +181,7 @@ Cookiecutter AWS Services Stack Installation Options
 
   *default value: Y*
 
-- **stack_install_k8s_prometheus:**
+- **stack_add_k8s_prometheus:**
 
   'Y' to install `Prometheus <https://prometheus.io/`_ in the AWS EKS cluster. This is required if you chose
   to install Karpenter.

diff --git a/hooks/post_gen_project.py b/hooks/post_gen_project.py
@@ -13,106 +13,121 @@ def rm_directory(dir_path: str):
 HINT = "\x1b[3;33m"
 SUCCESS = "\x1b[1;32m [SUCCESS]: "
 
-def remove_k8s_dashboard():
-    dir_path = os.path.join("terraform", "stacks", "modules", "kubernetes_dashboard")
+def remove_redis():
+    dir_path = os.path.join("terraform", "stacks", "modules", "redis")
     rm_directory(dir_path)
 
-    dir_path = os.path.join("terraform", "environments", "modules", "kubernetes_dashboard")
+    dir_path = os.path.join("terraform", "stacks", "{{ cookiecutter.global_platform_shared_resource_identifier }}", "redis")
     rm_directory(dir_path)
 
-def remove_k8s_kubeapps():
-    dir_path = os.path.join("terraform", "stacks", "modules", "kubernetes_kubeapps")
+    dir_path = os.path.join("terraform", "environments", "modules", "redis")
     rm_directory(dir_path)
 
-    dir_path = os.path.join("terraform", "environments", "modules", "kubernetes_kubeapps")
+    dir_path = os.path.join("terraform", "environments", "{{ cookiecutter.environment_name }}", "redis")
     rm_directory(dir_path)
 
-def remove_k8s_karpenter():
-    dir_path = os.path.join("terraform", "stacks", "modules", "kubernetes_karpenter")
+def remove_mysql():
+    dir_path = os.path.join("terraform", "stacks", "modules", "mysql")
     rm_directory(dir_path)
 
-    dir_path = os.path.join("terraform", "environments", "modules", "kubernetes_karpenter")
+    dir_path = os.path.join("terraform", "stacks", "{{ cookiecutter.global_platform_shared_resource_identifier }}", "mysql")
     rm_directory(dir_path)
 
-def remove_k8s_prometheus():
-    dir_path = os.path.join("terraform", "stacks", "modules", "kubernetes_prometheus")
+    dir_path = os.path.join("terraform", "environments", "modules", "mysql")
     rm_directory(dir_path)
 
-    dir_path = os.path.join("terraform", "environments", "modules", "kubernetes_prometheus")
+    dir_path = os.path.join("terraform", "environments", "{{ cookiecutter.environment_name }}", "mysql")
     rm_directory(dir_path)
 
-def remove_bastion():
-    dir_path = os.path.join("terraform", "stacks", "modules", "ec2_bastion")
+def remove_mongodb():
+    dir_path = os.path.join("terraform", "stacks", "modules", "mongodb")
     rm_directory(dir_path)
 
-    dir_path = os.path.join(
-        "terraform", "stacks", "{{ cookiecutter.environment_name }}", "ec2_bastion"
-    )
+    dir_path = os.path.join("terraform", "stacks", "{{ cookiecutter.global_platform_shared_resource_identifier }}", "mongodb")
     rm_directory(dir_path)
 
-    dir_path = os.path.join("terraform", "environments", "modules", "ec2_bastion")
+    dir_path = os.path.join("terraform", "environments", "modules", "mongodb")
     rm_directory(dir_path)
 
-    dir_path = os.path.join(
-        "terraform",
-        "environments",
-        "{{ cookiecutter.environment_name }}",
-        "ec2_bastion",
-    )
+    dir_path = os.path.join("terraform", "environments", "{{ cookiecutter.environment_name }}", "mongodb")
     rm_directory(dir_path)
 
+def remove_mongodb_volume():
+    dir_path = os.path.join("terraform", "stacks", "modules", "mongodb_volume")
+    rm_directory(dir_path)
 
-def remove_mongodb():
-    dir_path = os.path.join("terraform", "stacks", "modules", "mongodb")
+    dir_path = os.path.join("terraform", "stacks", "{{ cookiecutter.global_platform_shared_resource_identifier }}", "mongodb_volume")
     rm_directory(dir_path)
 
-    dir_path = os.path.join(
-        "terraform",
-        "stacks",
-        "{{ cookiecutter.global_platform_shared_resource_identifier }}",
-        "mongodb",
-    )
+    dir_path = os.path.join("terraform", "environments", "modules", "mongodb_volume")
     rm_directory(dir_path)
 
-    dir_path = os.path.join("terraform", "stacks", "modules", "mongodb_volume")
+    dir_path = os.path.join("terraform", "environments", "{{ cookiecutter.environment_name }}", "mongodb_volume")
     rm_directory(dir_path)
 
-    dir_path = os.path.join(
-        "terraform",
-        "stacks",
-        "{{ cookiecutter.global_platform_shared_resource_identifier }}",
-        "mongodb_volume",
-    )
+def remove_k8s_dashboard():
+    dir_path = os.path.join("terraform", "stacks", "modules", "kubernetes_dashboard")
     rm_directory(dir_path)
 
-    dir_path = os.path.join(
-        "terraform", "environments", "{{ cookiecutter.environment_name }}", "mongodb"
-    )
+    dir_path = os.path.join("terraform", "stacks", "{{ cookiecutter.global_platform_shared_resource_identifier }}", "kubernetes_dashboard")
     rm_directory(dir_path)
 
-    dir_path = os.path.join("terraform", "environments", "modules", "mongodb")
+def remove_k8s_kubeapps():
+    dir_path = os.path.join("terraform", "stacks", "modules", "kubernetes_kubeapps")
+    rm_directory(dir_path)
+
+    dir_path = os.path.join("terraform", "stacks", "{{ cookiecutter.global_platform_shared_resource_identifier }}", "kubernetes_kubeapps")
+    rm_directory(dir_path)
+
+def remove_k8s_karpenter():
+    dir_path = os.path.join("terraform", "stacks", "modules", "kubernetes_karpenter")
+    rm_directory(dir_path)
+
+    dir_path = os.path.join("terraform", "stacks", "{{ cookiecutter.global_platform_shared_resource_identifier }}", "kubernetes_karpenter")
+    rm_directory(dir_path)
+
+def remove_k8s_prometheus():
+    dir_path = os.path.join("terraform", "stacks", "modules", "kubernetes_prometheus")
+    rm_directory(dir_path)
+
+    dir_path = os.path.join("terraform", "stacks", "{{ cookiecutter.global_platform_shared_resource_identifier }}", "kubernetes_prometheus")
     rm_directory(dir_path)
 
+def remove_bastion():
+    dir_path = os.path.join("terraform", "stacks", "modules", "ec2_bastion")
+    rm_directory(dir_path)
+
+    dir_path = os.path.join(
+        "terraform", "stacks", "{{ cookiecutter.environment_name }}", "ec2_bastion"
+    )
+    rm_directory(dir_path)
 
 def main():
     if "{{ cookiecutter.stack_add_bastion }}".upper() != "Y":
         remove_bastion()
 
-    if "{{ cookiecutter.stack_add_remote_mongodb }}".upper() != "Y":
-        remove_mongodb()
-
-    if "{{ cookiecutter.stack_install_k8s_dashboard }}".upper() != "Y":
+    if "{{ cookiecutter.stack_add_k8s_dashboard }}".upper() != "Y":
         remove_k8s_dashboard()
 
-    if "{{ cookiecutter.stack_install_k8s_kubeapps }}".upper() != "Y":
+    if "{{ cookiecutter.stack_add_k8s_kubeapps }}".upper() != "Y":
         remove_k8s_kubeapps()
 
-    if "{{ cookiecutter.stack_install_k8s_karpenter }}".upper() != "Y":
+    if "{{ cookiecutter.stack_add_k8s_karpenter }}".upper() != "Y":
         remove_k8s_karpenter()
 
-    if "{{ cookiecutter.stack_install_k8s_prometheus }}".upper() != "Y":
+    if "{{ cookiecutter.stack_add_k8s_prometheus }}".upper() != "Y":
         remove_k8s_prometheus()
 
+    if "{{ cookiecutter.stack_add_remote_redis }}".upper() != "Y":
+        remove_redis()
+
+    if "{{ cookiecutter.stack_add_remote_mysql }}".upper() != "Y":
+        remove_mysql()
+
+    if "{{ cookiecutter.stack_add_remote_mongodb }}".upper() != "Y":
+        remove_mongodb()
+        remove_mongodb_volume
+
     print(SUCCESS + "Your Open edX devops repo has been initialized." + TERMINATOR)
 
 

diff --git a/{{cookiecutter.github_repo_name}}/README.rst b/{{cookiecutter.github_repo_name}}/README.rst
@@ -63,13 +63,13 @@ Services Endpoints
 - **Bastion**: bastion.{{ cookiecutter.global_services_subdomain }}.{{ cookiecutter.global_root_domain }}:22. Public ssh access to a {{ cookiecutter.bastion_instance_type }} Ubuntu 20.04 LTS bastion EC2 instance that's preconfigure with all of the software that you'll need to adminster this stack.
 - **MySQL**: mysql.{{ cookiecutter.global_services_subdomain }}.{{ cookiecutter.global_root_domain }}:3306. Private VPC access to your AWS RDS MySQL {{ cookiecutter.mysql_instance_class }} instance with allocated storage of {{ cookiecutter.mysql_allocated_storage }}.
 - **MongoDB**: mongodb.{{ cookiecutter.global_services_subdomain }}.{{ cookiecutter.global_root_domain }}:27017. Private VPC access to your EC2-based installation of MongoDB on a {{ cookiecutter.mongodb_instance_type }} instance with allocated storage of {{ cookiecutter.mongodb_allocated_storage }}.
-{% if cookiecutter.stack_install_k8s_dashboard|upper == "Y" -%}
+{% if cookiecutter.stack_add_k8s_dashboard|upper == "Y" -%}
 - **Kubernetes Dashboard**: https://dashboard.{{ cookiecutter.global_services_subdomain }}.{{ cookiecutter.global_root_domain }}. Dashboard is a web-based Kubernetes user interface. You can use Dashboard to deploy containerized applications to a Kubernetes cluster, troubleshoot your containerized application, and manage the cluster resources. You can use Dashboard to get an overview of applications running on your cluster, as well as for creating or modifying individual Kubernetes resources (such as Deployments, Jobs, DaemonSets, etc). For example, you can scale a Deployment, initiate a rolling update, restart a pod or deploy new applications using a deploy wizard.
 {% endif -%}
-{% if cookiecutter.stack_install_k8s_kubeapps|upper == "Y" -%}
+{% if cookiecutter.stack_add_k8s_kubeapps|upper == "Y" -%}
 - **Kubeapps**: https://kubeapps.{{ cookiecutter.global_services_subdomain }}.{{ cookiecutter.global_root_domain }}. Kubeapps is an in-cluster web-based application that enables users with a one-time installation to deploy, manage, and upgrade applications on a Kubernetes cluster
 {% endif -%}
-{% if cookiecutter.stack_install_k8s_prometheus|upper == "Y" -%}
+{% if cookiecutter.stack_add_k8s_prometheus|upper == "Y" -%}
 - **Grafana**: https://grafana.{{ cookiecutter.global_services_subdomain }}.{{ cookiecutter.global_root_domain }}. Grafana is a multi-platform open source analytics and interactive visualization web application. It provides charts, graphs, and alerts for the web when connected to supported data sources.
 {% endif -%}
 

diff --git a/{{cookiecutter.github_repo_name}}/make.sh b/{{cookiecutter.github_repo_name}}/make.sh
@@ -13,38 +13,44 @@ GITHUB_BRANCH="main"
 OUTPUT_FOLDER="../"
 
 cookiecutter --checkout $GITHUB_BRANCH \
-             --output-dir $OUTPUT_FOLDER \
-             --overwrite-if-exists \
-             --no-input \
-             $GITHUB_REPO \
-             ci_deploy_install_credentials_server={{ cookiecutter.ci_deploy_install_credentials_server }} \
-             ci_deploy_install_license_manager_service={{ cookiecutter.ci_deploy_install_license_manager_service }} \
-             ci_deploy_install_discovery_service={{ cookiecutter.ci_deploy_install_discovery_service }} \
-             ci_deploy_install_mfe_service={{ cookiecutter.ci_deploy_install_mfe_service }} \
-             ci_deploy_install_notes_service={{ cookiecutter.ci_deploy_install_notes_service }} \
-             ci_deploy_install_ecommerce_service={{ cookiecutter.ci_deploy_install_ecommerce_service }} \
-             global_platform_name={{ cookiecutter.global_platform_name }} \
-             global_platform_region={{ cookiecutter.global_platform_region }} \
-             global_aws_region={{ cookiecutter.global_aws_region }} \
-             global_account_id={{ cookiecutter.global_account_id }} \
-             global_root_domain={{ cookiecutter.global_root_domain }} \
-             global_aws_route53_hosted_zone_id={{ cookiecutter.global_aws_route53_hosted_zone_id }} \
-             environment_name={{ cookiecutter.environment_name }} \
-             environment_subdomain={{ cookiecutter.environment_subdomain }} \
-             eks_worker_group_instance_type={{ cookiecutter.eks_worker_group_instance_type }} \
-             eks_worker_group_min_size={{ cookiecutter.eks_worker_group_min_size }} \
-             eks_worker_group_max_size={{ cookiecutter.eks_worker_group_max_size }} \
-             eks_worker_group_desired_size={{ cookiecutter.eks_worker_group_desired_size }} \
-             eks_karpenter_group_instance_type={{ cookiecutter.eks_karpenter_group_instance_type }} \
-             eks_karpenter_group_min_size={{ cookiecutter.eks_karpenter_group_min_size }} \
-             eks_karpenter_group_max_size={{ cookiecutter.eks_karpenter_group_max_size }} \
-             eks_karpenter_group_desired_size={{ cookiecutter.eks_karpenter_group_desired_size }} \
-             mysql_instance_class={{ cookiecutter.mysql_instance_class }} \
-             mysql_allocated_storage={{ cookiecutter.mysql_allocated_storage }} \
-             redis_node_type={{ cookiecutter.redis_node_type }} \
-             stack_add_bastion={{ cookiecutter.stack_add_bastion }} \
-             stack_add_remote_mongodb={{ cookiecutter.stack_add_remote_mongodb }} \
-             {% if cookiecutter.stack_add_remote_mongodb == "Y" -%}
-             mongodb_instance_type={{ cookiecutter.mongodb_instance_type }} \
-             mongodb_allocated_storage={{ cookiecutter.mongodb_allocated_storage }} \
-             {% endif %}
+            --output-dir $OUTPUT_FOLDER \
+            --overwrite-if-exists \
+            --no-input \
+            $GITHUB_REPO \
+            ci_deploy_install_credentials_server={{ cookiecutter.ci_deploy_install_credentials_server }} \
+            ci_deploy_install_license_manager_service={{ cookiecutter.ci_deploy_install_license_manager_service }} \
+            ci_deploy_install_discovery_service={{ cookiecutter.ci_deploy_install_discovery_service }} \
+            ci_deploy_install_mfe_service={{ cookiecutter.ci_deploy_install_mfe_service }} \
+            ci_deploy_install_notes_service={{ cookiecutter.ci_deploy_install_notes_service }} \
+            ci_deploy_install_ecommerce_service={{ cookiecutter.ci_deploy_install_ecommerce_service }} \
+            global_platform_name={{ cookiecutter.global_platform_name }} \
+            global_platform_region={{ cookiecutter.global_platform_region }} \
+            global_aws_region={{ cookiecutter.global_aws_region }} \
+            global_account_id={{ cookiecutter.global_account_id }} \
+            global_root_domain={{ cookiecutter.global_root_domain }} \
+            global_aws_route53_hosted_zone_id={{ cookiecutter.global_aws_route53_hosted_zone_id }} \
+            environment_name={{ cookiecutter.environment_name }} \
+            environment_subdomain={{ cookiecutter.environment_subdomain }} \
+            eks_worker_group_instance_type={{ cookiecutter.eks_worker_group_instance_type }} \
+            eks_worker_group_min_size={{ cookiecutter.eks_worker_group_min_size }} \
+            eks_worker_group_max_size={{ cookiecutter.eks_worker_group_max_size }} \
+            eks_worker_group_desired_size={{ cookiecutter.eks_worker_group_desired_size }} \
+            eks_karpenter_group_instance_type={{ cookiecutter.eks_karpenter_group_instance_type }} \
+            eks_karpenter_group_min_size={{ cookiecutter.eks_karpenter_group_min_size }} \
+            eks_karpenter_group_max_size={{ cookiecutter.eks_karpenter_group_max_size }} \
+            eks_karpenter_group_desired_size={{ cookiecutter.eks_karpenter_group_desired_size }} \
+            mysql_instance_class={{ cookiecutter.mysql_instance_class }} \
+            mysql_allocated_storage={{ cookiecutter.mysql_allocated_storage }} \
+            redis_node_type={{ cookiecutter.redis_node_type }} \
+            stack_add_bastion={{ cookiecutter.stack_add_bastion }} \
+            stack_add_k8s_dashboard={{ cookiecutter.stack_add_k8s_dashboard }} \
+            stack_add_k8s_kubeapps={{ cookiecutter.stack_add_k8s_kubeapps }} \
+            stack_add_k8s_karpenter={{ cookiecutter.stack_add_k8s_karpenter }} \
+            stack_add_k8s_prometheus={{ cookiecutter.stack_add_k8s_prometheus }} \
+            stack_add_remote_mysql={{ cookiecutter.stack_add_remote_mysql }} \
+            stack_add_remote_mongodb={{ cookiecutter.stack_add_remote_mongodb }} \
+            stack_add_remote_redis={{ cookiecutter.stack_add_remote_redis }} \
+            {% if cookiecutter.stack_add_remote_mongodb == "Y" -%}
+            mongodb_instance_type={{ cookiecutter.mongodb_instance_type }} \
+            mongodb_allocated_storage={{ cookiecutter.mongodb_allocated_storage }} \
+            {% endif %}
diff --git a/{{cookiecutter.github_repo_name}}/terraform/environments/modules/acm/outputs.tf b/{{cookiecutter.github_repo_name}}/terraform/environments/modules/acm/outputs.tf
@@ -1,5 +1,5 @@
 
 
 output "acm_root_domain_environment_region_acm_certificate_status" {
-  value       = module.acm_root_domain_environment_region.acm_certificate_status
+  value = module.acm_root_domain_environment_region.acm_certificate_status
 }
diff --git a/...utter.github_repo_name}}/terraform/environments/modules/cloudfront/certificate_manager.tf b/...utter.github_repo_name}}/terraform/environments/modules/cloudfront/certificate_manager.tf
@@ -7,7 +7,6 @@
 # usage: Add DNS records and tls certs to environment aws_region for ALB.
 # Also add certs to us-east-1 for Cloudfront distributions.
 #------------------------------------------------------------------------------
-
 {% if cookiecutter.global_aws_region != "us-east-1" -%}
 #------------------------------------------------------------------------------
 # SSL/TLS certs issued in the AWS region for ALB

diff --git a/{{cookiecutter.github_repo_name}}/terraform/environments/modules/cloudfront/main.tf b/{{cookiecutter.github_repo_name}}/terraform/environments/modules/cloudfront/main.tf
@@ -40,9 +40,7 @@ data "aws_acm_certificate" "environment_domain" {
   domain   = var.environment_domain
   statuses = ["ISSUED"]
   provider = aws.us-east-1
-  {% if cookiecutter.global_aws_region != "us-east-1" -%}
-  depends_on = [module.acm_environment_domain]
-  {% endif %}
+  {% if cookiecutter.global_aws_region != "us-east-1" -%}depends_on = [module.acm_environment_domain]{% endif %}
 }
 
 data "aws_s3_bucket" "environment_domain" {