From 4647263c21d3d130864e9ec51211537a271b3552 Mon Sep 17 00:00:00 2001 From: Etienne Date: Thu, 8 Feb 2024 20:13:36 -0500 Subject: [PATCH 01/12] Add fact prefix per node Include hieradata subfolder structure for hostname and prefix --- aws/infrastructure.tf | 1 + azure/infrastructure.tf | 1 + common/configuration/main.tf | 3 +- common/configuration/puppet.yaml | 10 +++ common/provision/main.tf | 138 ++++++++++++++++++++++--------- common/variables.tf | 6 ++ docs/README.md | 14 ++++ gcp/infrastructure.tf | 1 + openstack/infrastructure.tf | 1 + 9 files changed, 136 insertions(+), 39 deletions(-) diff --git a/aws/infrastructure.tf b/aws/infrastructure.tf index d8e12d34..881305ea 100644 --- a/aws/infrastructure.tf +++ b/aws/infrastructure.tf @@ -42,6 +42,7 @@ module "provision" { terraform_data = module.configuration.terraform_data terraform_facts = module.configuration.terraform_facts hieradata = var.hieradata + hieradata_folder = var.hieradata_folder_path sudoer_username = var.sudoer_username eyaml_key = var.eyaml_key depends_on = [aws_instance.instances, aws_eip.public_ip] diff --git a/azure/infrastructure.tf b/azure/infrastructure.tf index 0f0716b2..735b5675 100644 --- a/azure/infrastructure.tf +++ b/azure/infrastructure.tf @@ -42,6 +42,7 @@ module "provision" { terraform_data = module.configuration.terraform_data terraform_facts = module.configuration.terraform_facts hieradata = var.hieradata + hieradata_folder = var.hieradata_folder_path sudoer_username = var.sudoer_username eyaml_key = var.eyaml_key depends_on = [ azurerm_linux_virtual_machine.instances ] diff --git a/common/configuration/main.tf b/common/configuration/main.tf index 057f1ded..fdb398a7 100644 --- a/common/configuration/main.tf +++ b/common/configuration/main.tf @@ -86,7 +86,7 @@ locals { }) terraform_facts = yamlencode({ - software_stack = var.software_stack + software_stack = var.software_stack, cloud = { provider = var.cloud_provider region = var.cloud_region @@ -100,6 +100,7 @@ locals { cloud_provider = var.cloud_provider tags = values.tags node_name = key, + node_prefix = values.prefix, domain_name = var.domain_name puppetenv_git = var.config_git_url, puppetenv_rev = var.config_version, diff --git a/common/configuration/puppet.yaml b/common/configuration/puppet.yaml index cfdfeee1..0afe0333 100644 --- a/common/configuration/puppet.yaml +++ b/common/configuration/puppet.yaml @@ -90,6 +90,7 @@ runcmd: - chgrp -R puppet /etc/puppetlabs/data /etc/puppetlabs/facts - ln -sf /etc/puppetlabs/data/terraform_data.yaml /etc/puppetlabs/code/environments/production/data/ - ln -sf /etc/puppetlabs/data/user_data.yaml /etc/puppetlabs/code/environments/production/data/ + - ln -sf /etc/puppetlabs/data/hieradata /etc/puppetlabs/code/environments/production/data/ - ln -sf /etc/puppetlabs/facts/terraform_facts.yaml /etc/puppetlabs/code/environments/production/site/profile/facts.d # We use r10k solely to install the modules of the main branch environment. - "(cd /etc/puppetlabs/code/environments/production; /opt/puppetlabs/puppet/bin/r10k puppetfile install)" @@ -140,8 +141,17 @@ write_files: %{ if cloud_provider != "gcp" } "GCE", %{ endif } ], } + global : { + external-dir : ["/etc/puppetlabs/facts/external"], + } + path: /etc/puppetlabs/facter/facter.conf permissions: "0644" + - path: /etc/puppetlabs/facts/external/prefix.yaml + content: | + --- + prefix : "${node_prefix}" + permissions: "0640" %{ if contains(tags, "puppet") ~} - content: | --- diff --git a/common/provision/main.tf b/common/provision/main.tf index 354fa23e..379f030e 100644 --- a/common/provision/main.tf +++ b/common/provision/main.tf @@ -2,59 +2,94 @@ variable "bastions" { } variable "puppetservers" { } variable "terraform_data" { } variable "terraform_facts" { } +variable "hieradata_folder" { } variable "hieradata" { } variable "sudoer_username" { } variable "tf_ssh_key" { } variable "eyaml_key" { } -locals { - provision_folder = "puppetserver_etc" +data "local_file" "hieradata_yaml" { + for_each = var.hieradata_folder != "" ? fileset("${var.hieradata_folder}", "*.yaml") : [] + filename = "${var.hieradata_folder}/${each.value}" } -data "archive_file" "puppetserver_files" { - type = "zip" - output_path = "${path.module}/files/${local.provision_folder}.zip" +data "local_file" "hieradata_subfolder" { + for_each = var.hieradata_folder != "" ? fileset("${var.hieradata_folder}", "{prefix,hostname}/**/*.yaml") : [] + filename = "${var.hieradata_folder}/${each.value}" +} - source { - content = var.terraform_data - filename = "${local.provision_folder}/data/terraform_data.yaml" +locals { + connection_parameters = length(var.bastions) > 0 ? { + bastion_host = var.bastions[keys(var.bastions)[0]].public_ip + bastion_user = var.sudoer_username + bastion_private_key = var.tf_ssh_key.private + user = var.sudoer_username + private_key = var.tf_ssh_key.private + } : null + + hieradata_md5 = merge( + {for value in data.local_file.hieradata_yaml: replace(value.filename, var.hieradata_folder, "") => value.content_md5}, + {for value in data.local_file.hieradata_subfolder: replace(value.filename, var.hieradata_folder, "") => value.content_md5}, + ) + triggers_hieradata_folder = { + hieradata_yaml = local.hieradata_md5 } - source { - content = var.terraform_facts - filename = "${local.provision_folder}/facts/terraform_facts.yaml" + triggers_hieradata = { + user_data = md5(var.hieradata) + terraform_data = md5(var.terraform_data) + facts = md5(var.terraform_facts) } +} + +resource "terraform_data" "deploy_hieradata_folder" { + for_each = local.connection_parameters != null && length(local.hieradata_md5) > 0 ? var.puppetservers : { } - source { - content = var.hieradata - filename = "${local.provision_folder}/data/user_data.yaml" + connection { + type = "ssh" + host = each.value + bastion_host = local.connection_parameters["bastion_host"] + bastion_user = local.connection_parameters["bastion_user"] + bastion_private_key = local.connection_parameters["bastion_private_key"] + user = local.connection_parameters["user"] + private_key = local.connection_parameters["private_key"] + } + + triggers_replace = local.triggers_hieradata_folder + + provisioner "file" { + source = "${path.cwd}/hieradata" + destination = "hieradata" } - dynamic "source" { - for_each = var.eyaml_key != "" ? [var.eyaml_key] : [] - content { - content = var.eyaml_key - filename = "${local.provision_folder}/puppet/eyaml/private_key.pkcs7.pem" - } + provisioner "remote-exec" { + inline = [ + # clean up + "sudo rm -rf /etc/puppetlabs/data/hieradata || true", + "sudo mkdir -p /etc/puppetlabs/data", + # puppet user and group have been assigned the reserved UID/GID 52 + "sudo cp -r hieradata /etc/puppetlabs/data/", + "sudo chown -R root:52 /etc/puppetlabs/data/hieradata", + "sudo chmod -R 650 /etc/puppetlabs/data/hieradata", + "rm -rf hieradata", + ] } } -resource "terraform_data" "deploy_puppetserver_files" { - for_each = length(var.bastions) > 0 ? var.puppetservers : { } +resource "terraform_data" "deploy_hieradata" { + for_each = local.connection_parameters != null ? var.puppetservers : { } connection { type = "ssh" - bastion_host = var.bastions[keys(var.bastions)[0]].public_ip - bastion_user = var.sudoer_username - bastion_private_key = var.tf_ssh_key.private - user = var.sudoer_username host = each.value - private_key = var.tf_ssh_key.private + bastion_host = local.connection_parameters["bastion_host"] + bastion_user = local.connection_parameters["bastion_user"] + bastion_private_key = local.connection_parameters["bastion_private_key"] + user = local.connection_parameters["user"] + private_key = local.connection_parameters["private_key"] } - triggers_replace = { - archive = data.archive_file.puppetserver_files.output_sha256 - } + triggers_replace = local.triggers_hieradata provisioner "file" { source = "${path.module}/files/${local.provision_folder}.zip" @@ -63,14 +98,41 @@ resource "terraform_data" "deploy_puppetserver_files" { provisioner "remote-exec" { inline = [ - # unzip is not necessarily installed when connecting, but python is. - "/usr/libexec/platform-python -c 'import zipfile; zipfile.ZipFile(\"${local.provision_folder}.zip\").extractall()'", - "sudo chmod g-w,o-rwx $(find ${local.provision_folder}/ -type f)", - "sudo chown -R root:52 ${local.provision_folder}", - "sudo mkdir -p -m 755 /etc/puppetlabs/", - "sudo rsync -avh --no-t ${local.provision_folder}/ /etc/puppetlabs/", - "sudo rm -rf ${local.provision_folder}/ ${local.provision_folder}.zip", - "[ -f /usr/local/bin/consul ] && [ -f /usr/bin/jq ] && consul event -token=$(sudo jq -r .acl.tokens.agent /etc/consul/config.json) -name=puppet $(date +%s) || true", + "sudo mkdir -p /etc/puppetlabs/data /etc/puppetlabs/facts", + # puppet user and group have been assigned the reserved UID/GID 52 + "sudo install -o root -g 52 -m 640 terraform_data.yaml user_data.yaml /etc/puppetlabs/data/", + "sudo install -o root -g 52 -m 640 terraform_facts.yaml /etc/puppetlabs/facts/", + "rm -f terraform_data.yaml user_data.yaml terraform_facts.yaml", + ] + } +} + +resource "terraform_data" "update_consul" { + for_each = local.connection_parameters != null ? var.puppetservers : { } + + connection { + type = "ssh" + host = each.value + bastion_host = local.connection_parameters["bastion_host"] + bastion_user = local.connection_parameters["bastion_user"] + bastion_private_key = local.connection_parameters["bastion_private_key"] + user = local.connection_parameters["user"] + private_key = local.connection_parameters["private_key"] + } + + triggers_replace = merge( + local.triggers_hieradata, + local.triggers_hieradata_folder, + ) + + depends_on = [ + terraform_data.deploy_hieradata, + terraform_data.deploy_hieradata_folder, + ] + + provisioner "remote-exec" { + inline = [ + "[ -f /usr/local/bin/consul ] && [ -f /usr/bin/jq ] && consul event -token=$(sudo jq -r .acl.tokens.agent /etc/consul/config.json) -name=puppet $(date +%s) || true" ] } } diff --git a/common/variables.tf b/common/variables.tf index c22c57cb..8e481e6a 100644 --- a/common/variables.tf +++ b/common/variables.tf @@ -94,6 +94,12 @@ variable "hieradata" { } } +variable "hieradata_folder_path" { + type = string + default = "" + description = "Path to hieradata folder containing YAML files to be included in the puppet environment" +} + variable "sudoer_username" { type = string default = "centos" diff --git a/docs/README.md b/docs/README.md index f552657a..78111fb4 100644 --- a/docs/README.md +++ b/docs/README.md @@ -956,6 +956,20 @@ the provided content will replace entirely the Magic Castle environment's **Post build modification effect**: None. To modify the Puppetfile after the cluster is initialized, log on the Puppet server and modify `/etc/puppetlabs/code/environments/production/Puppetfile`. +### 4.20 hieradata_folder_path (optional) + +**Default Value:** Empty string + +This parameter specifies the path to a folder containing multiple YAML files used for configuring hieradata based on instance prefixes or hostnames. + +**Folder Structure:** + +- All Instances: `*.yaml` +- Per Instance Prefix: `/*.yaml` and `.yaml` +- Per Instance Hostname: `/*.yaml` and `.yaml` + +For more information on hieradata definition, refer to section [4.13 hieradata (optional)](#413-hieradata-optional). + ## 5. Cloud Specific Configuration ### 5.1 Amazon Web Services diff --git a/gcp/infrastructure.tf b/gcp/infrastructure.tf index 068b2f5c..400081fa 100644 --- a/gcp/infrastructure.tf +++ b/gcp/infrastructure.tf @@ -42,6 +42,7 @@ module "provision" { terraform_data = module.configuration.terraform_data terraform_facts = module.configuration.terraform_facts hieradata = var.hieradata + hieradata_folder = var.hieradata_folder_path sudoer_username = var.sudoer_username eyaml_key = var.eyaml_key depends_on = [ google_compute_instance.instances ] diff --git a/openstack/infrastructure.tf b/openstack/infrastructure.tf index 4c89a959..655b7a22 100644 --- a/openstack/infrastructure.tf +++ b/openstack/infrastructure.tf @@ -37,6 +37,7 @@ module "provision" { terraform_data = module.configuration.terraform_data terraform_facts = module.configuration.terraform_facts hieradata = var.hieradata + hieradata_folder = var.hieradata_folder_path sudoer_username = var.sudoer_username eyaml_key = var.eyaml_key depends_on = [ From 251e180dc078b35e724fb3a71e9f32ef65b31768 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Tue, 7 May 2024 17:15:37 -0400 Subject: [PATCH 02/12] Put yaml files in zip --- common/provision/main.tf | 152 ++++++++++++++------------------------- 1 file changed, 55 insertions(+), 97 deletions(-) diff --git a/common/provision/main.tf b/common/provision/main.tf index 379f030e..11934009 100644 --- a/common/provision/main.tf +++ b/common/provision/main.tf @@ -2,94 +2,79 @@ variable "bastions" { } variable "puppetservers" { } variable "terraform_data" { } variable "terraform_facts" { } -variable "hieradata_folder" { } variable "hieradata" { } +variable "hieradata_folder" { } variable "sudoer_username" { } variable "tf_ssh_key" { } variable "eyaml_key" { } -data "local_file" "hieradata_yaml" { - for_each = var.hieradata_folder != "" ? fileset("${var.hieradata_folder}", "*.yaml") : [] - filename = "${var.hieradata_folder}/${each.value}" -} - -data "local_file" "hieradata_subfolder" { - for_each = var.hieradata_folder != "" ? fileset("${var.hieradata_folder}", "{prefix,hostname}/**/*.yaml") : [] - filename = "${var.hieradata_folder}/${each.value}" +locals { + provision_folder = "puppetserver_etc" } -locals { - connection_parameters = length(var.bastions) > 0 ? { - bastion_host = var.bastions[keys(var.bastions)[0]].public_ip - bastion_user = var.sudoer_username - bastion_private_key = var.tf_ssh_key.private - user = var.sudoer_username - private_key = var.tf_ssh_key.private - } : null +data "archive_file" "puppetserver_files" { + type = "zip" + output_path = "${path.module}/files/${local.provision_folder}.zip" - hieradata_md5 = merge( - {for value in data.local_file.hieradata_yaml: replace(value.filename, var.hieradata_folder, "") => value.content_md5}, - {for value in data.local_file.hieradata_subfolder: replace(value.filename, var.hieradata_folder, "") => value.content_md5}, - ) - triggers_hieradata_folder = { - hieradata_yaml = local.hieradata_md5 + source { + content = var.terraform_data + filename = "${local.provision_folder}/data/terraform_data.yaml" } - triggers_hieradata = { - user_data = md5(var.hieradata) - terraform_data = md5(var.terraform_data) - facts = md5(var.terraform_facts) + source { + content = var.terraform_facts + filename = "${local.provision_folder}/facts/terraform_facts.yaml" } -} -resource "terraform_data" "deploy_hieradata_folder" { - for_each = local.connection_parameters != null && length(local.hieradata_md5) > 0 ? var.puppetservers : { } - connection { - type = "ssh" - host = each.value - bastion_host = local.connection_parameters["bastion_host"] - bastion_user = local.connection_parameters["bastion_user"] - bastion_private_key = local.connection_parameters["bastion_private_key"] - user = local.connection_parameters["user"] - private_key = local.connection_parameters["private_key"] + source { + content = var.hieradata + filename = "${local.provision_folder}/data/user_data.yaml" } - triggers_replace = local.triggers_hieradata_folder + dynamic "source" { + for_each = var.hieradata_folder != "" ? fileset("${var.hieradata_folder}", "*.yaml") : [] + iterator = filename + content { + content = file("${var.hieradata_folder}/${filename.value}") + filename = "${local.provision_folder}/data/${filename.value}" + } + } - provisioner "file" { - source = "${path.cwd}/hieradata" - destination = "hieradata" + dynamic "source" { + for_each = var.hieradata_folder != "" ? fileset("${var.hieradata_folder}", "{prefix,hostname}/**/*.yaml") : [] + iterator = filename + content { + content = file("${var.hieradata_folder}/${filename.value}") + filename = "${local.provision_folder}/data/${filename.value}" + } } - provisioner "remote-exec" { - inline = [ - # clean up - "sudo rm -rf /etc/puppetlabs/data/hieradata || true", - "sudo mkdir -p /etc/puppetlabs/data", - # puppet user and group have been assigned the reserved UID/GID 52 - "sudo cp -r hieradata /etc/puppetlabs/data/", - "sudo chown -R root:52 /etc/puppetlabs/data/hieradata", - "sudo chmod -R 650 /etc/puppetlabs/data/hieradata", - "rm -rf hieradata", - ] + dynamic "source" { + for_each = var.eyaml_key != "" ? [var.eyaml_key] : [] + content { + content = var.eyaml_key + filename = "${local.provision_folder}/puppet/eyaml/private_key.pkcs7.pem" + } } } -resource "terraform_data" "deploy_hieradata" { - for_each = local.connection_parameters != null ? var.puppetservers : { } +resource "terraform_data" "deploy_puppetserver_files" { + for_each = length(var.bastions) > 0 ? var.puppetservers : { } connection { type = "ssh" + bastion_host = var.bastions[keys(var.bastions)[0]].public_ip + bastion_user = var.sudoer_username + bastion_private_key = var.tf_ssh_key.private + user = var.sudoer_username host = each.value - bastion_host = local.connection_parameters["bastion_host"] - bastion_user = local.connection_parameters["bastion_user"] - bastion_private_key = local.connection_parameters["bastion_private_key"] - user = local.connection_parameters["user"] - private_key = local.connection_parameters["private_key"] + private_key = var.tf_ssh_key.private } - triggers_replace = local.triggers_hieradata + triggers_replace = { + archive = data.archive_file.puppetserver_files.output_sha256 + } provisioner "file" { source = "${path.module}/files/${local.provision_folder}.zip" @@ -98,41 +83,14 @@ resource "terraform_data" "deploy_hieradata" { provisioner "remote-exec" { inline = [ - "sudo mkdir -p /etc/puppetlabs/data /etc/puppetlabs/facts", - # puppet user and group have been assigned the reserved UID/GID 52 - "sudo install -o root -g 52 -m 640 terraform_data.yaml user_data.yaml /etc/puppetlabs/data/", - "sudo install -o root -g 52 -m 640 terraform_facts.yaml /etc/puppetlabs/facts/", - "rm -f terraform_data.yaml user_data.yaml terraform_facts.yaml", + # unzip is not necessarily installed when connecting, but python is. + "/usr/libexec/platform-python -c 'import zipfile; zipfile.ZipFile(\"${local.provision_folder}.zip\").extractall()'", + "sudo chmod g-w,o-rwx $(find ${local.provision_folder}/ -type f)", + "sudo chown -R root:52 ${local.provision_folder}", + "sudo mkdir -p -m 755 /etc/puppetlabs/", + "sudo rsync -avh --no-t ${local.provision_folder}/ /etc/puppetlabs/", + "sudo rm -rf ${local.provision_folder}/ ${local.provision_folder}.zip", + "[ -f /usr/local/bin/consul ] && [ -f /usr/bin/jq ] && consul event -token=$(sudo jq -r .acl.tokens.agent /etc/consul/config.json) -name=puppet $(date +%s) || true", ] } -} - -resource "terraform_data" "update_consul" { - for_each = local.connection_parameters != null ? var.puppetservers : { } - - connection { - type = "ssh" - host = each.value - bastion_host = local.connection_parameters["bastion_host"] - bastion_user = local.connection_parameters["bastion_user"] - bastion_private_key = local.connection_parameters["bastion_private_key"] - user = local.connection_parameters["user"] - private_key = local.connection_parameters["private_key"] - } - - triggers_replace = merge( - local.triggers_hieradata, - local.triggers_hieradata_folder, - ) - - depends_on = [ - terraform_data.deploy_hieradata, - terraform_data.deploy_hieradata_folder, - ] - - provisioner "remote-exec" { - inline = [ - "[ -f /usr/local/bin/consul ] && [ -f /usr/bin/jq ] && consul event -token=$(sudo jq -r .acl.tokens.agent /etc/consul/config.json) -name=puppet $(date +%s) || true" - ] - } -} +} \ No newline at end of file From 14835a59e208ac2a021e180d34ba692fc59790ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Tue, 7 May 2024 17:18:15 -0400 Subject: [PATCH 03/12] Rename hieradata_folder_path to hieradata_dir --- aws/infrastructure.tf | 2 +- azure/infrastructure.tf | 2 +- common/provision/main.tf | 13 ++++++------- common/variables.tf | 2 +- docs/README.md | 2 +- gcp/infrastructure.tf | 2 +- openstack/infrastructure.tf | 2 +- 7 files changed, 12 insertions(+), 13 deletions(-) diff --git a/aws/infrastructure.tf b/aws/infrastructure.tf index 881305ea..c3ee6386 100644 --- a/aws/infrastructure.tf +++ b/aws/infrastructure.tf @@ -42,7 +42,7 @@ module "provision" { terraform_data = module.configuration.terraform_data terraform_facts = module.configuration.terraform_facts hieradata = var.hieradata - hieradata_folder = var.hieradata_folder_path + hieradata_dir = var.hieradata_dir sudoer_username = var.sudoer_username eyaml_key = var.eyaml_key depends_on = [aws_instance.instances, aws_eip.public_ip] diff --git a/azure/infrastructure.tf b/azure/infrastructure.tf index 735b5675..4a896dcb 100644 --- a/azure/infrastructure.tf +++ b/azure/infrastructure.tf @@ -42,7 +42,7 @@ module "provision" { terraform_data = module.configuration.terraform_data terraform_facts = module.configuration.terraform_facts hieradata = var.hieradata - hieradata_folder = var.hieradata_folder_path + hieradata_dir = var.hieradata_dir sudoer_username = var.sudoer_username eyaml_key = var.eyaml_key depends_on = [ azurerm_linux_virtual_machine.instances ] diff --git a/common/provision/main.tf b/common/provision/main.tf index 11934009..4f02a562 100644 --- a/common/provision/main.tf +++ b/common/provision/main.tf @@ -3,7 +3,7 @@ variable "puppetservers" { } variable "terraform_data" { } variable "terraform_facts" { } variable "hieradata" { } -variable "hieradata_folder" { } +variable "hieradata_dir" { } variable "sudoer_username" { } variable "tf_ssh_key" { } variable "eyaml_key" { } @@ -26,26 +26,25 @@ data "archive_file" "puppetserver_files" { filename = "${local.provision_folder}/facts/terraform_facts.yaml" } - source { content = var.hieradata filename = "${local.provision_folder}/data/user_data.yaml" } dynamic "source" { - for_each = var.hieradata_folder != "" ? fileset("${var.hieradata_folder}", "*.yaml") : [] + for_each = var.hieradata_dir != "" ? fileset("${var.hieradata_dir}", "*.yaml") : [] iterator = filename content { - content = file("${var.hieradata_folder}/${filename.value}") + content = file("${var.hieradata_dir}/${filename.value}") filename = "${local.provision_folder}/data/${filename.value}" } } dynamic "source" { - for_each = var.hieradata_folder != "" ? fileset("${var.hieradata_folder}", "{prefix,hostname}/**/*.yaml") : [] + for_each = var.hieradata_dir != "" ? fileset("${var.hieradata_dir}", "{prefix,hostname}/**/*.yaml") : [] iterator = filename content { - content = file("${var.hieradata_folder}/${filename.value}") + content = file("${var.hieradata_dir}/${filename.value}") filename = "${local.provision_folder}/data/${filename.value}" } } @@ -93,4 +92,4 @@ resource "terraform_data" "deploy_puppetserver_files" { "[ -f /usr/local/bin/consul ] && [ -f /usr/bin/jq ] && consul event -token=$(sudo jq -r .acl.tokens.agent /etc/consul/config.json) -name=puppet $(date +%s) || true", ] } -} \ No newline at end of file +} diff --git a/common/variables.tf b/common/variables.tf index 8e481e6a..38293057 100644 --- a/common/variables.tf +++ b/common/variables.tf @@ -94,7 +94,7 @@ variable "hieradata" { } } -variable "hieradata_folder_path" { +variable "hieradata_dir" { type = string default = "" description = "Path to hieradata folder containing YAML files to be included in the puppet environment" diff --git a/docs/README.md b/docs/README.md index 78111fb4..3d112e95 100644 --- a/docs/README.md +++ b/docs/README.md @@ -956,7 +956,7 @@ the provided content will replace entirely the Magic Castle environment's **Post build modification effect**: None. To modify the Puppetfile after the cluster is initialized, log on the Puppet server and modify `/etc/puppetlabs/code/environments/production/Puppetfile`. -### 4.20 hieradata_folder_path (optional) +### 4.20 hieradata_dir (optional) **Default Value:** Empty string diff --git a/gcp/infrastructure.tf b/gcp/infrastructure.tf index 400081fa..6845a745 100644 --- a/gcp/infrastructure.tf +++ b/gcp/infrastructure.tf @@ -42,7 +42,7 @@ module "provision" { terraform_data = module.configuration.terraform_data terraform_facts = module.configuration.terraform_facts hieradata = var.hieradata - hieradata_folder = var.hieradata_folder_path + hieradata_dir = var.hieradata_dir sudoer_username = var.sudoer_username eyaml_key = var.eyaml_key depends_on = [ google_compute_instance.instances ] diff --git a/openstack/infrastructure.tf b/openstack/infrastructure.tf index 655b7a22..b0092042 100644 --- a/openstack/infrastructure.tf +++ b/openstack/infrastructure.tf @@ -37,7 +37,7 @@ module "provision" { terraform_data = module.configuration.terraform_data terraform_facts = module.configuration.terraform_facts hieradata = var.hieradata - hieradata_folder = var.hieradata_folder_path + hieradata_dir = var.hieradata_dir sudoer_username = var.sudoer_username eyaml_key = var.eyaml_key depends_on = [ From d1c056f7bc15c4a231030b1e15c819054c65a4fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Wed, 8 May 2024 09:40:44 -0400 Subject: [PATCH 04/12] Update docs and fix path --- common/provision/main.tf | 2 +- docs/README.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/common/provision/main.tf b/common/provision/main.tf index 4f02a562..06c204cb 100644 --- a/common/provision/main.tf +++ b/common/provision/main.tf @@ -41,7 +41,7 @@ data "archive_file" "puppetserver_files" { } dynamic "source" { - for_each = var.hieradata_dir != "" ? fileset("${var.hieradata_dir}", "{prefix,hostname}/**/*.yaml") : [] + for_each = var.hieradata_dir != "" ? fileset("${var.hieradata_dir}", "{prefixes,hostnames}/**/*.yaml") : [] iterator = filename content { content = file("${var.hieradata_dir}/${filename.value}") diff --git a/docs/README.md b/docs/README.md index 3d112e95..38437f5a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -956,7 +956,7 @@ the provided content will replace entirely the Magic Castle environment's **Post build modification effect**: None. To modify the Puppetfile after the cluster is initialized, log on the Puppet server and modify `/etc/puppetlabs/code/environments/production/Puppetfile`. -### 4.20 hieradata_dir (optional) +### 4.21 hieradata_dir (optional) **Default Value:** Empty string @@ -965,8 +965,8 @@ This parameter specifies the path to a folder containing multiple YAML files use **Folder Structure:** - All Instances: `*.yaml` -- Per Instance Prefix: `/*.yaml` and `.yaml` -- Per Instance Hostname: `/*.yaml` and `.yaml` +- Per Instance Prefix: `prefixes//*.yaml` and `prefixes/.yaml` +- Per Instance Hostname: `hostnames//*.yaml` and `hostnames/.yaml` For more information on hieradata definition, refer to section [4.13 hieradata (optional)](#413-hieradata-optional). From 5b8e4c98f9708104e4d1dd094f60d16995f065fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Wed, 8 May 2024 09:45:45 -0400 Subject: [PATCH 05/12] Exclude data from first rsync and remove extra yaml file from target --- common/provision/main.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/provision/main.tf b/common/provision/main.tf index 06c204cb..9b23fe3d 100644 --- a/common/provision/main.tf +++ b/common/provision/main.tf @@ -87,7 +87,8 @@ resource "terraform_data" "deploy_puppetserver_files" { "sudo chmod g-w,o-rwx $(find ${local.provision_folder}/ -type f)", "sudo chown -R root:52 ${local.provision_folder}", "sudo mkdir -p -m 755 /etc/puppetlabs/", - "sudo rsync -avh --no-t ${local.provision_folder}/ /etc/puppetlabs/", + "sudo rsync -avh --no-t --exclude 'data' ${local.provision_folder}/ /etc/puppetlabs/", + "sudo rsync -avh --no-t --del ${local.provision_folder}/data/ /etc/puppetlabs/data/", "sudo rm -rf ${local.provision_folder}/ ${local.provision_folder}.zip", "[ -f /usr/local/bin/consul ] && [ -f /usr/bin/jq ] && consul event -token=$(sudo jq -r .acl.tokens.agent /etc/consul/config.json) -name=puppet $(date +%s) || true", ] From 0ae3f3692dfc02c325cd2c74ecd5030fa0711b99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Wed, 8 May 2024 15:10:15 -0400 Subject: [PATCH 06/12] Update documentation --- docs/README.md | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/README.md b/docs/README.md index 38437f5a..6a24cba9 100644 --- a/docs/README.md +++ b/docs/README.md @@ -958,17 +958,21 @@ on the Puppet server and modify `/etc/puppetlabs/code/environments/production/Pu ### 4.21 hieradata_dir (optional) -**Default Value:** Empty string +**default_value:** Empty string -This parameter specifies the path to a folder containing multiple YAML files used for configuring hieradata based on instance prefixes or hostnames. +Defines the path to a directory containing a hierarchy of YAML data files. -**Folder Structure:** +**Hierarchy structure:** -- All Instances: `*.yaml` -- Per Instance Prefix: `prefixes//*.yaml` and `prefixes/.yaml` -- Per Instance Hostname: `hostnames//*.yaml` and `hostnames/.yaml` +- per node hostname: + - `/hostnames//*.yaml` + - `/hostnames/.yaml` +- per node prefix: + - `/prefixes//*.yaml` + - `/prefixes/.yaml` +- all nodes: `/*.yaml` -For more information on hieradata definition, refer to section [4.13 hieradata (optional)](#413-hieradata-optional). +For more information on hieradata, refer to section [4.13 hieradata (optional)](#413-hieradata-optional). ## 5. Cloud Specific Configuration From 792cbecf730379d7bc3b97b9dc4546249d355ef1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Wed, 8 May 2024 15:34:25 -0400 Subject: [PATCH 07/12] Update docs --- docs/README.md | 61 +++++++++++++++++++++++++++----------------------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/docs/README.md b/docs/README.md index 6a24cba9..e6c958fd 100644 --- a/docs/README.md +++ b/docs/README.md @@ -783,7 +783,30 @@ The file created from this string can be found on `puppet` as **Post build modification effect**: trigger scp of hieradata files at next `terraform apply`. Each instance's Puppet agent will be reloaded following the copy of the hieradata files. -### 4.14 eyaml_private (optional) +### 4.14 hieradata_dir (optional) + +**default_value:** Empty string + +Defines the path to a directory containing a hierarchy of YAML data files. +The hierarchy is copied on the Puppet server in `/etc/puppetlabs/data/user_data`. + +**Hierarchy structure:** + +- per node hostname: + - `/hostnames//*.yaml` + - `/hostnames/.yaml` +- per node prefix: + - `/prefixes//*.yaml` + - `/prefixes/.yaml` +- all nodes: `/*.yaml` + +For more information on hieradata, refer to section [4.13 hieradata (optional)](#413-hieradata-optional). + +**Post build modification effect**: trigger scp of hieradata files at next `terraform apply`. +Each instance's Puppet agent will be reloaded following the copy of the hieradata files. + + +### 4.15 eyaml_private (optional) **default value**: empty string @@ -792,7 +815,7 @@ This key will be copied on the Puppet server. **Post build modification effect**: trigger scp of private key file at next `terraform apply`. -#### 4.14.1 Generate eyaml encryption keys +#### 4.15.1 Generate eyaml encryption keys If you plan to track the cluster configuration files in git (i.e:`main.tf`, `user_data.yaml`), it would be a good idea to encrypt the sensitive property values. @@ -812,7 +835,7 @@ or with `eyaml`: eyaml createkeys --pkcs7-public-key=public_key.pkcs7.pem --pkcs7-private-key=private_key.pkcs7.pem ``` -#### 4.14.2 Encrypting sensitive properties +#### 4.15.2 Encrypting sensitive properties To encrypt a sensitive property with openssl: ```sh @@ -824,7 +847,7 @@ To encrypt a sensitive property with eyaml: eyaml encrypt -s 'your-secret' --pkcs7-public-key=public_key.pkcs7.pem -o string ``` -#### 4.14.3 Terraform cloud +#### 4.15.3 Terraform cloud To provide the value of this variable via Terraform Cloud, encode the private key content with base64: @@ -852,7 +875,7 @@ module "openstack" { } ``` -### 4.15 firewall_rules (optional) +### 4.16 firewall_rules (optional) **default value**: ```hcl @@ -886,7 +909,7 @@ about this requirement, refer to Magic Castle's **Post build modification effect**: modify the cloud provider firewall rules at next `terraform apply`. -### 4.16 generate_ssh_key (optional) +### 4.17 generate_ssh_key (optional) **default_value**: `false` @@ -907,7 +930,7 @@ next terraform apply. The Terraform public SSH key will be removed from the sudoer account `authorized_keys` file at next Puppet agent run. -### 4.17 software_stack (optional) +### 4.18 software_stack (optional) **default_value**: `"alliance"` @@ -920,7 +943,7 @@ Possible values are: **Post build modification effect**: trigger scp of hieradata files at next `terraform apply`. -### 4.18 pool (optional) +### 4.19 pool (optional) **default_value**: `[]` @@ -932,7 +955,7 @@ managed by the workload scheduler through Terraform API. For more information, r will be instantiated, others will stay uninstantiated or will be destroyed if previously instantiated. -### 4.19 skip_upgrade (optional) +### 4.20 skip_upgrade (optional) **default_value** = `false` @@ -943,7 +966,7 @@ all packages are upgraded. after the modification will take into consideration the new value of the parameter to determine whether they should upgrade the base image packages or not. -### 4.20 puppetfile (optional) +### 4.21 puppetfile (optional) **default_value** = `""` @@ -956,24 +979,6 @@ the provided content will replace entirely the Magic Castle environment's **Post build modification effect**: None. To modify the Puppetfile after the cluster is initialized, log on the Puppet server and modify `/etc/puppetlabs/code/environments/production/Puppetfile`. -### 4.21 hieradata_dir (optional) - -**default_value:** Empty string - -Defines the path to a directory containing a hierarchy of YAML data files. - -**Hierarchy structure:** - -- per node hostname: - - `/hostnames//*.yaml` - - `/hostnames/.yaml` -- per node prefix: - - `/prefixes//*.yaml` - - `/prefixes/.yaml` -- all nodes: `/*.yaml` - -For more information on hieradata, refer to section [4.13 hieradata (optional)](#413-hieradata-optional). - ## 5. Cloud Specific Configuration ### 5.1 Amazon Web Services From c54ce52ffff096e65e733c891d764d0e799264bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Wed, 8 May 2024 15:34:52 -0400 Subject: [PATCH 08/12] Create hierarchy under user_data/ --- common/configuration/puppet.yaml | 4 +--- common/provision/main.tf | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/common/configuration/puppet.yaml b/common/configuration/puppet.yaml index 0afe0333..b17d67f5 100644 --- a/common/configuration/puppet.yaml +++ b/common/configuration/puppet.yaml @@ -88,9 +88,7 @@ runcmd: %{ endif ~} - mkdir -p /etc/puppetlabs/data /etc/puppetlabs/facts - chgrp -R puppet /etc/puppetlabs/data /etc/puppetlabs/facts - - ln -sf /etc/puppetlabs/data/terraform_data.yaml /etc/puppetlabs/code/environments/production/data/ - - ln -sf /etc/puppetlabs/data/user_data.yaml /etc/puppetlabs/code/environments/production/data/ - - ln -sf /etc/puppetlabs/data/hieradata /etc/puppetlabs/code/environments/production/data/ + - ln -sf /etc/puppetlabs/data/{user_data,user_data.yaml,terraform_data.yaml} /etc/puppetlabs/code/environments/production/data/ - ln -sf /etc/puppetlabs/facts/terraform_facts.yaml /etc/puppetlabs/code/environments/production/site/profile/facts.d # We use r10k solely to install the modules of the main branch environment. - "(cd /etc/puppetlabs/code/environments/production; /opt/puppetlabs/puppet/bin/r10k puppetfile install)" diff --git a/common/provision/main.tf b/common/provision/main.tf index 9b23fe3d..890d5204 100644 --- a/common/provision/main.tf +++ b/common/provision/main.tf @@ -36,7 +36,7 @@ data "archive_file" "puppetserver_files" { iterator = filename content { content = file("${var.hieradata_dir}/${filename.value}") - filename = "${local.provision_folder}/data/${filename.value}" + filename = "${local.provision_folder}/data/user_data/${filename.value}" } } @@ -45,7 +45,7 @@ data "archive_file" "puppetserver_files" { iterator = filename content { content = file("${var.hieradata_dir}/${filename.value}") - filename = "${local.provision_folder}/data/${filename.value}" + filename = "${local.provision_folder}/data/user_data/${filename.value}" } } From 2a3536589b934b9b287f66b3ae230cacf3814b5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Wed, 8 May 2024 16:18:21 -0400 Subject: [PATCH 09/12] Update docs --- docs/README.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docs/README.md b/docs/README.md index e6c958fd..6fd9c1c4 100644 --- a/docs/README.md +++ b/docs/README.md @@ -772,11 +772,7 @@ Refer to the following Puppet modules' documentation to know more about the key- - [puppet-jupyterhub](https://github.com/ComputeCanada/puppet-jupyterhub/blob/main/README.md#hieradata-configuration) - [puppet-prometheus](https://forge.puppet.com/modules/puppet/prometheus/) - -The file created from this string can be found on `puppet` as -``` -/etc/puppetlabs/data/user_data.yaml -``` +The file created from this string can be found on the Puppet server as `/etc/puppetlabs/data/user_data.yaml` **Requirement**: The string needs to respect the [YAML syntax](https://en.wikipedia.org/wiki/YAML#Syntax). From 5bc602bd5a10725c075318cad8d3ed7f4c899264 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Thu, 9 May 2024 09:55:10 -0400 Subject: [PATCH 10/12] Add validation to hieradata_dir --- common/variables.tf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/common/variables.tf b/common/variables.tf index 38293057..a002e390 100644 --- a/common/variables.tf +++ b/common/variables.tf @@ -98,6 +98,10 @@ variable "hieradata_dir" { type = string default = "" description = "Path to hieradata folder containing YAML files to be included in the puppet environment" + validation { + condition = alltrue([for filename in fileset("${var.hieradata_dir}", "**/*.yaml"): can(yamldecode(file("${var.hieradata_dir}/${filename}")))]) + error_message = "At least one YAML file in ${var.hieradata_dir} is not a valid." + } } variable "sudoer_username" { From 6b90c769df9d0dcaca6e098f523eaae5849fb633 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Thu, 9 May 2024 10:22:15 -0400 Subject: [PATCH 11/12] Reduce to a single dynamic source for hieradata_dir files --- common/provision/main.tf | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/common/provision/main.tf b/common/provision/main.tf index 890d5204..d81397f8 100644 --- a/common/provision/main.tf +++ b/common/provision/main.tf @@ -32,16 +32,7 @@ data "archive_file" "puppetserver_files" { } dynamic "source" { - for_each = var.hieradata_dir != "" ? fileset("${var.hieradata_dir}", "*.yaml") : [] - iterator = filename - content { - content = file("${var.hieradata_dir}/${filename.value}") - filename = "${local.provision_folder}/data/user_data/${filename.value}" - } - } - - dynamic "source" { - for_each = var.hieradata_dir != "" ? fileset("${var.hieradata_dir}", "{prefixes,hostnames}/**/*.yaml") : [] + for_each = var.hieradata_dir != "" ? fileset("${var.hieradata_dir}", "**/*.yaml") : [] iterator = filename content { content = file("${var.hieradata_dir}/${filename.value}") From b7735cae2557585f54d846c57a0829f1a7f10482 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Fortin?= Date: Thu, 9 May 2024 15:57:06 -0400 Subject: [PATCH 12/12] Use standard facter config directory for prefix fact --- common/configuration/puppet.yaml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/common/configuration/puppet.yaml b/common/configuration/puppet.yaml index b17d67f5..8ae42330 100644 --- a/common/configuration/puppet.yaml +++ b/common/configuration/puppet.yaml @@ -139,17 +139,14 @@ write_files: %{ if cloud_provider != "gcp" } "GCE", %{ endif } ], } - global : { - external-dir : ["/etc/puppetlabs/facts/external"], - } path: /etc/puppetlabs/facter/facter.conf permissions: "0644" - - path: /etc/puppetlabs/facts/external/prefix.yaml + - path: /etc/puppetlabs/facter/facts.d/prefix.yaml content: | --- prefix : "${node_prefix}" - permissions: "0640" + permissions: "0644" %{ if contains(tags, "puppet") ~} - content: | ---