Merge pull request #291 from etiennedub/hieradata_node

Hieradata per instances group

aws/infrastructure.tf

@@ -42,6 +42,7 @@ module "provision" {
   terraform_data  = module.configuration.terraform_data
   terraform_facts = module.configuration.terraform_facts
   hieradata       = var.hieradata
+  hieradata_dir   = var.hieradata_dir
   sudoer_username = var.sudoer_username
   eyaml_key       = var.eyaml_key
   depends_on      = [aws_instance.instances, aws_eip.public_ip]

azure/infrastructure.tf

@@ -42,6 +42,7 @@ module "provision" {
   terraform_data  = module.configuration.terraform_data
   terraform_facts = module.configuration.terraform_facts
   hieradata       = var.hieradata
+  hieradata_dir   = var.hieradata_dir
   sudoer_username = var.sudoer_username
   eyaml_key       = var.eyaml_key
   depends_on      = [ azurerm_linux_virtual_machine.instances ]

common/configuration/main.tf

@@ -86,7 +86,7 @@ locals {
   })
 
   terraform_facts = yamlencode({
-    software_stack = var.software_stack
+    software_stack = var.software_stack,
     cloud          = {
       provider = var.cloud_provider
       region = var.cloud_region
@@ -100,6 +100,7 @@ locals {
         cloud_provider        = var.cloud_provider
         tags                  = values.tags
         node_name             = key,
+        node_prefix           = values.prefix,
         domain_name           = var.domain_name
         puppetenv_git         = var.config_git_url,
         puppetenv_rev         = var.config_version,

common/configuration/puppet.yaml

@@ -88,8 +88,7 @@ runcmd:
 %{ endif ~}
   - mkdir -p /etc/puppetlabs/data /etc/puppetlabs/facts
   - chgrp -R puppet /etc/puppetlabs/data /etc/puppetlabs/facts
-  - ln -sf /etc/puppetlabs/data/terraform_data.yaml /etc/puppetlabs/code/environments/production/data/
-  - ln -sf /etc/puppetlabs/data/user_data.yaml /etc/puppetlabs/code/environments/production/data/
+  - ln -sf /etc/puppetlabs/data/{user_data,user_data.yaml,terraform_data.yaml} /etc/puppetlabs/code/environments/production/data/
   - ln -sf /etc/puppetlabs/facts/terraform_facts.yaml /etc/puppetlabs/code/environments/production/site/profile/facts.d
 # We use r10k solely to install the modules of the main branch environment.
   - "(cd /etc/puppetlabs/code/environments/production; /opt/puppetlabs/puppet/bin/r10k puppetfile install)"
@@ -140,8 +139,14 @@ write_files:
           %{ if cloud_provider != "gcp" } "GCE", %{ endif }
         ],
       }
+
     path: /etc/puppetlabs/facter/facter.conf
     permissions: "0644"
+  - path: /etc/puppetlabs/facter/facts.d/prefix.yaml
+    content: |
+      ---
+      prefix : "${node_prefix}"
+    permissions: "0644"
 %{ if contains(tags, "puppet") ~}
   - content: |
       ---

common/provision/main.tf

@@ -3,6 +3,7 @@ variable "puppetservers" { }
 variable "terraform_data" { }
 variable "terraform_facts" { }
 variable "hieradata" { }
+variable "hieradata_dir" { }
 variable "sudoer_username" { }
 variable "tf_ssh_key" { }
 variable "eyaml_key" { }
@@ -30,6 +31,15 @@ data "archive_file" "puppetserver_files" {
     filename = "${local.provision_folder}/data/user_data.yaml"
   }
 
+  dynamic "source" {
+    for_each = var.hieradata_dir != "" ? fileset("${var.hieradata_dir}", "**/*.yaml") : []
+    iterator = filename
+    content {
+      content  = file("${var.hieradata_dir}/${filename.value}")
+      filename = "${local.provision_folder}/data/user_data/${filename.value}"
+    }
+  }
+
   dynamic "source" {
     for_each =  var.eyaml_key != "" ? [var.eyaml_key] : []
     content {
@@ -68,7 +78,8 @@ resource "terraform_data" "deploy_puppetserver_files" {
       "sudo chmod g-w,o-rwx $(find ${local.provision_folder}/ -type f)",
       "sudo chown -R root:52 ${local.provision_folder}",
       "sudo mkdir -p -m 755 /etc/puppetlabs/",
-      "sudo rsync -avh --no-t ${local.provision_folder}/ /etc/puppetlabs/",
+      "sudo rsync -avh --no-t --exclude 'data' ${local.provision_folder}/ /etc/puppetlabs/",
+      "sudo rsync -avh --no-t --del ${local.provision_folder}/data/ /etc/puppetlabs/data/",
       "sudo rm -rf ${local.provision_folder}/ ${local.provision_folder}.zip",
       "[ -f /usr/local/bin/consul ] && [ -f /usr/bin/jq ] && consul event -token=$(sudo jq -r .acl.tokens.agent /etc/consul/config.json) -name=puppet $(date +%s) || true",
     ]

common/variables.tf

@@ -94,6 +94,16 @@ variable "hieradata" {
   }
 }
 
+variable "hieradata_dir" {
+  type        = string
+  default     = ""
+  description = "Path to hieradata folder containing YAML files to be included in the puppet environment"
+  validation {
+    condition     = alltrue([for filename in fileset("${var.hieradata_dir}", "**/*.yaml"): can(yamldecode(file("${var.hieradata_dir}/${filename}")))])
+    error_message = "At least one YAML file in ${var.hieradata_dir} is not a valid."
+  }
+}
+
 variable "sudoer_username" {
   type        = string
   default     = "centos"

docs/README.md

@@ -772,18 +772,37 @@ Refer to the following Puppet modules' documentation to know more about the key-
 - [puppet-jupyterhub](https://github.com/ComputeCanada/puppet-jupyterhub/blob/main/README.md#hieradata-configuration)
 - [puppet-prometheus](https://forge.puppet.com/modules/puppet/prometheus/)
 
-
-The file created from this string can be found on `puppet` as
-```
-/etc/puppetlabs/data/user_data.yaml
-```
+The file created from this string can be found on the Puppet server as `/etc/puppetlabs/data/user_data.yaml`
 
 **Requirement**: The string needs to respect the [YAML syntax](https://en.wikipedia.org/wiki/YAML#Syntax).
 
 **Post build modification effect**: trigger scp of hieradata files at next `terraform apply`.
 Each instance's Puppet agent will be reloaded following the copy of the hieradata files.
 
-### 4.14 eyaml_private (optional)
+### 4.14 hieradata_dir (optional)
+
+**default_value:** Empty string
+
+Defines the path to a directory containing a hierarchy of YAML data files.
+The hierarchy is copied on the Puppet server in `/etc/puppetlabs/data/user_data`.
+
+**Hierarchy structure:**
+
+- per node hostname:
+  - `<dir>/hostnames/<hostname>/*.yaml`
+  - `<dir>/hostnames/<hostname>.yaml`
+- per node prefix:
+  - `<dir>/prefixes/<prefix>/*.yaml`
+  - `<dir>/prefixes/<prefix>.yaml`
+- all nodes: `<dir>/*.yaml`
+
+For more information on hieradata, refer to section [4.13 hieradata (optional)](#413-hieradata-optional).
+
+**Post build modification effect**: trigger scp of hieradata files at next `terraform apply`.
+Each instance's Puppet agent will be reloaded following the copy of the hieradata files.
+
+
+### 4.15 eyaml_private (optional)
 
 **default value**: empty string
 
@@ -792,7 +811,7 @@ This key will be copied on the Puppet server.
 
 **Post build modification effect**: trigger scp of private key file at next `terraform apply`.
 
-#### 4.14.1 Generate eyaml encryption keys
+#### 4.15.1 Generate eyaml encryption keys
 
 If you plan to track the cluster configuration files in git (i.e:`main.tf`, `user_data.yaml`),
 it would be a good idea to encrypt the sensitive property values.
@@ -812,7 +831,7 @@ or with `eyaml`:
 eyaml createkeys --pkcs7-public-key=public_key.pkcs7.pem --pkcs7-private-key=private_key.pkcs7.pem
 ```
 
-#### 4.14.2 Encrypting sensitive properties
+#### 4.15.2 Encrypting sensitive properties
 
 To encrypt a sensitive property with openssl:
 ```sh
@@ -824,7 +843,7 @@ To encrypt a sensitive property with eyaml:
 eyaml encrypt -s 'your-secret' --pkcs7-public-key=public_key.pkcs7.pem -o string
 ```
 
-#### 4.14.3 Terraform cloud
+#### 4.15.3 Terraform cloud
 
 To provide the value of this variable via Terraform Cloud, encode the private key content with base64:
 
@@ -852,7 +871,7 @@ module "openstack" {
 }
 ```
 
-### 4.15 firewall_rules (optional)
+### 4.16 firewall_rules (optional)
 
 **default value**:
 ```hcl
@@ -886,7 +905,7 @@ about this requirement, refer to Magic Castle's
 
 **Post build modification effect**: modify the cloud provider firewall rules at next `terraform apply`.
 
-### 4.16 generate_ssh_key (optional)
+### 4.17 generate_ssh_key (optional)
 
 **default_value**: `false`
 
@@ -907,7 +926,7 @@ next terraform apply. The Terraform public SSH key will be removed
 from the sudoer account `authorized_keys` file at next
 Puppet agent run.
 
-### 4.17 software_stack (optional)
+### 4.18 software_stack (optional)
 
 **default_value**: `"alliance"`
 
@@ -920,7 +939,7 @@ Possible values are:
 
 **Post build modification effect**: trigger scp of hieradata files at next `terraform apply`.
 
-### 4.18 pool (optional)
+### 4.19 pool (optional)
 
 **default_value**: `[]`
 
@@ -932,7 +951,7 @@ managed by the workload scheduler through Terraform API. For more information, r
 will be instantiated, others will stay uninstantiated or will be destroyed
 if previously instantiated.
 
-### 4.19 skip_upgrade (optional)
+### 4.20 skip_upgrade (optional)
 
 **default_value** = `false`
 
@@ -943,7 +962,7 @@ all packages are upgraded.
 after the modification will take into consideration the new value of the parameter to determine
 whether they should upgrade the base image packages or not.
 
-### 4.20 puppetfile (optional)
+### 4.21 puppetfile (optional)
 
 **default_value** = `""`
 

gcp/infrastructure.tf

@@ -42,6 +42,7 @@ module "provision" {
   terraform_data  = module.configuration.terraform_data
   terraform_facts = module.configuration.terraform_facts
   hieradata       = var.hieradata
+  hieradata_dir   = var.hieradata_dir
   sudoer_username = var.sudoer_username
   eyaml_key       = var.eyaml_key
   depends_on      = [ google_compute_instance.instances ]

openstack/infrastructure.tf

@@ -37,6 +37,7 @@ module "provision" {
   terraform_data  = module.configuration.terraform_data
   terraform_facts = module.configuration.terraform_facts
   hieradata       = var.hieradata
+  hieradata_dir   = var.hieradata_dir
   sudoer_username = var.sudoer_username
   eyaml_key       = var.eyaml_key
   depends_on      = [