Updates needed to install PKIX files needed for deployment and in CI

.gitlab-ci.yml

@@ -42,6 +42,7 @@ variables:
   DOCKER_BUILDKIT: 1
   # for Python/PIP
   PIP_CERT: /etc/pki/tls/certs/ca-bundle.crt
+  PIP_DEFAULT_TIMEOUT: 300
   # for Ruby/bolt
   SSL_CERT_FILE: /etc/pki/tls/certs/ca-bundle.crt
   # Project-specific environment
@@ -346,6 +347,10 @@ deploy:
       anms::docker_image_tag: "${DOCKER_IMAGE_TAG}"
       anms::docker_registry_user: "${DOCKER_REGISTRY_USERNAME}"
       anms::docker_registry_pass: "${DOCKER_REGISTRY_PASSWORD}"
+      anms::tls_server_key: 'puppet:///modules/apl_test/anms/tls/private/ammos-server-key.pem'
+      anms::tls_server_cert: 'puppet:///modules/apl_test/anms/tls/certs/ammos-server-cert.pem'
+      anms::tls_server_ca: 'puppet:///modules/apl_test/anms/tls/certs/ammos-ca-bundle.crt'
+      anms::hostenv::use_fips: false
       selinux::mode: permissive
       selinux::type: targeted
       EOF

anms-core/pyproject.toml

@@ -26,7 +26,7 @@ dependencies = [
   "asyncio ~=3.4.3",
   "asyncpg ~=0.27.0",
   "authlib ~=0.15.5",
-  "docker ~=6.0.1",
+  "docker ~=7.1.0",
   "emails ~=0.6",
   "email-validator ~=1.3",
   "fastapi ~=0.86.0",

puppet/.gitignore

@@ -9,6 +9,7 @@ bolt-debug.log
 # Files replaced by prep.sh
 modules/anms/files/docker-compose.yml
 modules/anms/files/agent-compose.yml
+modules/anms/files/create_volume.sh
 
 # Files replaced by apply_local.sh
 data/override.yaml

puppet/Puppetfile

@@ -4,10 +4,9 @@
 # The following directive installs modules to the managed moduledir.
 moduledir '.modules'
 
-mod 'simp/crypto_policy', '0.5.0'
+mod 'simp/crypto_policy', '0.6.0'
 mod 'simp/fips', '0.9.0'
 mod 'puppetlabs/java_ks', '5.0.0'
-mod 'puppetlabs/docker', '7.0.0'
 mod 'puppetlabs/concat', '7.4.0'
 mod 'puppetlabs/apt', '9.0.2'
 mod 'puppet/trusted_ca', '4.1.0'
@@ -17,8 +16,5 @@ mod 'puppet/firewalld', '5.0.0'
 mod 'puppetlabs/stdlib', '8.6.0'
 mod 'simp/simplib', '4.12.2'
 mod 'puppet/augeasproviders_grub', '5.1.0'
-mod 'puppetlabs/powershell', '6.0.0'
-mod 'puppetlabs/reboot', '5.0.0'
-mod 'simp/simpkv', '0.12.0'
+mod 'simp/simpkv', '0.13.0'
 mod 'puppet/augeasproviders_core', '4.1.0'
-mod 'puppetlabs/pwshlib', '1.1.1'

puppet/modules/anms/files/selinux/authnz.cil

@@ -1,6 +1,7 @@
 (block authnz
     (blockinherit container)
     (blockinherit restricted_net_container)
+    (allow process container_file_t ( chr_file ( map )))
     (allow process http_port_t ( tcp_socket ( name_bind )))
     (allow process http_port_t ( tcp_socket ( name_connect )))
 )

puppet/modules/anms/manifests/docker.pp

@@ -5,9 +5,9 @@
 class anms::docker() {
   case $facts['os']['family'] {
     'RedHat':  {
-      package { ['podman', 'runc']:
+      package { ['podman-docker', 'podman', 'runc']:
         ensure => 'absent',
-        before => Package['docker-ce'],
+        before => [Package['docker-ce'], Package['docker-ce-cli']],
       }
       package { 'yum-utils':
         ensure => 'installed',
@@ -30,17 +30,20 @@
     ensure => 'installed',
   }
   service { 'docker':
+    ensure  => 'running',
+    enable  => true,
     require => Package['docker-ce'],
   }
 
   file { '/usr/local/bin/docker-compose':
     ensure => 'absent',
   }
   file { '/etc/docker/daemon.json':
-    source => 'puppet:///modules/anms/docker-daemon.json',
-    owner  => 'root',
-    group  => 'root',
-    mode   => '0644',
-    notify => Service['docker'],
+    source  => 'puppet:///modules/anms/docker-daemon.json',
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    require => Package['docker-ce'],
+    notify  => Service['docker'],
   }
 }

puppet/modules/anms/manifests/docker_compose.pp

@@ -8,6 +8,7 @@
 define anms::docker_compose(
   Enum['present','absent'] $ensure,
   Array[String] $compose_files,
+  Boolean $pull_first = true,
   String $up_args = '',
 ) {
   require anms::docker
@@ -17,16 +18,23 @@
 
   case $ensure {
     'present': {
+      if $pull_first {
+        exec { "docker-compose-${title}-pull":
+          path    => $facts['path'],
+          command => "docker compose -p ${title} -f ${files_args} pull",
+          before  => Exec["docker-compose-${title}-up"],
+        }
+      }
       exec { "docker-compose-${title}-up":
-        path => $facts['path'],
-  command    => "docker compose -p ${title} -f ${files_args} up --detach --remove-orphans ${up_args}",
+        path    => $facts['path'],
+        command => "docker compose -p ${title} -f ${files_args} up --detach --remove-orphans ${up_args}",
       }
     }
     'absent': {
-      exec { "docker-compose-${title}-up":
+      exec { "docker-compose-${title}-rm":
         path    => $facts['path'],
         command => "docker compose -p ${title} -f ${files_args} rm --force --stop",
-  onlyif        => $is_running,
+        onlyif  => $is_running,
       }
     }
     default: {

puppet/modules/anms/manifests/hostenv.pp

@@ -1,12 +1,16 @@
 # Define host environment configuration for ANMS installation.
 #
-class anms::hostenv() {
+class anms::hostenv(
+  Boolean $use_fips = true,
+) {
   case $facts['os']['family'] {
     'RedHat':  {
       # This halts on reboot_notify()
       # instead run with:
       #  bolt apply --execute 'class {"fips": }'
-#      class { 'fips': }
+      if $use_fips {
+        class { 'fips': }
+      }
 
       file { '/var/cache/puppet':
         ensure => 'directory',
@@ -60,20 +64,22 @@
     'Debian': {
       case $facts['os']['distro']['codename'] {
         'focal':  {
-          # Based on guidance at: https://aplwiki.jhuapl.edu/confluence/pages/viewpage.action?spaceKey=LAPLKEY&title=Ubuntu+FIPS+Packages
-          apt::source { 'focal-fips':
-            location => 'https://apllinuxdepot.jhuapl.edu/linux/apl-software/focal-fips/',
-            release  => '',
-            repos    => '/',
-            key      => {
-              id     => '6F6B15509CF8E59E6E469F327F438280EF8D349F',
-              server => 'https://apllinuxdepot.jhuapl.edu/linux/apl-software/focal-fips/apl-software-repo.gpg',
-            },
-          }
+          if $use_fips {
+            # Based on guidance at: https://aplwiki.jhuapl.edu/confluence/pages/viewpage.action?spaceKey=LAPLKEY&title=Ubuntu+FIPS+Packages
+            apt::source { 'focal-fips':
+              location => 'https://apllinuxdepot.jhuapl.edu/linux/apl-software/focal-fips/',
+              release  => '',
+              repos    => '/',
+              key      => {
+                id     => '6F6B15509CF8E59E6E469F327F438280EF8D349F',
+                server => 'https://apllinuxdepot.jhuapl.edu/linux/apl-software/focal-fips/apl-software-repo.gpg',
+              },
+            }
 
-          package { 'ubuntu-fips':
-            ensure  => 'installed',
-            require => Apt::Source['focal-fips'],
+            package { 'ubuntu-fips':
+              ensure  => 'installed',
+              require => Apt::Source['focal-fips'],
+            }
           }
         }
         default: {

puppet/modules/anms/manifests/init.pp

@@ -12,9 +12,9 @@
   String $cam_admin_password = '',
   String $cam_agent_name = $facts['networking']['fqdn'],
   String $cam_agent_password = '',
-#  String $tls_server_key,
-#  String $tls_server_cert,
-#  String $tls_server_ca,
+  Optional[String] $tls_server_key = undef,
+  Optional[String] $tls_server_cert = undef,
+  Optional[String] $tls_server_ca = undef,
 ) {
   require Class['anms::hostenv']
 
@@ -24,29 +24,31 @@
   file { '/ammos/anms/.env':
     ensure  => 'file',
     content => epp('anms/env.epp'),
-  }
-  file { '/ammos/anms/docker-compose.yml':
-    ensure => 'file',
-    source => 'puppet:///modules/anms/docker-compose.yml',
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
   }
   file { ['/ammos/etc', '/ammos/etc/pki', '/ammos/etc/pki/tls',
           '/ammos/etc/pki/tls/private', '/ammos/etc/pki/tls/certs']:
     ensure => 'directory'
   }
   file { '/ammos/etc/pki/tls/private/ammos-server-key.pem':
     ensure => 'file',
+    source => $tls_server_key,
     owner  => 'root',
     group  => 'root',
     mode   => '0644',
   }
   file { '/ammos/etc/pki/tls/certs/ammos-server-cert.pem':
     ensure => 'file',
+    source => $tls_server_cert,
     owner  => 'root',
     group  => 'root',
     mode   => '0644',
   }
   file { '/ammos/etc/pki/tls/certs/ammos-ca-bundle.crt':
     ensure => 'file',
+    source => $tls_server_ca,
     owner  => 'root',
     group  => 'root',
     mode   => '0644',
@@ -101,32 +103,48 @@
   # Images pulled from remote registry
   if !empty($docker_image_prefix) and !empty($docker_registry_user) and !empty($docker_registry_pass) {
     exec { 'docker-login':
-      command => "docker login ${docker_image_prefix} --username \"${docker_registry_user}\" --password \"${docker_registry_pass}\"",
       path    => $facts['path'],
+      command => "docker login ${docker_image_prefix} --username \"${docker_registry_user}\" --password \"${docker_registry_pass}\"",
+      require => Service['docker'],
       before  => [
-        Exec['anms-pull'],
-        Exec['agents-pull'],
-      ],
-    }
-    exec { 'anms-pull':
-      command => 'docker-compose -f /ammos/anms/docker-compose.yml pull',
-      path    => $facts['path'],
-      require => [
-        File['/ammos/anms/docker-compose.yml'],
-        File['/ammos/anms/.env'],
-      ],
-      before  => Anms::Docker_compose['anms'],
-    }
-    exec { 'agents-pull':
-      command => 'docker-compose -f /ammos/anms/agent-compose.yml pull',
-      path    => $facts['path'],
-      require => [
-        File['/ammos/anms/agent-compose.yml'],
-        File['/ammos/anms/.env'],
+        Anms::Docker_compose['anms'],
+        Anms::Docker_compose['agents'],
       ],
-      before  => Anms::Docker_compose['agents'],
     }
   }
+
+  # volume for TLS-related PKIX files
+  file { '/ammos/anms/create_volume.sh':
+    ensure => 'file',
+    source => 'puppet:///modules/anms/create_volume.sh',
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0755',
+  }
+  exec { 'volume-ammos-tls':
+    path      => $facts['path'],
+    command   => '/ammos/anms/create_volume.sh',
+    unless    => 'docker volume inspect ammos-tls',
+    require   => [
+      Service['docker'],
+      File['/ammos/anms/create_volume.sh'],
+    ],
+    subscribe => [
+      File['/ammos/etc/pki/tls/private/ammos-server-key.pem'],
+      File['/ammos/etc/pki/tls/certs/ammos-server-cert.pem'],
+      File['/ammos/etc/pki/tls/certs/ammos-ca-bundle.crt'],
+    ],
+    before    => Anms::Docker_compose['anms'],
+    notify    => Anms::Docker_compose['anms'],
+  }
+
+  file { '/ammos/anms/docker-compose.yml':
+    ensure => 'file',
+    source => 'puppet:///modules/anms/docker-compose.yml',
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0644',
+  }
   anms::docker_compose { 'anms':
     ensure        => 'present',
     compose_files => ['/ammos/anms/docker-compose.yml'],
@@ -140,6 +158,9 @@
   file { '/ammos/anms/agent-compose.yml':
     ensure => 'file',
     source => 'puppet:///modules/anms/agent-compose.yml',
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0644',
   }
   anms::docker_compose { 'agents':
     ensure        => 'present',

puppet/prep.sh

@@ -28,8 +28,11 @@ set -e
 SELFDIR=$(dirname "${BASH_SOURCE[0]}")
 source ${SELFDIR}/getenv.sh
 
-rm -f ${SELFDIR}/modules/anms/files/*-compose.yml
-cp "${SELFDIR}/../docker-compose.yml" "${SELFDIR}/../agent-compose.yml" "${SELFDIR}/modules/anms/files/"
+rm -f "${SELFDIR}"/modules/anms/files/*-compose.yml
+cp "${SELFDIR}/../docker-compose.yml" \
+   "${SELFDIR}/../agent-compose.yml" \
+   "${SELFDIR}/../create_volume.sh" \
+   "${SELFDIR}/modules/anms/files/"
 
 export BOLT_PROJECT="${SELFDIR}"
 bolt module install --force