From 90ae38692efa9bed0211cb5580433d8230a82777 Mon Sep 17 00:00:00 2001 From: Brian Sipos Date: Thu, 30 May 2024 09:19:29 -0400 Subject: [PATCH 1/5] Updates needed to install PKIX files needed for deployment and in CI --- .gitlab-ci.yml | 3 + puppet/.gitignore | 1 + puppet/Puppetfile | 8 +- puppet/modules/anms/manifests/docker.pp | 17 ++-- .../modules/anms/manifests/docker_compose.pp | 16 +++- puppet/modules/anms/manifests/init.pp | 77 ++++++++++++------- puppet/prep.sh | 7 +- 7 files changed, 82 insertions(+), 47 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 74e304c..32f545c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -346,6 +346,9 @@ deploy: anms::docker_image_tag: "${DOCKER_IMAGE_TAG}" anms::docker_registry_user: "${DOCKER_REGISTRY_USERNAME}" anms::docker_registry_pass: "${DOCKER_REGISTRY_PASSWORD}" + anms::tls_server_key: 'puppet:///modules/apl_test/anms/tls/private/ammos-server-key.pem' + anms::tls_server_cert: 'puppet:///modules/apl_test/anms/tls/certs/ammos-server-cert.pem' + anms::tls_server_ca: 'puppet:///modules/apl_test/anms/tls/certs/ammos-ca-bundle.crt' selinux::mode: permissive selinux::type: targeted EOF diff --git a/puppet/.gitignore b/puppet/.gitignore index 7537cb3..5e9a20d 100644 --- a/puppet/.gitignore +++ b/puppet/.gitignore @@ -9,6 +9,7 @@ bolt-debug.log # Files replaced by prep.sh modules/anms/files/docker-compose.yml modules/anms/files/agent-compose.yml +modules/anms/files/create_volume.sh # Files replaced by apply_local.sh data/override.yaml diff --git a/puppet/Puppetfile b/puppet/Puppetfile index 3fa34c9..74ab725 100644 --- a/puppet/Puppetfile +++ b/puppet/Puppetfile @@ -4,10 +4,9 @@ # The following directive installs modules to the managed moduledir. moduledir '.modules' -mod 'simp/crypto_policy', '0.5.0' +mod 'simp/crypto_policy', '0.6.0' mod 'simp/fips', '0.9.0' mod 'puppetlabs/java_ks', '5.0.0' -mod 'puppetlabs/docker', '7.0.0' mod 'puppetlabs/concat', '7.4.0' mod 'puppetlabs/apt', '9.0.2' mod 'puppet/trusted_ca', '4.1.0' @@ -17,8 +16,5 @@ mod 'puppet/firewalld', '5.0.0' mod 'puppetlabs/stdlib', '8.6.0' mod 'simp/simplib', '4.12.2' mod 'puppet/augeasproviders_grub', '5.1.0' -mod 'puppetlabs/powershell', '6.0.0' -mod 'puppetlabs/reboot', '5.0.0' -mod 'simp/simpkv', '0.12.0' +mod 'simp/simpkv', '0.13.0' mod 'puppet/augeasproviders_core', '4.1.0' -mod 'puppetlabs/pwshlib', '1.1.1' diff --git a/puppet/modules/anms/manifests/docker.pp b/puppet/modules/anms/manifests/docker.pp index 77888c2..4e7df77 100644 --- a/puppet/modules/anms/manifests/docker.pp +++ b/puppet/modules/anms/manifests/docker.pp @@ -5,9 +5,9 @@ class anms::docker() { case $facts['os']['family'] { 'RedHat': { - package { ['podman', 'runc']: + package { ['podman-docker', 'podman', 'runc']: ensure => 'absent', - before => Package['docker-ce'], + before => [Package['docker-ce'], Package['docker-ce-cli']], } package { 'yum-utils': ensure => 'installed', @@ -30,6 +30,8 @@ ensure => 'installed', } service { 'docker': + ensure => 'running', + enable => true, require => Package['docker-ce'], } @@ -37,10 +39,11 @@ ensure => 'absent', } file { '/etc/docker/daemon.json': - source => 'puppet:///modules/anms/docker-daemon.json', - owner => 'root', - group => 'root', - mode => '0644', - notify => Service['docker'], + source => 'puppet:///modules/anms/docker-daemon.json', + owner => 'root', + group => 'root', + mode => '0644', + require => Package['docker-ce'], + notify => Service['docker'], } } diff --git a/puppet/modules/anms/manifests/docker_compose.pp b/puppet/modules/anms/manifests/docker_compose.pp index cb518dc..65338bd 100644 --- a/puppet/modules/anms/manifests/docker_compose.pp +++ b/puppet/modules/anms/manifests/docker_compose.pp @@ -8,6 +8,7 @@ define anms::docker_compose( Enum['present','absent'] $ensure, Array[String] $compose_files, + Boolean $pull_first = true, String $up_args = '', ) { require anms::docker @@ -17,16 +18,23 @@ case $ensure { 'present': { + if $pull_first { + exec { "docker-compose-${title}-pull": + path => $facts['path'], + command => "docker compose -p ${title} -f ${files_args} pull", + before => Exec["docker-compose-${title}-up"], + } + } exec { "docker-compose-${title}-up": - path => $facts['path'], - command => "docker compose -p ${title} -f ${files_args} up --detach --remove-orphans ${up_args}", + path => $facts['path'], + command => "docker compose -p ${title} -f ${files_args} up --detach --remove-orphans ${up_args}", } } 'absent': { - exec { "docker-compose-${title}-up": + exec { "docker-compose-${title}-rm": path => $facts['path'], command => "docker compose -p ${title} -f ${files_args} rm --force --stop", - onlyif => $is_running, + onlyif => $is_running, } } default: { diff --git a/puppet/modules/anms/manifests/init.pp b/puppet/modules/anms/manifests/init.pp index 8539339..660f315 100644 --- a/puppet/modules/anms/manifests/init.pp +++ b/puppet/modules/anms/manifests/init.pp @@ -12,9 +12,9 @@ String $cam_admin_password = '', String $cam_agent_name = $facts['networking']['fqdn'], String $cam_agent_password = '', -# String $tls_server_key, -# String $tls_server_cert, -# String $tls_server_ca, + Optional[String] $tls_server_key = undef, + Optional[String] $tls_server_cert = undef, + Optional[String] $tls_server_ca = undef, ) { require Class['anms::hostenv'] @@ -24,10 +24,9 @@ file { '/ammos/anms/.env': ensure => 'file', content => epp('anms/env.epp'), - } - file { '/ammos/anms/docker-compose.yml': - ensure => 'file', - source => 'puppet:///modules/anms/docker-compose.yml', + owner => 'root', + group => 'root', + mode => '0644', } file { ['/ammos/etc', '/ammos/etc/pki', '/ammos/etc/pki/tls', '/ammos/etc/pki/tls/private', '/ammos/etc/pki/tls/certs']: @@ -35,18 +34,21 @@ } file { '/ammos/etc/pki/tls/private/ammos-server-key.pem': ensure => 'file', + source => $tls_server_key, owner => 'root', group => 'root', mode => '0644', } file { '/ammos/etc/pki/tls/certs/ammos-server-cert.pem': ensure => 'file', + source => $tls_server_cert, owner => 'root', group => 'root', mode => '0644', } file { '/ammos/etc/pki/tls/certs/ammos-ca-bundle.crt': ensure => 'file', + source => $tls_server_ca, owner => 'root', group => 'root', mode => '0644', @@ -101,32 +103,48 @@ # Images pulled from remote registry if !empty($docker_image_prefix) and !empty($docker_registry_user) and !empty($docker_registry_pass) { exec { 'docker-login': - command => "docker login ${docker_image_prefix} --username \"${docker_registry_user}\" --password \"${docker_registry_pass}\"", path => $facts['path'], + command => "docker login ${docker_image_prefix} --username \"${docker_registry_user}\" --password \"${docker_registry_pass}\"", + require => Service['docker'], before => [ - Exec['anms-pull'], - Exec['agents-pull'], - ], - } - exec { 'anms-pull': - command => 'docker-compose -f /ammos/anms/docker-compose.yml pull', - path => $facts['path'], - require => [ - File['/ammos/anms/docker-compose.yml'], - File['/ammos/anms/.env'], - ], - before => Anms::Docker_compose['anms'], - } - exec { 'agents-pull': - command => 'docker-compose -f /ammos/anms/agent-compose.yml pull', - path => $facts['path'], - require => [ - File['/ammos/anms/agent-compose.yml'], - File['/ammos/anms/.env'], + Anms::Docker_compose['anms'], + Anms::Docker_compose['agents'], ], - before => Anms::Docker_compose['agents'], } } + + # volume for TLS-related PKIX files + file { '/ammos/anms/create_volume.sh': + ensure => 'file', + source => 'puppet:///modules/anms/create_volume.sh', + owner => 'root', + group => 'root', + mode => '0755', + } + exec { 'volume-ammos-tls': + path => $facts['path'], + command => '/ammos/anms/create_volume.sh', + unless => 'docker volume inspect ammos-tls', + require => [ + Service['docker'], + File['/ammos/anms/create_volume.sh'], + ], + subscribe => [ + File['/ammos/etc/pki/tls/private/ammos-server-key.pem'], + File['/ammos/etc/pki/tls/certs/ammos-server-cert.pem'], + File['/ammos/etc/pki/tls/certs/ammos-ca-bundle.crt'], + ], + before => Anms::Docker_compose['anms'], + notify => Anms::Docker_compose['anms'], + } + + file { '/ammos/anms/docker-compose.yml': + ensure => 'file', + source => 'puppet:///modules/anms/docker-compose.yml', + owner => 'root', + group => 'root', + mode => '0644', + } anms::docker_compose { 'anms': ensure => 'present', compose_files => ['/ammos/anms/docker-compose.yml'], @@ -140,6 +158,9 @@ file { '/ammos/anms/agent-compose.yml': ensure => 'file', source => 'puppet:///modules/anms/agent-compose.yml', + owner => 'root', + group => 'root', + mode => '0644', } anms::docker_compose { 'agents': ensure => 'present', diff --git a/puppet/prep.sh b/puppet/prep.sh index 8f7dab2..22abaf1 100755 --- a/puppet/prep.sh +++ b/puppet/prep.sh @@ -28,8 +28,11 @@ set -e SELFDIR=$(dirname "${BASH_SOURCE[0]}") source ${SELFDIR}/getenv.sh -rm -f ${SELFDIR}/modules/anms/files/*-compose.yml -cp "${SELFDIR}/../docker-compose.yml" "${SELFDIR}/../agent-compose.yml" "${SELFDIR}/modules/anms/files/" +rm -f "${SELFDIR}"/modules/anms/files/*-compose.yml +cp "${SELFDIR}/../docker-compose.yml" \ + "${SELFDIR}/../agent-compose.yml" \ + "${SELFDIR}/../create_volume.sh" \ + "${SELFDIR}/modules/anms/files/" export BOLT_PROJECT="${SELFDIR}" bolt module install --force From 61f75902003aea6a83c3b7ad8cafe986127f0ff4 Mon Sep 17 00:00:00 2001 From: Brian Sipos Date: Thu, 30 May 2024 09:28:04 -0400 Subject: [PATCH 2/5] Fixing indentation --- puppet/modules/anms/manifests/init.pp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/puppet/modules/anms/manifests/init.pp b/puppet/modules/anms/manifests/init.pp index 660f315..360bf1b 100644 --- a/puppet/modules/anms/manifests/init.pp +++ b/puppet/modules/anms/manifests/init.pp @@ -24,9 +24,9 @@ file { '/ammos/anms/.env': ensure => 'file', content => epp('anms/env.epp'), - owner => 'root', - group => 'root', - mode => '0644', + owner => 'root', + group => 'root', + mode => '0644', } file { ['/ammos/etc', '/ammos/etc/pki', '/ammos/etc/pki/tls', '/ammos/etc/pki/tls/private', '/ammos/etc/pki/tls/certs']: From 246243ec3b7b49066634b535b93cb32d4b43cf68 Mon Sep 17 00:00:00 2001 From: Brian Sipos Date: Thu, 30 May 2024 10:06:09 -0400 Subject: [PATCH 3/5] Add hostenv option to control FIPS mode at all --- .gitlab-ci.yml | 1 + puppet/modules/anms/files/selinux/authnz.cil | 1 + puppet/modules/anms/manifests/hostenv.pp | 36 ++++++++++++-------- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 32f545c..81100b3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -349,6 +349,7 @@ deploy: anms::tls_server_key: 'puppet:///modules/apl_test/anms/tls/private/ammos-server-key.pem' anms::tls_server_cert: 'puppet:///modules/apl_test/anms/tls/certs/ammos-server-cert.pem' anms::tls_server_ca: 'puppet:///modules/apl_test/anms/tls/certs/ammos-ca-bundle.crt' + anms::hostenv::use_fips: false selinux::mode: permissive selinux::type: targeted EOF diff --git a/puppet/modules/anms/files/selinux/authnz.cil b/puppet/modules/anms/files/selinux/authnz.cil index 8a7bcda..ea5b392 100644 --- a/puppet/modules/anms/files/selinux/authnz.cil +++ b/puppet/modules/anms/files/selinux/authnz.cil @@ -1,6 +1,7 @@ (block authnz (blockinherit container) (blockinherit restricted_net_container) + (allow process container_file_t ( chr_file ( map ))) (allow process http_port_t ( tcp_socket ( name_bind ))) (allow process http_port_t ( tcp_socket ( name_connect ))) ) \ No newline at end of file diff --git a/puppet/modules/anms/manifests/hostenv.pp b/puppet/modules/anms/manifests/hostenv.pp index f84a0a8..a306c25 100644 --- a/puppet/modules/anms/manifests/hostenv.pp +++ b/puppet/modules/anms/manifests/hostenv.pp @@ -1,12 +1,16 @@ # Define host environment configuration for ANMS installation. # -class anms::hostenv() { +class anms::hostenv( + Boolean $use_fips = true, +) { case $facts['os']['family'] { 'RedHat': { # This halts on reboot_notify() # instead run with: # bolt apply --execute 'class {"fips": }' -# class { 'fips': } + if $use_fips { + class { 'fips': } + } file { '/var/cache/puppet': ensure => 'directory', @@ -60,20 +64,22 @@ 'Debian': { case $facts['os']['distro']['codename'] { 'focal': { - # Based on guidance at: https://aplwiki.jhuapl.edu/confluence/pages/viewpage.action?spaceKey=LAPLKEY&title=Ubuntu+FIPS+Packages - apt::source { 'focal-fips': - location => 'https://apllinuxdepot.jhuapl.edu/linux/apl-software/focal-fips/', - release => '', - repos => '/', - key => { - id => '6F6B15509CF8E59E6E469F327F438280EF8D349F', - server => 'https://apllinuxdepot.jhuapl.edu/linux/apl-software/focal-fips/apl-software-repo.gpg', - }, - } + if $use_fips { + # Based on guidance at: https://aplwiki.jhuapl.edu/confluence/pages/viewpage.action?spaceKey=LAPLKEY&title=Ubuntu+FIPS+Packages + apt::source { 'focal-fips': + location => 'https://apllinuxdepot.jhuapl.edu/linux/apl-software/focal-fips/', + release => '', + repos => '/', + key => { + id => '6F6B15509CF8E59E6E469F327F438280EF8D349F', + server => 'https://apllinuxdepot.jhuapl.edu/linux/apl-software/focal-fips/apl-software-repo.gpg', + }, + } - package { 'ubuntu-fips': - ensure => 'installed', - require => Apt::Source['focal-fips'], + package { 'ubuntu-fips': + ensure => 'installed', + require => Apt::Source['focal-fips'], + } } } default: { From d43d0cd116b0637064705c29ee14bb2361e28868 Mon Sep 17 00:00:00 2001 From: Brian Sipos Date: Thu, 30 May 2024 10:18:15 -0400 Subject: [PATCH 4/5] Increasing pip timeout --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 81100b3..17e15f5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -42,6 +42,7 @@ variables: DOCKER_BUILDKIT: 1 # for Python/PIP PIP_CERT: /etc/pki/tls/certs/ca-bundle.crt + PIP_DEFAULT_TIMEOUT: 300 # for Ruby/bolt SSL_CERT_FILE: /etc/pki/tls/certs/ca-bundle.crt # Project-specific environment From 9ba195f2cefb8dcea18404e3517ced26811632a8 Mon Sep 17 00:00:00 2001 From: linkodm1 Date: Thu, 30 May 2024 14:21:36 -0400 Subject: [PATCH 5/5] bumping version of docker to fix request issue --- anms-core/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/anms-core/pyproject.toml b/anms-core/pyproject.toml index 24a8cd8..92ae104 100644 --- a/anms-core/pyproject.toml +++ b/anms-core/pyproject.toml @@ -26,7 +26,7 @@ dependencies = [ "asyncio ~=3.4.3", "asyncpg ~=0.27.0", "authlib ~=0.15.5", - "docker ~=6.0.1", + "docker ~=7.1.0", "emails ~=0.6", "email-validator ~=1.3", "fastapi ~=0.86.0",