From 4d2ad41860faf9731c02606fa92ccd276dccf3c9 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 12 Aug 2024 14:18:18 -0600 Subject: [PATCH 01/44] Debugging --- salt/minion.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/salt/minion.py b/salt/minion.py index cb4ef919b305..8c83a2283c02 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -279,6 +279,12 @@ def get_proc_dir(cachedir, **kwargs): made. Same applies if the directory is already owned by this gid. Must be int. Works only on unix/unix like systems. """ + # pylint: disable=logging-fstring-interpolation + log.warning(f"DGM get_proc_dir entry, cachedir '{cachedir}', kwargs '{kwargs}'") + print( + f"DGM get_proc_dir entry, cachedir '{cachedir}', kwargs '{kwargs}'", flush=True + ) + fn_ = os.path.join(cachedir, "proc") mode = kwargs.pop("mode", None) @@ -305,11 +311,29 @@ def get_proc_dir(cachedir, **kwargs): uid = kwargs.pop("uid", -1) gid = kwargs.pop("gid", -1) + # pylint: disable=logging-fstring-interpolation + log.warning( + f"DGM get_proc_dir chown, d_stat '{d_stat}', uid '{uid}', gid '{gid}'" + ) + print( + f"DGM get_proc_dir chown, d_stat '{d_stat}', uid '{uid}', gid '{gid}'", + flush=True, + ) + # if uid and gid are both -1 then go ahead with # no changes at all if (d_stat.st_uid != uid or d_stat.st_gid != gid) and [ i for i in (uid, gid) if i != -1 ]: + # pylint: disable=logging-fstring-interpolation + log.warning( + f"DGM get_proc_dir chown file, file '{fn_}' , uid '{uid}', gid '{gid}'" + ) + print( + f"DGM get_proc_dir chown file, file '{fn_}' , uid '{uid}', gid '{gid}'", + flush=True, + ) + os.chown(fn_, uid, gid) return fn_ From d3ad2520d33edc56c26ed0d5c25db5f3012bbacc Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 14 Aug 2024 11:12:43 -0600 Subject: [PATCH 02/44] Updated salt-syndic to utilize user/group for Salt (master settings), and maintain on upgrade --- pkg/common/salt-api.service | 1 + pkg/debian/salt-api.preinst | 1 - pkg/debian/salt-syndic.postinst | 37 ++++++++++++++++++++++++++++++++ pkg/debian/salt-syndic.preinst | 27 +++++++++++++++++++++++ pkg/debian/salt-syndic.templates | 17 +++++++++++++++ pkg/rpm/salt.spec | 31 +++++++++++++++++++++++++- salt/minion.py | 36 +++++++++++++++---------------- 7 files changed, 130 insertions(+), 20 deletions(-) create mode 100644 pkg/debian/salt-syndic.postinst create mode 100644 pkg/debian/salt-syndic.preinst create mode 100644 pkg/debian/salt-syndic.templates diff --git a/pkg/common/salt-api.service b/pkg/common/salt-api.service index d0b6d741202b..3eefa7649009 100644 --- a/pkg/common/salt-api.service +++ b/pkg/common/salt-api.service @@ -2,6 +2,7 @@ Description=The Salt API Documentation=man:salt-api(1) file:///usr/share/doc/salt/html/contents.html https://docs.saltproject.io/en/latest/contents.html After=network.target +PartOf=salt-master.service [Service] Type=notify diff --git a/pkg/debian/salt-api.preinst b/pkg/debian/salt-api.preinst index ddc7c9e0ec7b..c063108ea55e 100644 --- a/pkg/debian/salt-api.preinst +++ b/pkg/debian/salt-api.preinst @@ -22,7 +22,6 @@ case "$1" in else db_set salt-api/enabled enabled db_set salt-api/active active - fi ;; esac diff --git a/pkg/debian/salt-syndic.postinst b/pkg/debian/salt-syndic.postinst new file mode 100644 index 000000000000..071ba38e1859 --- /dev/null +++ b/pkg/debian/salt-syndic.postinst @@ -0,0 +1,37 @@ +#!/bin/sh + +. /usr/share/debconf/confmodule + +case "$1" in + configure) + db_get salt-syndic/user + if [ "$RET" != "root" ]; then + if [ ! -e "/var/log/salt/syndic" ]; then + touch /var/log/salt/syndic + chmod 640 /var/log/salt/syndic + fi + chown $RET:$RET /var/log/salt/syndic + fi + if command -v systemctl; then + db_get salt-syndic/active + RESLT=$(echo "$RET" | cut -d ' ' -f 1) + if [ "$RESLT" != 10 ]; then + systemctl daemon-reload + if [ "$RESLT" = "active" ]; then + systemctl restart salt-syndic + fi + db_get salt-syndic/enabled + RESLT=$(echo "$RET" | cut -d ' ' -f 1) + if [ "$RESLT" = "disabled" ]; then + systemctl disable salt-syndic + else + systemctl enable salt-syndic + fi + else + systemctl daemon-reload + systemctl restart salt-syndic + systemctl enable salt-syndic + fi + fi + ;; +esac diff --git a/pkg/debian/salt-syndic.preinst b/pkg/debian/salt-syndic.preinst new file mode 100644 index 000000000000..da43d779163c --- /dev/null +++ b/pkg/debian/salt-syndic.preinst @@ -0,0 +1,27 @@ +#!/bin/sh + +. /usr/share/debconf/confmodule + +case "$1" in + upgrade) + [ -z "$SALT_HOME" ] && SALT_HOME=/opt/saltstack/salt + [ -z "$SALT_USER" ] && SALT_USER=salt + [ -z "$SALT_NAME" ] && SALT_NAME="Salt" + [ -z "$SALT_GROUP" ] && SALT_GROUP=salt + + # Reset permissions to fix previous installs + CUR_USER=$(ls -dl /run/salt-syndic.pid | cut -d ' ' -f 3) + CUR_GROUP=$(ls -dl /run/salt-syndic.pid | cut -d ' ' -f 4) + db_set salt-syndic/user $CUR_USER + chown -R $CUR_USER:$CUR_GROUP /var/log/salt/syndic + if command -v systemctl; then + SM_ENABLED=$(systemctl show -p UnitFileState salt-syndic | cut -d '=' -f 2) + db_set salt-syndic/enabled $SM_ENABLED + SM_ACTIVE=$(systemctl is-active salt-syndic) + db_set salt-syndic/active $SM_ACTIVE + else + db_set salt-syndic/enabled enabled + db_set salt-syndic/active active + fi + ;; +esac diff --git a/pkg/debian/salt-syndic.templates b/pkg/debian/salt-syndic.templates new file mode 100644 index 000000000000..c27859e0a24f --- /dev/null +++ b/pkg/debian/salt-syndic.templates @@ -0,0 +1,17 @@ +Template: salt-syndic/user +Type: string +Default: salt +Description: User for salt-syndic + User to run the salt-syndic process as + +Template: salt-syndic/enabled +Type: string +Default: enabled +Description: Systemd enable state for salt-syndic + default enable state for salt-syndic systemd state + +Template: salt-syndic/active +Type: string +Default: active +Description: Systemd active state for salt-syndic + default active state for salt-syndic systemd state diff --git a/pkg/rpm/salt.spec b/pkg/rpm/salt.spec index 180df99f34b1..f1707cb5dd07 100644 --- a/pkg/rpm/salt.spec +++ b/pkg/rpm/salt.spec @@ -441,6 +441,15 @@ if [ $1 -gt 1 ] ; then %global _MS_CUR_GROUP %{_MS_LCUR_GROUP} fi +%pre syndic +if [ $1 -gt 1 ] ; then + # Reset permissions to match previous installs - performing upgrade + _MS_LCUR_USER=$(ls -dl /run/salt/master | cut -d ' ' -f 3) + _MS_LCUR_GROUP=$(ls -dl /run/salt/master | cut -d ' ' -f 4) + %global _MS_CUR_USER %{_MS_LCUR_USER} + %global _MS_CUR_GROUP %{_MS_LCUR_GROUP} +fi + %pre minion if [ $1 -gt 1 ] ; then # Reset permissions to match previous installs - performing upgrade @@ -463,6 +472,14 @@ if [ $1 -eq 0 ] ; then /bin/systemctl stop salt-syndic.service > /dev/null 2>&1 || : fi +%preun syndic +# %%systemd_preun salt-syndic.service +if [ $1 -eq 0 ] ; then + # Package removal, not upgrade + /bin/systemctl --no-reload disable salt-syndic.service > /dev/null 2>&1 || : + /bin/systemctl stop salt-syndic.service > /dev/null 2>&1 || : +fi + %preun minion # %%systemd_preun salt-minion.service if [ $1 -eq 0 ] ; then @@ -471,7 +488,6 @@ if [ $1 -eq 0 ] ; then /bin/systemctl stop salt-minion.service > /dev/null 2>&1 || : fi - %preun api # %%systemd_preun salt-api.service if [ $1 -eq 0 ] ; then @@ -602,6 +618,19 @@ else fi +%posttrans syndic +if [ ! -e "/var/log/salt/syndic" ]; then + touch /var/log/salt/syndic + chmod 640 /var/log/salt/syndic +fi +if [ $1 -gt 1 ] ; then + # Reset permissions to match previous installs - performing upgrade + chown -R %{_MS_CUR_USER}:%{_MS_CUR_GROUP} /var/log/salt/syndic +else + chown -R %{_SALT_USER}:%{_SALT_GROUP} /var/log/salt/syndic +fi + + %posttrans api if [ ! -e "/var/log/salt/api" ]; then touch /var/log/salt/api diff --git a/salt/minion.py b/salt/minion.py index 8c83a2283c02..64057e81fccc 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -280,10 +280,10 @@ def get_proc_dir(cachedir, **kwargs): gid. Must be int. Works only on unix/unix like systems. """ # pylint: disable=logging-fstring-interpolation - log.warning(f"DGM get_proc_dir entry, cachedir '{cachedir}', kwargs '{kwargs}'") - print( - f"DGM get_proc_dir entry, cachedir '{cachedir}', kwargs '{kwargs}'", flush=True - ) + ## DGM log.warning(f"DGM get_proc_dir entry, cachedir '{cachedir}', kwargs '{kwargs}'") + ## DGM print( + ## DGM f"DGM get_proc_dir entry, cachedir '{cachedir}', kwargs '{kwargs}'", flush=True + ## DGM ) fn_ = os.path.join(cachedir, "proc") mode = kwargs.pop("mode", None) @@ -312,13 +312,13 @@ def get_proc_dir(cachedir, **kwargs): gid = kwargs.pop("gid", -1) # pylint: disable=logging-fstring-interpolation - log.warning( - f"DGM get_proc_dir chown, d_stat '{d_stat}', uid '{uid}', gid '{gid}'" - ) - print( - f"DGM get_proc_dir chown, d_stat '{d_stat}', uid '{uid}', gid '{gid}'", - flush=True, - ) + ## DGM log.warning( + ## DGM f"DGM get_proc_dir chown, d_stat '{d_stat}', uid '{uid}', gid '{gid}'" + ## DGM ) + ## DGM print( + ## DGM f"DGM get_proc_dir chown, d_stat '{d_stat}', uid '{uid}', gid '{gid}'", + ## DGM flush=True, + ## DGM ) # if uid and gid are both -1 then go ahead with # no changes at all @@ -326,13 +326,13 @@ def get_proc_dir(cachedir, **kwargs): i for i in (uid, gid) if i != -1 ]: # pylint: disable=logging-fstring-interpolation - log.warning( - f"DGM get_proc_dir chown file, file '{fn_}' , uid '{uid}', gid '{gid}'" - ) - print( - f"DGM get_proc_dir chown file, file '{fn_}' , uid '{uid}', gid '{gid}'", - flush=True, - ) + ## DGM log.warning( + ## DGM f"DGM get_proc_dir chown file, file '{fn_}' , uid '{uid}', gid '{gid}'" + ## DGM ) + ## DGM print( + ## DGM f"DGM get_proc_dir chown file, file '{fn_}' , uid '{uid}', gid '{gid}'", + ## DGM flush=True, + ## DGM ) os.chown(fn_, uid, gid) From 5298d333fa8585d8fed6495281cd0ce24ff0d15d Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 14 Aug 2024 13:20:50 -0600 Subject: [PATCH 03/44] Update tests --- tests/pytests/pkg/integration/test_salt_user.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/pytests/pkg/integration/test_salt_user.py b/tests/pytests/pkg/integration/test_salt_user.py index fb42ae3c9f6b..aad608275181 100644 --- a/tests/pytests/pkg/integration/test_salt_user.py +++ b/tests/pytests/pkg/integration/test_salt_user.py @@ -67,8 +67,12 @@ def pkg_paths_salt_user(): "/var/log/salt/master", "/var/log/salt/api", "/var/log/salt/key", + "/var/log/salt/syndic", "/var/cache/salt/master", "/var/run/salt/master", + "/run/salt-master.pid", + "/run/salt-syndic.pid", + "/run/salt-api.pid", ] From e097b541c916de4587b8f95502a39b30f9eed454 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 19 Aug 2024 16:39:15 -0600 Subject: [PATCH 04/44] Testing with master_opts user for syndic_user --- salt/config/__init__.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/salt/config/__init__.py b/salt/config/__init__.py index 6d0bc947fc22..cffce8929f6a 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -2502,7 +2502,11 @@ def syndic_config( ), ) ), - "user": opts.get("syndic_user", opts["user"]), + # DGM conf/suse/master has syndic_user for running syndic as a different user + # DGM use urrently set it to 'salt' + # DGM testing getting the master_opts user, and run syndic as that + # DGM "user": opts.get("syndic_user", opts["user"]), + "user": opts.get("syndic_user", master_opts["user"]), "sock_dir": os.path.join( opts["cachedir"], opts.get("syndic_sock_dir", opts["sock_dir"]) ), @@ -2510,6 +2514,10 @@ def syndic_config( "cachedir": master_opts["cachedir"], } opts.update(syndic_opts) + log.warning("DGM syndic_config, user is set to '%s'", opts["user"]) + dgm_user = opts["user"] + print(f"DGM syndic_config, user is set to '{dgm_user}'", flush=True) + # Prepend root_dir to other paths prepend_root_dirs = [ "pki_dir", From 853d3dafe3b9228cbecdca317268d7d9dfc2026d Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 20 Aug 2024 11:38:59 -0600 Subject: [PATCH 05/44] Added delay for salt_master running, and updated some package tests --- .../pytests/pkg/integration/test_salt_api.py | 5 ---- .../pytests/pkg/integration/test_salt_user.py | 30 ++++++++++++------- tests/pytests/pkg/integration/test_version.py | 8 +++-- 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/tests/pytests/pkg/integration/test_salt_api.py b/tests/pytests/pkg/integration/test_salt_api.py index e962fbe32213..a4f11973748f 100644 --- a/tests/pytests/pkg/integration/test_salt_api.py +++ b/tests/pytests/pkg/integration/test_salt_api.py @@ -9,11 +9,6 @@ def test_salt_api(api_request, salt_master, install_salt): """ Test running a command against the salt api """ - if install_salt.distro_id in ("ubuntu", "debian"): - pytest.skip( - "Package test are getting reworked in https://github.com/saltstack/salt/issues/66672" - ) - assert salt_master.is_running() ret = api_request.post( diff --git a/tests/pytests/pkg/integration/test_salt_user.py b/tests/pytests/pkg/integration/test_salt_user.py index aad608275181..96180a972fe9 100644 --- a/tests/pytests/pkg/integration/test_salt_user.py +++ b/tests/pytests/pkg/integration/test_salt_user.py @@ -2,6 +2,7 @@ import pathlib import subprocess import sys +import time import packaging.version import psutil @@ -91,6 +92,13 @@ def test_salt_user_master(salt_master, install_salt): """ Test the correct user is running the Salt Master """ + # DGM assert salt_master.is_running() + + for count in range(0, 10): + if salt_master.is_running(): + break + else: + time.sleep(1) assert salt_master.is_running() match = False @@ -227,17 +235,17 @@ def test_paths_log_rotation( ): pytest.skip("Package path ownership was changed in salt 3006.4") - if install_salt.distro_id not in ( - "almalinux", - "rocky", - "centos", - "redhat", - "amzn", - "fedora", - ): - pytest.skip( - "Only tests RedHat family packages till logrotation paths are resolved on Ubuntu/Debian, see issue 65231" - ) + # DGM if install_salt.distro_id not in ( + # DGM "almalinux", + # DGM "rocky", + # DGM "centos", + # DGM "redhat", + # DGM "amzn", + # DGM "fedora", + # DGM ): + # DGM pytest.skip( + # DGM "Only tests RedHat family packages till logrotation paths are resolved on Ubuntu/Debian, see issue 65231" + # DGM ) match = False for proc in psutil.Process(salt_master.pid).children(): diff --git a/tests/pytests/pkg/integration/test_version.py b/tests/pytests/pkg/integration/test_version.py index b1bee9d60afa..dc710f1b7ccc 100644 --- a/tests/pytests/pkg/integration/test_version.py +++ b/tests/pytests/pkg/integration/test_version.py @@ -6,7 +6,7 @@ from pytestskipmarkers.utils import platform -@pytest.mark.skip_on_windows +# DGM @pytest.mark.skip_on_windows def test_salt_version(version, install_salt): """ Test version output from salt --version @@ -35,6 +35,7 @@ def test_salt_version(version, install_salt): @pytest.mark.skip_on_windows +@pytest.mark.skip_on_darwin def test_salt_versions_report_master(install_salt): """ Test running --versions-report on master @@ -52,7 +53,7 @@ def test_salt_versions_report_master(install_salt): ret.stdout.matcher.fnmatch_lines([f"*{py_version}*"]) -@pytest.mark.skip_on_windows +# DGM @pytest.mark.skip_on_windows def test_salt_versions_report_minion(salt_cli, salt_call_cli, salt_minion): """ Test running test.versions_report on minion @@ -104,7 +105,8 @@ def test_compare_versions(binary, install_salt): ) -@pytest.mark.skip_unless_on_darwin() +# DGM +@pytest.mark.skip_on_windows @pytest.mark.parametrize( "symlink", [ From 6874ff572e2ba832c827df6ebfb63b2f11d24035 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 21 Aug 2024 10:53:55 -0600 Subject: [PATCH 06/44] Revert syndic from master opts to check test failures --- salt/config/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/config/__init__.py b/salt/config/__init__.py index cffce8929f6a..87649b5e2ad5 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -2505,8 +2505,8 @@ def syndic_config( # DGM conf/suse/master has syndic_user for running syndic as a different user # DGM use urrently set it to 'salt' # DGM testing getting the master_opts user, and run syndic as that - # DGM "user": opts.get("syndic_user", opts["user"]), - "user": opts.get("syndic_user", master_opts["user"]), + "user": opts.get("syndic_user", opts["user"]), + # DGM test revert "user": opts.get("syndic_user", master_opts["user"]), "sock_dir": os.path.join( opts["cachedir"], opts.get("syndic_sock_dir", opts["sock_dir"]) ), From b0e1dfd7b32c9b155b9190868ea17ec6823fef46 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 21 Aug 2024 14:04:53 -0600 Subject: [PATCH 07/44] Revert logrotation test to RedHat family only --- .../pytests/pkg/integration/test_salt_user.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/pytests/pkg/integration/test_salt_user.py b/tests/pytests/pkg/integration/test_salt_user.py index 96180a972fe9..864528b01cb2 100644 --- a/tests/pytests/pkg/integration/test_salt_user.py +++ b/tests/pytests/pkg/integration/test_salt_user.py @@ -235,17 +235,17 @@ def test_paths_log_rotation( ): pytest.skip("Package path ownership was changed in salt 3006.4") - # DGM if install_salt.distro_id not in ( - # DGM "almalinux", - # DGM "rocky", - # DGM "centos", - # DGM "redhat", - # DGM "amzn", - # DGM "fedora", - # DGM ): - # DGM pytest.skip( - # DGM "Only tests RedHat family packages till logrotation paths are resolved on Ubuntu/Debian, see issue 65231" - # DGM ) + if install_salt.distro_id not in ( + "almalinux", + "rocky", + "centos", + "redhat", + "amzn", + "fedora", + ): + pytest.skip( + "Only tests RedHat family packages till logrotation paths are resolved on Ubuntu/Debian, see issue 65231" + ) match = False for proc in psutil.Process(salt_master.pid).children(): From 2c8becaf8115ba272668c730bf3b04e51f121a06 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 23 Aug 2024 13:18:17 -0600 Subject: [PATCH 08/44] Adjust tests for syndic --- salt/config/__init__.py | 4 ++-- .../pytests/pkg/integration/test_salt_api.py | 6 ++++++ .../pytests/pkg/integration/test_salt_user.py | 8 ++++---- tests/pytests/pkg/integration/test_version.py | 19 ++++++++++++++----- 4 files changed, 26 insertions(+), 11 deletions(-) diff --git a/salt/config/__init__.py b/salt/config/__init__.py index 87649b5e2ad5..cffce8929f6a 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -2505,8 +2505,8 @@ def syndic_config( # DGM conf/suse/master has syndic_user for running syndic as a different user # DGM use urrently set it to 'salt' # DGM testing getting the master_opts user, and run syndic as that - "user": opts.get("syndic_user", opts["user"]), - # DGM test revert "user": opts.get("syndic_user", master_opts["user"]), + # DGM "user": opts.get("syndic_user", opts["user"]), + "user": opts.get("syndic_user", master_opts["user"]), "sock_dir": os.path.join( opts["cachedir"], opts.get("syndic_sock_dir", opts["sock_dir"]) ), diff --git a/tests/pytests/pkg/integration/test_salt_api.py b/tests/pytests/pkg/integration/test_salt_api.py index a4f11973748f..f01e7c5fd7c2 100644 --- a/tests/pytests/pkg/integration/test_salt_api.py +++ b/tests/pytests/pkg/integration/test_salt_api.py @@ -9,6 +9,12 @@ def test_salt_api(api_request, salt_master, install_salt): """ Test running a command against the salt api """ + # DGM TBD 66672 is merged, need to remove this check and see if test passes + if install_salt.distro_id in ("ubuntu", "debian"): + pytest.skip( + "Package test are getting reworked in https://github.com/saltstack/salt/issues/66672" + ) + assert salt_master.is_running() ret = api_request.post( diff --git a/tests/pytests/pkg/integration/test_salt_user.py b/tests/pytests/pkg/integration/test_salt_user.py index 864528b01cb2..941f8c758d9f 100644 --- a/tests/pytests/pkg/integration/test_salt_user.py +++ b/tests/pytests/pkg/integration/test_salt_user.py @@ -92,13 +92,13 @@ def test_salt_user_master(salt_master, install_salt): """ Test the correct user is running the Salt Master """ - # DGM assert salt_master.is_running() - - for count in range(0, 10): + for count in range(0, 30): if salt_master.is_running(): break else: - time.sleep(1) + time.sleep(2) + + print(f"DGM test_salt_user_master, salt_master '{salt_master}'", flush=True) assert salt_master.is_running() match = False diff --git a/tests/pytests/pkg/integration/test_version.py b/tests/pytests/pkg/integration/test_version.py index dc710f1b7ccc..815df570d936 100644 --- a/tests/pytests/pkg/integration/test_version.py +++ b/tests/pytests/pkg/integration/test_version.py @@ -1,12 +1,13 @@ import os.path import pathlib import subprocess +import time import pytest from pytestskipmarkers.utils import platform -# DGM @pytest.mark.skip_on_windows +@pytest.mark.skip_on_windows def test_salt_version(version, install_salt): """ Test version output from salt --version @@ -53,12 +54,19 @@ def test_salt_versions_report_master(install_salt): ret.stdout.matcher.fnmatch_lines([f"*{py_version}*"]) -# DGM @pytest.mark.skip_on_windows +@pytest.mark.skip_on_windows def test_salt_versions_report_minion(salt_cli, salt_call_cli, salt_minion): """ Test running test.versions_report on minion """ # Make sure the minion is running + for count in range(0, 30): + if salt_minion.is_running(): + break + else: + time.sleep(2) + + print(f"DGM test_salt_user_mnion, salt_minion '{salt_minion}'", flush=True) assert salt_minion.is_running() # Make sure we can ping the minion ... @@ -78,6 +86,8 @@ def test_salt_versions_report_minion(salt_cli, salt_call_cli, salt_minion): ret.stdout.matcher.fnmatch_lines(["*Salt Version:*"]) +@pytest.mark.skip_on_windows +@pytest.mark.skip_on_darwin @pytest.mark.parametrize( "binary", ["master", "cloud", "syndic", "minion", "call", "api"] ) @@ -105,8 +115,8 @@ def test_compare_versions(binary, install_salt): ) -# DGM @pytest.mark.skip_on_windows +@pytest.mark.skip_on_darwin @pytest.mark.parametrize( "symlink", [ @@ -134,8 +144,7 @@ def test_symlinks_created(version, symlink, install_salt): ret.stdout.matcher.fnmatch_lines([f"*{version}*"]) -@pytest.mark.skip_on_windows -@pytest.mark.skip_on_darwin +@pytest.mark.skip_unless_on_linux def test_compare_pkg_versions_redhat_rc(version, install_salt): """ Test compare pkg versions for redhat RC packages. A tilde should be included From fbd6b7490022eaed1f0eb7845105c1a387bbcd4a Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 23 Aug 2024 15:20:25 -0600 Subject: [PATCH 09/44] Further debug tests --- pkg/common/salt-api.service | 2 +- .../pytests/pkg/integration/test_salt_user.py | 5 +++- tests/pytests/pkg/integration/test_version.py | 24 +++++++++++++++++-- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/pkg/common/salt-api.service b/pkg/common/salt-api.service index 3eefa7649009..3e47a2e6366a 100644 --- a/pkg/common/salt-api.service +++ b/pkg/common/salt-api.service @@ -2,7 +2,7 @@ Description=The Salt API Documentation=man:salt-api(1) file:///usr/share/doc/salt/html/contents.html https://docs.saltproject.io/en/latest/contents.html After=network.target -PartOf=salt-master.service +# DGM PartOf=salt-master.service [Service] Type=notify diff --git a/tests/pytests/pkg/integration/test_salt_user.py b/tests/pytests/pkg/integration/test_salt_user.py index 941f8c758d9f..07c29837e901 100644 --- a/tests/pytests/pkg/integration/test_salt_user.py +++ b/tests/pytests/pkg/integration/test_salt_user.py @@ -98,7 +98,10 @@ def test_salt_user_master(salt_master, install_salt): else: time.sleep(2) - print(f"DGM test_salt_user_master, salt_master '{salt_master}'", flush=True) + print( + f"DGM test_salt_user_master, salt_master '{salt_master}' and is_running '{salt_master.is_running()}'", + flush=True, + ) assert salt_master.is_running() match = False diff --git a/tests/pytests/pkg/integration/test_version.py b/tests/pytests/pkg/integration/test_version.py index 815df570d936..eb4278732570 100644 --- a/tests/pytests/pkg/integration/test_version.py +++ b/tests/pytests/pkg/integration/test_version.py @@ -55,7 +55,7 @@ def test_salt_versions_report_master(install_salt): @pytest.mark.skip_on_windows -def test_salt_versions_report_minion(salt_cli, salt_call_cli, salt_minion): +def test_salt_versions_report_minion(salt_cli, salt_call_cli, salt_minion, salt_master): """ Test running test.versions_report on minion """ @@ -66,13 +66,33 @@ def test_salt_versions_report_minion(salt_cli, salt_call_cli, salt_minion): else: time.sleep(2) - print(f"DGM test_salt_user_mnion, salt_minion '{salt_minion}'", flush=True) + print( + f"DGM test_salt_user_minion, salt_minion '{salt_minion}' and is_running '{salt_minion.is_running()}'", + flush=True, + ) assert salt_minion.is_running() + # DGM + # Make sure the master is running + for count in range(0, 30): + if salt_master.is_running(): + break + else: + time.sleep(2) + + print( + f"DGM test_salt_user_minion, salt_master '{salt_master}' and is_running '{salt_master.is_running()}'", + flush=True, + ) + assert salt_master.is_running() + # DGM + # Make sure we can ping the minion ... ret = salt_cli.run( "--timeout=300", "test.ping", minion_tgt=salt_minion.id, _timeout=300 ) + print(f"DGM test_salt_user_minion, test.ping ret '{ret}'", flush=True) + assert ret.returncode == 0 assert ret.data is True ret = salt_cli.run( From 7be41f56db50bac4fcfb7d32dda160ed3a0e2953 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 26 Aug 2024 17:28:53 -0600 Subject: [PATCH 10/44] Updated tests, and debugging --- tests/pytests/pkg/integration/test_version.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/tests/pytests/pkg/integration/test_version.py b/tests/pytests/pkg/integration/test_version.py index eb4278732570..1402a4ef79fd 100644 --- a/tests/pytests/pkg/integration/test_version.py +++ b/tests/pytests/pkg/integration/test_version.py @@ -62,6 +62,10 @@ def test_salt_versions_report_minion(salt_cli, salt_call_cli, salt_minion, salt_ # Make sure the minion is running for count in range(0, 30): if salt_minion.is_running(): + print( + f"DGM test_salt_user_minion, salt_minion is running, final count '{count}'", + flush=True, + ) break else: time.sleep(2) @@ -76,6 +80,10 @@ def test_salt_versions_report_minion(salt_cli, salt_call_cli, salt_minion, salt_ # Make sure the master is running for count in range(0, 30): if salt_master.is_running(): + print( + f"DGM test_salt_user_minion, salt_master is running, final count '{count}'", + flush=True, + ) break else: time.sleep(2) @@ -85,11 +93,16 @@ def test_salt_versions_report_minion(salt_cli, salt_call_cli, salt_minion, salt_ flush=True, ) assert salt_master.is_running() + + ret = salt_call_cli.run("--local", "test.ping") + print( + f"DGM test_salt_user_minion, salt_call local test.ping ret '{ret}'", flush=True + ) # DGM # Make sure we can ping the minion ... ret = salt_cli.run( - "--timeout=300", "test.ping", minion_tgt=salt_minion.id, _timeout=300 + "--timeout=600", "test.ping", minion_tgt=salt_minion.id, _timeout=600 ) print(f"DGM test_salt_user_minion, test.ping ret '{ret}'", flush=True) @@ -135,8 +148,7 @@ def test_compare_versions(binary, install_salt): ) -@pytest.mark.skip_on_windows -@pytest.mark.skip_on_darwin +@pytest.mark.skip_unless_on_darwin @pytest.mark.parametrize( "symlink", [ From 4ee0f9410b23d787d75e9ab1b39f393c349d89d9 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 27 Aug 2024 09:45:46 -0600 Subject: [PATCH 11/44] Rearrangment of test parameters --- tests/pytests/pkg/integration/test_salt_api.py | 2 +- tests/pytests/pkg/integration/test_salt_ufw.py | 4 ++-- tests/pytests/pkg/integration/test_salt_user.py | 10 +++++++--- tests/pytests/pkg/integration/test_version.py | 2 +- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/pytests/pkg/integration/test_salt_api.py b/tests/pytests/pkg/integration/test_salt_api.py index f01e7c5fd7c2..ce9ffce8656e 100644 --- a/tests/pytests/pkg/integration/test_salt_api.py +++ b/tests/pytests/pkg/integration/test_salt_api.py @@ -5,7 +5,7 @@ ] -def test_salt_api(api_request, salt_master, install_salt): +def test_salt_api(api_request, install_salt, salt_master): """ Test running a command against the salt api """ diff --git a/tests/pytests/pkg/integration/test_salt_ufw.py b/tests/pytests/pkg/integration/test_salt_ufw.py index 0e0471aebf24..6c86e0a3339f 100644 --- a/tests/pytests/pkg/integration/test_salt_ufw.py +++ b/tests/pytests/pkg/integration/test_salt_ufw.py @@ -9,8 +9,8 @@ @pytest.fixture def salt_systemd_setup( - salt_call_cli, install_salt, + salt_call_cli, ): """ Fixture to set systemd for salt packages to enabled and active @@ -31,7 +31,7 @@ def salt_systemd_setup( @pytest.mark.skip_if_binaries_missing("ufw") -def test_salt_ufw(salt_systemd_setup, salt_call_cli, install_salt): +def test_salt_ufw(salt_systemd_setup, install_salt, salt_call_cli): """ Test salt.ufw for Debian/Ubuntu salt-master """ diff --git a/tests/pytests/pkg/integration/test_salt_user.py b/tests/pytests/pkg/integration/test_salt_user.py index 07c29837e901..70a08742970a 100644 --- a/tests/pytests/pkg/integration/test_salt_user.py +++ b/tests/pytests/pkg/integration/test_salt_user.py @@ -16,8 +16,8 @@ @pytest.fixture def salt_systemd_setup( - salt_call_cli, install_salt, + salt_call_cli, ): """ Fixture to set systemd for salt packages to enabled and active @@ -88,12 +88,16 @@ def pkg_paths_salt_user_exclusions(): return paths -def test_salt_user_master(salt_master, install_salt): +def test_salt_user_master(install_salt, salt_master): """ Test the correct user is running the Salt Master """ for count in range(0, 30): if salt_master.is_running(): + print( + f"DGM test_salt_user_master, salt_master is_running, count '{count}'", + flush=True, + ) break else: time.sleep(2) @@ -221,10 +225,10 @@ def test_pkg_paths( @pytest.mark.skip_if_binaries_missing("logrotate") def test_paths_log_rotation( + install_salt, salt_master, salt_minion, salt_call_cli, - install_salt, pkg_tests_account, ): """ diff --git a/tests/pytests/pkg/integration/test_version.py b/tests/pytests/pkg/integration/test_version.py index 1402a4ef79fd..01225dd2e6be 100644 --- a/tests/pytests/pkg/integration/test_version.py +++ b/tests/pytests/pkg/integration/test_version.py @@ -55,7 +55,7 @@ def test_salt_versions_report_master(install_salt): @pytest.mark.skip_on_windows -def test_salt_versions_report_minion(salt_cli, salt_call_cli, salt_minion, salt_master): +def test_salt_versions_report_minion(salt_cli, salt_call_cli, salt_master, salt_minion): """ Test running test.versions_report on minion """ From 770def921331000b5aaacbe8b821ad6311f27ca9 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 28 Aug 2024 11:34:13 -0600 Subject: [PATCH 12/44] Ensure salt_master is left running after test stopped it --- tests/pytests/pkg/integration/test_salt_user.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/pytests/pkg/integration/test_salt_user.py b/tests/pytests/pkg/integration/test_salt_user.py index 70a08742970a..bb79e9f5293f 100644 --- a/tests/pytests/pkg/integration/test_salt_user.py +++ b/tests/pytests/pkg/integration/test_salt_user.py @@ -420,3 +420,7 @@ def test_paths_log_rotation( bkup_count += 1 assert ret.returncode == 0 + + # ensure leave salt_master running + salt_master.start() + assert salt_master.is_running() is True From ba558deaa869471ceaee65aee6eb1730cbc9b92e Mon Sep 17 00:00:00 2001 From: David Murphy Date: Thu, 29 Aug 2024 14:08:56 -0600 Subject: [PATCH 13/44] Altered debugging --- .../pytests/pkg/integration/test_salt_user.py | 46 ++++++++++----- tests/pytests/pkg/integration/test_version.py | 59 ++++++++++--------- 2 files changed, 61 insertions(+), 44 deletions(-) diff --git a/tests/pytests/pkg/integration/test_salt_user.py b/tests/pytests/pkg/integration/test_salt_user.py index bb79e9f5293f..99228d743049 100644 --- a/tests/pytests/pkg/integration/test_salt_user.py +++ b/tests/pytests/pkg/integration/test_salt_user.py @@ -2,13 +2,15 @@ import pathlib import subprocess import sys -import time import packaging.version import psutil import pytest from saltfactories.utils.tempfiles import temp_directory +## DGM import time + + pytestmark = [ pytest.mark.skip_unless_on_linux, ] @@ -92,20 +94,20 @@ def test_salt_user_master(install_salt, salt_master): """ Test the correct user is running the Salt Master """ - for count in range(0, 30): - if salt_master.is_running(): - print( - f"DGM test_salt_user_master, salt_master is_running, count '{count}'", - flush=True, - ) - break - else: - time.sleep(2) - - print( - f"DGM test_salt_user_master, salt_master '{salt_master}' and is_running '{salt_master.is_running()}'", - flush=True, - ) + ## DGM for count in range(0, 30): + ## DGM if salt_master.is_running(): + ## DGM print( + ## DGM f"DGM test_salt_user_master, salt_master is_running, count '{count}'", + ## DGM flush=True, + ## DGM ) + ## DGM break + ## DGM else: + ## DGM time.sleep(2) + + ## DGM print( + ## DGM f"DGM test_salt_user_master, salt_master '{salt_master}' and is_running '{salt_master.is_running()}'", + ## DGM flush=True, + ## DGM ) assert salt_master.is_running() match = False @@ -193,6 +195,12 @@ def test_pkg_paths( assert pkg_path.exists() for dirpath, sub_dirs, files in os.walk(pkg_path): path = pathlib.Path(dirpath) + + print( + f"DGM test_pkg_paths, dirpath '{str(dirpath)}', path '{str(path)}', path owner '{path.owner()}', path group '{path.group()}'", + flush=True, + ) + # Directories owned by salt:salt or their subdirs/files if ( str(path) in pkg_paths_salt_user or str(path) in salt_user_subdirs @@ -206,6 +214,10 @@ def test_pkg_paths( for file in files: file_path = path.joinpath(file) if str(file_path) not in pkg_paths_salt_user_exclusions: + print( + f"DGM test_pkg_paths, salt:salt file_path '{str(file_path)}', file_path owner '{file_path.owner()}', file_path group '{file_path.group()}'", + flush=True, + ) assert file_path.owner() == "salt" # Directories owned by root:root else: @@ -216,6 +228,10 @@ def test_pkg_paths( continue file_path = path.joinpath(file) # Individual files owned by salt user + print( + f"DGM test_pkg_paths, root:root file_path '{str(file_path)}', file_path owner '{file_path.owner()}', file_path group '{file_path.group()}'", + flush=True, + ) if str(file_path) in pkg_paths_salt_user: assert file_path.owner() == "salt" else: diff --git a/tests/pytests/pkg/integration/test_version.py b/tests/pytests/pkg/integration/test_version.py index 01225dd2e6be..ff78ee080b63 100644 --- a/tests/pytests/pkg/integration/test_version.py +++ b/tests/pytests/pkg/integration/test_version.py @@ -1,11 +1,12 @@ import os.path import pathlib import subprocess -import time import pytest from pytestskipmarkers.utils import platform +## DGM import time + @pytest.mark.skip_on_windows def test_salt_version(version, install_salt): @@ -60,38 +61,38 @@ def test_salt_versions_report_minion(salt_cli, salt_call_cli, salt_master, salt_ Test running test.versions_report on minion """ # Make sure the minion is running - for count in range(0, 30): - if salt_minion.is_running(): - print( - f"DGM test_salt_user_minion, salt_minion is running, final count '{count}'", - flush=True, - ) - break - else: - time.sleep(2) - - print( - f"DGM test_salt_user_minion, salt_minion '{salt_minion}' and is_running '{salt_minion.is_running()}'", - flush=True, - ) + ## DGM for count in range(0, 30): + ## DGM if salt_minion.is_running(): + ## DGM print( + ## DGM f"DGM test_salt_user_minion, salt_minion is running, final count '{count}'", + ## DGM flush=True, + ## DGM ) + ## DGM break + ## DGM else: + ## DGM time.sleep(2) + + ## DGM print( + ## DGM f"DGM test_salt_user_minion, salt_minion '{salt_minion}' and is_running '{salt_minion.is_running()}'", + ## DGM flush=True, + ## DGM ) assert salt_minion.is_running() # DGM # Make sure the master is running - for count in range(0, 30): - if salt_master.is_running(): - print( - f"DGM test_salt_user_minion, salt_master is running, final count '{count}'", - flush=True, - ) - break - else: - time.sleep(2) - - print( - f"DGM test_salt_user_minion, salt_master '{salt_master}' and is_running '{salt_master.is_running()}'", - flush=True, - ) + ## DGM for count in range(0, 30): + ## DGM if salt_master.is_running(): + ## DGM print( + ## DGM f"DGM test_salt_user_minion, salt_master is running, final count '{count}'", + ## DGM flush=True, + ## DGM ) + ## DGM break + ## DGM else: + ## DGM time.sleep(2) + + ## DGM print( + ## DGM f"DGM test_salt_user_minion, salt_master '{salt_master}' and is_running '{salt_master.is_running()}'", + ## DGM flush=True, + ## DGM ) assert salt_master.is_running() ret = salt_call_cli.run("--local", "test.ping") From 067ee3ca04c2b9f8f9c50e29e970537b78e70135 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 3 Sep 2024 16:30:53 -0600 Subject: [PATCH 14/44] Further debugging --- .../pytests/pkg/integration/test_salt_user.py | 69 +++++++++++++------ tests/pytests/pkg/integration/test_version.py | 59 ++++++++-------- 2 files changed, 76 insertions(+), 52 deletions(-) diff --git a/tests/pytests/pkg/integration/test_salt_user.py b/tests/pytests/pkg/integration/test_salt_user.py index 99228d743049..023da47d24ac 100644 --- a/tests/pytests/pkg/integration/test_salt_user.py +++ b/tests/pytests/pkg/integration/test_salt_user.py @@ -2,15 +2,13 @@ import pathlib import subprocess import sys +import time import packaging.version import psutil import pytest from saltfactories.utils.tempfiles import temp_directory -## DGM import time - - pytestmark = [ pytest.mark.skip_unless_on_linux, ] @@ -94,20 +92,20 @@ def test_salt_user_master(install_salt, salt_master): """ Test the correct user is running the Salt Master """ - ## DGM for count in range(0, 30): - ## DGM if salt_master.is_running(): - ## DGM print( - ## DGM f"DGM test_salt_user_master, salt_master is_running, count '{count}'", - ## DGM flush=True, - ## DGM ) - ## DGM break - ## DGM else: - ## DGM time.sleep(2) - - ## DGM print( - ## DGM f"DGM test_salt_user_master, salt_master '{salt_master}' and is_running '{salt_master.is_running()}'", - ## DGM flush=True, - ## DGM ) + for count in range(0, 30): + if salt_master.is_running(): + print( + f"DGM test_salt_user_master, salt_master is_running, count '{count}'", + flush=True, + ) + break + else: + time.sleep(2) + + print( + f"DGM test_salt_user_master, salt_master '{salt_master}' and is_running '{salt_master.is_running()}'", + flush=True, + ) assert salt_master.is_running() match = False @@ -179,6 +177,7 @@ def test_pkg_paths( pkg_paths, pkg_paths_salt_user, pkg_paths_salt_user_exclusions, + salt_call_cli, ): """ Test package paths ownership @@ -190,6 +189,32 @@ def test_pkg_paths( salt_user_subdirs = [] + print( + f"DGM test_pkg_paths, pkg_paths '{pkg_paths}', pkg_paths_salt_user '{pkg_paths_salt_user}', exclusions '{pkg_paths_salt_user_exclusions}'", + flush=True, + ) + + dgm_cmd = "ps -ef" + ret = salt_call_cli("--local", "cmd.run", dgm_cmd) + print( + f"DGM test_pkg_paths, test ps -ef, ret '{ret}'", + flush=True, + ) + + dgm_cmd = "ls -al /var/log/" + ret = salt_call_cli("--local", "cmd.run", dgm_cmd) + print( + f"DGM test_pkg_paths, test ls -al /var/log/, ret '{ret}'", + flush=True, + ) + + dgm_cmd = "ls -al /var/log/salt" + ret = salt_call_cli("--local", "cmd.run", dgm_cmd) + print( + f"DGM test_pkg_paths, test ls -al /var/log/salt, ret '{ret}'", + flush=True, + ) + for _path in pkg_paths: pkg_path = pathlib.Path(_path) assert pkg_path.exists() @@ -197,7 +222,7 @@ def test_pkg_paths( path = pathlib.Path(dirpath) print( - f"DGM test_pkg_paths, dirpath '{str(dirpath)}', path '{str(path)}', path owner '{path.owner()}', path group '{path.group()}'", + f"DGM test_pkg_paths, dirpath '{str(dirpath)}', path '{str(path)}', path owner '{path.owner()}', path group '{path.group()}', salt_user_subdirs '{salt_user_subdirs}', sub_dirs '{sub_dirs}', files '{files}'", flush=True, ) @@ -213,11 +238,11 @@ def test_pkg_paths( # Individual files owned by salt user for file in files: file_path = path.joinpath(file) + print( + f"DGM test_pkg_paths, salt:salt file_path '{str(file_path)}', file_path owner '{file_path.owner()}', file_path group '{file_path.group()}'", + flush=True, + ) if str(file_path) not in pkg_paths_salt_user_exclusions: - print( - f"DGM test_pkg_paths, salt:salt file_path '{str(file_path)}', file_path owner '{file_path.owner()}', file_path group '{file_path.group()}'", - flush=True, - ) assert file_path.owner() == "salt" # Directories owned by root:root else: diff --git a/tests/pytests/pkg/integration/test_version.py b/tests/pytests/pkg/integration/test_version.py index ff78ee080b63..01225dd2e6be 100644 --- a/tests/pytests/pkg/integration/test_version.py +++ b/tests/pytests/pkg/integration/test_version.py @@ -1,12 +1,11 @@ import os.path import pathlib import subprocess +import time import pytest from pytestskipmarkers.utils import platform -## DGM import time - @pytest.mark.skip_on_windows def test_salt_version(version, install_salt): @@ -61,38 +60,38 @@ def test_salt_versions_report_minion(salt_cli, salt_call_cli, salt_master, salt_ Test running test.versions_report on minion """ # Make sure the minion is running - ## DGM for count in range(0, 30): - ## DGM if salt_minion.is_running(): - ## DGM print( - ## DGM f"DGM test_salt_user_minion, salt_minion is running, final count '{count}'", - ## DGM flush=True, - ## DGM ) - ## DGM break - ## DGM else: - ## DGM time.sleep(2) - - ## DGM print( - ## DGM f"DGM test_salt_user_minion, salt_minion '{salt_minion}' and is_running '{salt_minion.is_running()}'", - ## DGM flush=True, - ## DGM ) + for count in range(0, 30): + if salt_minion.is_running(): + print( + f"DGM test_salt_user_minion, salt_minion is running, final count '{count}'", + flush=True, + ) + break + else: + time.sleep(2) + + print( + f"DGM test_salt_user_minion, salt_minion '{salt_minion}' and is_running '{salt_minion.is_running()}'", + flush=True, + ) assert salt_minion.is_running() # DGM # Make sure the master is running - ## DGM for count in range(0, 30): - ## DGM if salt_master.is_running(): - ## DGM print( - ## DGM f"DGM test_salt_user_minion, salt_master is running, final count '{count}'", - ## DGM flush=True, - ## DGM ) - ## DGM break - ## DGM else: - ## DGM time.sleep(2) - - ## DGM print( - ## DGM f"DGM test_salt_user_minion, salt_master '{salt_master}' and is_running '{salt_master.is_running()}'", - ## DGM flush=True, - ## DGM ) + for count in range(0, 30): + if salt_master.is_running(): + print( + f"DGM test_salt_user_minion, salt_master is running, final count '{count}'", + flush=True, + ) + break + else: + time.sleep(2) + + print( + f"DGM test_salt_user_minion, salt_master '{salt_master}' and is_running '{salt_master.is_running()}'", + flush=True, + ) assert salt_master.is_running() ret = salt_call_cli.run("--local", "test.ping") From 2bd4e000a8b3632ee7a05aa38527e1e5557aa996 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 4 Sep 2024 10:44:19 -0600 Subject: [PATCH 15/44] Fix typo --- tests/pytests/pkg/integration/test_salt_user.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/pytests/pkg/integration/test_salt_user.py b/tests/pytests/pkg/integration/test_salt_user.py index 023da47d24ac..7f9a21e90e33 100644 --- a/tests/pytests/pkg/integration/test_salt_user.py +++ b/tests/pytests/pkg/integration/test_salt_user.py @@ -195,21 +195,21 @@ def test_pkg_paths( ) dgm_cmd = "ps -ef" - ret = salt_call_cli("--local", "cmd.run", dgm_cmd) + ret = salt_call_cli.run("--local", "cmd.run", dgm_cmd) print( f"DGM test_pkg_paths, test ps -ef, ret '{ret}'", flush=True, ) dgm_cmd = "ls -al /var/log/" - ret = salt_call_cli("--local", "cmd.run", dgm_cmd) + ret = salt_call_cli.run("--local", "cmd.run", dgm_cmd) print( f"DGM test_pkg_paths, test ls -al /var/log/, ret '{ret}'", flush=True, ) dgm_cmd = "ls -al /var/log/salt" - ret = salt_call_cli("--local", "cmd.run", dgm_cmd) + ret = salt_call_cli.run("--local", "cmd.run", dgm_cmd) print( f"DGM test_pkg_paths, test ls -al /var/log/salt, ret '{ret}'", flush=True, From eada163968045edece507e3cca1ac528a38cc154 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 4 Sep 2024 14:52:35 -0600 Subject: [PATCH 16/44] Commented out debugging print statements --- salt/config/__init__.py | 6 +- .../pytests/pkg/integration/test_salt_user.py | 90 +++++++++---------- tests/pytests/pkg/integration/test_version.py | 44 ++++----- 3 files changed, 70 insertions(+), 70 deletions(-) diff --git a/salt/config/__init__.py b/salt/config/__init__.py index cffce8929f6a..e9e033d656ce 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -2514,9 +2514,9 @@ def syndic_config( "cachedir": master_opts["cachedir"], } opts.update(syndic_opts) - log.warning("DGM syndic_config, user is set to '%s'", opts["user"]) - dgm_user = opts["user"] - print(f"DGM syndic_config, user is set to '{dgm_user}'", flush=True) + ## DGM log.warning("DGM syndic_config, user is set to '%s'", opts["user"]) + ## DGM dgm_user = opts["user"] + ## DGM print(f"DGM syndic_config, user is set to '{dgm_user}'", flush=True) # Prepend root_dir to other paths prepend_root_dirs = [ diff --git a/tests/pytests/pkg/integration/test_salt_user.py b/tests/pytests/pkg/integration/test_salt_user.py index 7f9a21e90e33..a8bb166f068e 100644 --- a/tests/pytests/pkg/integration/test_salt_user.py +++ b/tests/pytests/pkg/integration/test_salt_user.py @@ -94,18 +94,18 @@ def test_salt_user_master(install_salt, salt_master): """ for count in range(0, 30): if salt_master.is_running(): - print( - f"DGM test_salt_user_master, salt_master is_running, count '{count}'", - flush=True, - ) + ## DGM print( + ## DGM f"DGM test_salt_user_master, salt_master is_running, count '{count}'", + ## DGM flush=True, + ## DGM ) break else: time.sleep(2) - print( - f"DGM test_salt_user_master, salt_master '{salt_master}' and is_running '{salt_master.is_running()}'", - flush=True, - ) + ## DGM print( + ## DGM f"DGM test_salt_user_master, salt_master '{salt_master}' and is_running '{salt_master.is_running()}'", + ## DGM flush=True, + ## DGM ) assert salt_master.is_running() match = False @@ -189,31 +189,31 @@ def test_pkg_paths( salt_user_subdirs = [] - print( - f"DGM test_pkg_paths, pkg_paths '{pkg_paths}', pkg_paths_salt_user '{pkg_paths_salt_user}', exclusions '{pkg_paths_salt_user_exclusions}'", - flush=True, - ) - - dgm_cmd = "ps -ef" - ret = salt_call_cli.run("--local", "cmd.run", dgm_cmd) - print( - f"DGM test_pkg_paths, test ps -ef, ret '{ret}'", - flush=True, - ) - - dgm_cmd = "ls -al /var/log/" - ret = salt_call_cli.run("--local", "cmd.run", dgm_cmd) - print( - f"DGM test_pkg_paths, test ls -al /var/log/, ret '{ret}'", - flush=True, - ) - - dgm_cmd = "ls -al /var/log/salt" - ret = salt_call_cli.run("--local", "cmd.run", dgm_cmd) - print( - f"DGM test_pkg_paths, test ls -al /var/log/salt, ret '{ret}'", - flush=True, - ) + ## DGM print( + ## DGM f"DGM test_pkg_paths, pkg_paths '{pkg_paths}', pkg_paths_salt_user '{pkg_paths_salt_user}', exclusions '{pkg_paths_salt_user_exclusions}'", + ## DGM flush=True, + ## DGM ) + + ## DGM dgm_cmd = "ps -ef" + ## DGM ret = salt_call_cli.run("--local", "cmd.run", dgm_cmd) + ## DGM print( + ## DGM f"DGM test_pkg_paths, test ps -ef, ret '{ret}'", + ## DGM flush=True, + ## DGM ) + + ## DGM dgm_cmd = "ls -al /var/log/" + ## DGM ret = salt_call_cli.run("--local", "cmd.run", dgm_cmd) + ## DGM print( + ## DGM f"DGM test_pkg_paths, test ls -al /var/log/, ret '{ret}'", + ## DGM flush=True, + ## DGM ) + + ## DGM dgm_cmd = "ls -al /var/log/salt" + ## DGM ret = salt_call_cli.run("--local", "cmd.run", dgm_cmd) + ## DGM print( + ## DGM f"DGM test_pkg_paths, test ls -al /var/log/salt, ret '{ret}'", + ## DGM flush=True, + ## DGM ) for _path in pkg_paths: pkg_path = pathlib.Path(_path) @@ -221,10 +221,10 @@ def test_pkg_paths( for dirpath, sub_dirs, files in os.walk(pkg_path): path = pathlib.Path(dirpath) - print( - f"DGM test_pkg_paths, dirpath '{str(dirpath)}', path '{str(path)}', path owner '{path.owner()}', path group '{path.group()}', salt_user_subdirs '{salt_user_subdirs}', sub_dirs '{sub_dirs}', files '{files}'", - flush=True, - ) + ## DGM print( + ## DGM f"DGM test_pkg_paths, dirpath '{str(dirpath)}', path '{str(path)}', path owner '{path.owner()}', path group '{path.group()}', salt_user_subdirs '{salt_user_subdirs}', sub_dirs '{sub_dirs}', files '{files}'", + ## DGM flush=True, + ## DGM ) # Directories owned by salt:salt or their subdirs/files if ( @@ -238,10 +238,10 @@ def test_pkg_paths( # Individual files owned by salt user for file in files: file_path = path.joinpath(file) - print( - f"DGM test_pkg_paths, salt:salt file_path '{str(file_path)}', file_path owner '{file_path.owner()}', file_path group '{file_path.group()}'", - flush=True, - ) + ## DGM print( + ## DGM f"DGM test_pkg_paths, salt:salt file_path '{str(file_path)}', file_path owner '{file_path.owner()}', file_path group '{file_path.group()}'", + ## DGM flush=True, + ## DGM ) if str(file_path) not in pkg_paths_salt_user_exclusions: assert file_path.owner() == "salt" # Directories owned by root:root @@ -253,10 +253,10 @@ def test_pkg_paths( continue file_path = path.joinpath(file) # Individual files owned by salt user - print( - f"DGM test_pkg_paths, root:root file_path '{str(file_path)}', file_path owner '{file_path.owner()}', file_path group '{file_path.group()}'", - flush=True, - ) + ## DGM print( + ## DGM f"DGM test_pkg_paths, root:root file_path '{str(file_path)}', file_path owner '{file_path.owner()}', file_path group '{file_path.group()}'", + ## DGM flush=True, + ## DGM ) if str(file_path) in pkg_paths_salt_user: assert file_path.owner() == "salt" else: diff --git a/tests/pytests/pkg/integration/test_version.py b/tests/pytests/pkg/integration/test_version.py index 01225dd2e6be..1c29d033c77c 100644 --- a/tests/pytests/pkg/integration/test_version.py +++ b/tests/pytests/pkg/integration/test_version.py @@ -62,49 +62,49 @@ def test_salt_versions_report_minion(salt_cli, salt_call_cli, salt_master, salt_ # Make sure the minion is running for count in range(0, 30): if salt_minion.is_running(): - print( - f"DGM test_salt_user_minion, salt_minion is running, final count '{count}'", - flush=True, - ) + ## DGM print( + ## DGM f"DGM test_salt_user_minion, salt_minion is running, final count '{count}'", + ## DGM flush=True, + ## DGM ) break else: time.sleep(2) - print( - f"DGM test_salt_user_minion, salt_minion '{salt_minion}' and is_running '{salt_minion.is_running()}'", - flush=True, - ) + ## DGM print( + ## DGM f"DGM test_salt_user_minion, salt_minion '{salt_minion}' and is_running '{salt_minion.is_running()}'", + ## DGM flush=True, + ## DGM ) assert salt_minion.is_running() # DGM # Make sure the master is running for count in range(0, 30): if salt_master.is_running(): - print( - f"DGM test_salt_user_minion, salt_master is running, final count '{count}'", - flush=True, - ) + ## DGM print( + ## DGM f"DGM test_salt_user_minion, salt_master is running, final count '{count}'", + ## DGM flush=True, + ## DGM ) break else: time.sleep(2) - print( - f"DGM test_salt_user_minion, salt_master '{salt_master}' and is_running '{salt_master.is_running()}'", - flush=True, - ) + ## DGM print( + ## DGM f"DGM test_salt_user_minion, salt_master '{salt_master}' and is_running '{salt_master.is_running()}'", + ## DGM flush=True, + ## DGM ) assert salt_master.is_running() - ret = salt_call_cli.run("--local", "test.ping") - print( - f"DGM test_salt_user_minion, salt_call local test.ping ret '{ret}'", flush=True - ) - # DGM + ## DGM ret = salt_call_cli.run("--local", "test.ping") + ## DGM print( + ## DGM f"DGM test_salt_user_minion, salt_call local test.ping ret '{ret}'", flush=True + ## DGM ) + ## DGM # DGM # Make sure we can ping the minion ... ret = salt_cli.run( "--timeout=600", "test.ping", minion_tgt=salt_minion.id, _timeout=600 ) - print(f"DGM test_salt_user_minion, test.ping ret '{ret}'", flush=True) + ## DGM print(f"DGM test_salt_user_minion, test.ping ret '{ret}'", flush=True) assert ret.returncode == 0 assert ret.data is True From 1ebd3c6b8ebfcd5a3f8e8bdb49f34bc6b0d4fb2a Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 6 Sep 2024 09:54:10 -0600 Subject: [PATCH 17/44] Forced rebuild --- salt/config/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/config/__init__.py b/salt/config/__init__.py index e9e033d656ce..cd03a3ab745b 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -2502,6 +2502,7 @@ def syndic_config( ), ) ), + ## DGM line to force rebuild # DGM conf/suse/master has syndic_user for running syndic as a different user # DGM use urrently set it to 'salt' # DGM testing getting the master_opts user, and run syndic as that From a7da90d8f675925f87e2fab26b034bfc795e5110 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 6 Sep 2024 15:59:14 -0600 Subject: [PATCH 18/44] Removed commented out debug print statements --- pkg/common/salt-api.service | 1 - salt/config/__init__.py | 8 ---- salt/minion.py | 21 --------- .../pytests/pkg/integration/test_salt_api.py | 1 - .../pytests/pkg/integration/test_salt_user.py | 47 ------------------- tests/pytests/pkg/integration/test_version.py | 24 ---------- 6 files changed, 102 deletions(-) diff --git a/pkg/common/salt-api.service b/pkg/common/salt-api.service index 3e47a2e6366a..d0b6d741202b 100644 --- a/pkg/common/salt-api.service +++ b/pkg/common/salt-api.service @@ -2,7 +2,6 @@ Description=The Salt API Documentation=man:salt-api(1) file:///usr/share/doc/salt/html/contents.html https://docs.saltproject.io/en/latest/contents.html After=network.target -# DGM PartOf=salt-master.service [Service] Type=notify diff --git a/salt/config/__init__.py b/salt/config/__init__.py index cd03a3ab745b..8226ba452b80 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -2502,11 +2502,6 @@ def syndic_config( ), ) ), - ## DGM line to force rebuild - # DGM conf/suse/master has syndic_user for running syndic as a different user - # DGM use urrently set it to 'salt' - # DGM testing getting the master_opts user, and run syndic as that - # DGM "user": opts.get("syndic_user", opts["user"]), "user": opts.get("syndic_user", master_opts["user"]), "sock_dir": os.path.join( opts["cachedir"], opts.get("syndic_sock_dir", opts["sock_dir"]) @@ -2515,9 +2510,6 @@ def syndic_config( "cachedir": master_opts["cachedir"], } opts.update(syndic_opts) - ## DGM log.warning("DGM syndic_config, user is set to '%s'", opts["user"]) - ## DGM dgm_user = opts["user"] - ## DGM print(f"DGM syndic_config, user is set to '{dgm_user}'", flush=True) # Prepend root_dir to other paths prepend_root_dirs = [ diff --git a/salt/minion.py b/salt/minion.py index 64057e81fccc..2c2585637420 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -280,11 +280,6 @@ def get_proc_dir(cachedir, **kwargs): gid. Must be int. Works only on unix/unix like systems. """ # pylint: disable=logging-fstring-interpolation - ## DGM log.warning(f"DGM get_proc_dir entry, cachedir '{cachedir}', kwargs '{kwargs}'") - ## DGM print( - ## DGM f"DGM get_proc_dir entry, cachedir '{cachedir}', kwargs '{kwargs}'", flush=True - ## DGM ) - fn_ = os.path.join(cachedir, "proc") mode = kwargs.pop("mode", None) @@ -312,28 +307,12 @@ def get_proc_dir(cachedir, **kwargs): gid = kwargs.pop("gid", -1) # pylint: disable=logging-fstring-interpolation - ## DGM log.warning( - ## DGM f"DGM get_proc_dir chown, d_stat '{d_stat}', uid '{uid}', gid '{gid}'" - ## DGM ) - ## DGM print( - ## DGM f"DGM get_proc_dir chown, d_stat '{d_stat}', uid '{uid}', gid '{gid}'", - ## DGM flush=True, - ## DGM ) - # if uid and gid are both -1 then go ahead with # no changes at all if (d_stat.st_uid != uid or d_stat.st_gid != gid) and [ i for i in (uid, gid) if i != -1 ]: # pylint: disable=logging-fstring-interpolation - ## DGM log.warning( - ## DGM f"DGM get_proc_dir chown file, file '{fn_}' , uid '{uid}', gid '{gid}'" - ## DGM ) - ## DGM print( - ## DGM f"DGM get_proc_dir chown file, file '{fn_}' , uid '{uid}', gid '{gid}'", - ## DGM flush=True, - ## DGM ) - os.chown(fn_, uid, gid) return fn_ diff --git a/tests/pytests/pkg/integration/test_salt_api.py b/tests/pytests/pkg/integration/test_salt_api.py index ce9ffce8656e..b13775bd7942 100644 --- a/tests/pytests/pkg/integration/test_salt_api.py +++ b/tests/pytests/pkg/integration/test_salt_api.py @@ -9,7 +9,6 @@ def test_salt_api(api_request, install_salt, salt_master): """ Test running a command against the salt api """ - # DGM TBD 66672 is merged, need to remove this check and see if test passes if install_salt.distro_id in ("ubuntu", "debian"): pytest.skip( "Package test are getting reworked in https://github.com/saltstack/salt/issues/66672" diff --git a/tests/pytests/pkg/integration/test_salt_user.py b/tests/pytests/pkg/integration/test_salt_user.py index a8bb166f068e..3978bfe9ca75 100644 --- a/tests/pytests/pkg/integration/test_salt_user.py +++ b/tests/pytests/pkg/integration/test_salt_user.py @@ -94,18 +94,10 @@ def test_salt_user_master(install_salt, salt_master): """ for count in range(0, 30): if salt_master.is_running(): - ## DGM print( - ## DGM f"DGM test_salt_user_master, salt_master is_running, count '{count}'", - ## DGM flush=True, - ## DGM ) break else: time.sleep(2) - ## DGM print( - ## DGM f"DGM test_salt_user_master, salt_master '{salt_master}' and is_running '{salt_master.is_running()}'", - ## DGM flush=True, - ## DGM ) assert salt_master.is_running() match = False @@ -189,43 +181,12 @@ def test_pkg_paths( salt_user_subdirs = [] - ## DGM print( - ## DGM f"DGM test_pkg_paths, pkg_paths '{pkg_paths}', pkg_paths_salt_user '{pkg_paths_salt_user}', exclusions '{pkg_paths_salt_user_exclusions}'", - ## DGM flush=True, - ## DGM ) - - ## DGM dgm_cmd = "ps -ef" - ## DGM ret = salt_call_cli.run("--local", "cmd.run", dgm_cmd) - ## DGM print( - ## DGM f"DGM test_pkg_paths, test ps -ef, ret '{ret}'", - ## DGM flush=True, - ## DGM ) - - ## DGM dgm_cmd = "ls -al /var/log/" - ## DGM ret = salt_call_cli.run("--local", "cmd.run", dgm_cmd) - ## DGM print( - ## DGM f"DGM test_pkg_paths, test ls -al /var/log/, ret '{ret}'", - ## DGM flush=True, - ## DGM ) - - ## DGM dgm_cmd = "ls -al /var/log/salt" - ## DGM ret = salt_call_cli.run("--local", "cmd.run", dgm_cmd) - ## DGM print( - ## DGM f"DGM test_pkg_paths, test ls -al /var/log/salt, ret '{ret}'", - ## DGM flush=True, - ## DGM ) - for _path in pkg_paths: pkg_path = pathlib.Path(_path) assert pkg_path.exists() for dirpath, sub_dirs, files in os.walk(pkg_path): path = pathlib.Path(dirpath) - ## DGM print( - ## DGM f"DGM test_pkg_paths, dirpath '{str(dirpath)}', path '{str(path)}', path owner '{path.owner()}', path group '{path.group()}', salt_user_subdirs '{salt_user_subdirs}', sub_dirs '{sub_dirs}', files '{files}'", - ## DGM flush=True, - ## DGM ) - # Directories owned by salt:salt or their subdirs/files if ( str(path) in pkg_paths_salt_user or str(path) in salt_user_subdirs @@ -238,10 +199,6 @@ def test_pkg_paths( # Individual files owned by salt user for file in files: file_path = path.joinpath(file) - ## DGM print( - ## DGM f"DGM test_pkg_paths, salt:salt file_path '{str(file_path)}', file_path owner '{file_path.owner()}', file_path group '{file_path.group()}'", - ## DGM flush=True, - ## DGM ) if str(file_path) not in pkg_paths_salt_user_exclusions: assert file_path.owner() == "salt" # Directories owned by root:root @@ -253,10 +210,6 @@ def test_pkg_paths( continue file_path = path.joinpath(file) # Individual files owned by salt user - ## DGM print( - ## DGM f"DGM test_pkg_paths, root:root file_path '{str(file_path)}', file_path owner '{file_path.owner()}', file_path group '{file_path.group()}'", - ## DGM flush=True, - ## DGM ) if str(file_path) in pkg_paths_salt_user: assert file_path.owner() == "salt" else: diff --git a/tests/pytests/pkg/integration/test_version.py b/tests/pytests/pkg/integration/test_version.py index 1c29d033c77c..b7fa262fd53b 100644 --- a/tests/pytests/pkg/integration/test_version.py +++ b/tests/pytests/pkg/integration/test_version.py @@ -62,49 +62,25 @@ def test_salt_versions_report_minion(salt_cli, salt_call_cli, salt_master, salt_ # Make sure the minion is running for count in range(0, 30): if salt_minion.is_running(): - ## DGM print( - ## DGM f"DGM test_salt_user_minion, salt_minion is running, final count '{count}'", - ## DGM flush=True, - ## DGM ) break else: time.sleep(2) - ## DGM print( - ## DGM f"DGM test_salt_user_minion, salt_minion '{salt_minion}' and is_running '{salt_minion.is_running()}'", - ## DGM flush=True, - ## DGM ) assert salt_minion.is_running() - # DGM # Make sure the master is running for count in range(0, 30): if salt_master.is_running(): - ## DGM print( - ## DGM f"DGM test_salt_user_minion, salt_master is running, final count '{count}'", - ## DGM flush=True, - ## DGM ) break else: time.sleep(2) - ## DGM print( - ## DGM f"DGM test_salt_user_minion, salt_master '{salt_master}' and is_running '{salt_master.is_running()}'", - ## DGM flush=True, - ## DGM ) assert salt_master.is_running() - ## DGM ret = salt_call_cli.run("--local", "test.ping") - ## DGM print( - ## DGM f"DGM test_salt_user_minion, salt_call local test.ping ret '{ret}'", flush=True - ## DGM ) - ## DGM # DGM - # Make sure we can ping the minion ... ret = salt_cli.run( "--timeout=600", "test.ping", minion_tgt=salt_minion.id, _timeout=600 ) - ## DGM print(f"DGM test_salt_user_minion, test.ping ret '{ret}'", flush=True) assert ret.returncode == 0 assert ret.data is True From c738a2f2ed2e8821e0e16a22af69cdd257fcd61d Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 4 Sep 2024 10:22:25 -0600 Subject: [PATCH 19/44] Move installer tests to workflow --- .github/workflows/templates/ci.yml.jinja | 13 +++++++++++++ .github/workflows/test-installer-action-windows.yml | 12 ++++++++---- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/.github/workflows/templates/ci.yml.jinja b/.github/workflows/templates/ci.yml.jinja index eef2e77ba846..01b2fadc2100 100644 --- a/.github/workflows/templates/ci.yml.jinja +++ b/.github/workflows/templates/ci.yml.jinja @@ -39,6 +39,19 @@ <%- endif %> + <%- set job_name = "test-windows-installer" %> + <%- if includes.get(job_name, True) %> + <{ job_name }>: + <%- do conclusion_needs.append(job_name) %> + name: Installer + if: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} + uses: ./.github/workflows/test-installer-action-windows.yml + needs: + - prepare-workflow + with: + changed-files: ${{ needs.prepare-workflow.outputs.changed-files }} + + <%- endif %> <%- set job_name = "prepare-release" %> <%- if includes.get(job_name, True) %> diff --git a/.github/workflows/test-installer-action-windows.yml b/.github/workflows/test-installer-action-windows.yml index cf0b48556bb9..e3f91661f05a 100644 --- a/.github/workflows/test-installer-action-windows.yml +++ b/.github/workflows/test-installer-action-windows.yml @@ -1,13 +1,17 @@ --- name: Test Windows Installer -on: pull_request - -permissions: - contents: read +on: + workflow_call: + inputs: + changed-files: + required: true + type: string + description: JSON string containing information about changed files jobs: Test-Windows-Installer: + name: Test Windows Installer logic runs-on: - windows-latest From fb2297e11cd714fe5911e5d00916a00be56d566b Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 4 Sep 2024 10:45:19 -0600 Subject: [PATCH 20/44] Update workflows --- .github/workflows/ci.yml | 12 ++++ .github/workflows/nightly.yml | 12 ++++ .github/workflows/nsis-tests.yml | 66 +++++++++++++++++++ .github/workflows/scheduled.yml | 12 ++++ .github/workflows/staging.yml | 12 ++++ .github/workflows/templates/ci.yml.jinja | 6 +- .github/workflows/templates/layout.yml.jinja | 3 + .../test-installer-action-windows.yml | 42 ------------ 8 files changed, 120 insertions(+), 45 deletions(-) create mode 100644 .github/workflows/nsis-tests.yml delete mode 100644 .github/workflows/test-installer-action-windows.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 207699290786..0b6de9cff239 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -127,6 +127,9 @@ jobs: - pkg/** - *pkg_requirements - *salt_added_modified + nsis_tests: + - added|modified: &nsis_tests + - pkg/windows/nsis/** testrun: - added|modified: - *pkg_requirements @@ -254,6 +257,14 @@ jobs: - prepare-workflow with: changed-files: ${{ needs.prepare-workflow.outputs.changed-files }} + nsis-tests: + name: NSIS Tests + if: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} + uses: ./.github/workflows/nsis-tests.yml + needs: + - prepare-workflow + with: + changed-files: ${{ needs.prepare-workflow.outputs.changed-files }} prepare-release: name: "Prepare Release: ${{ needs.prepare-workflow.outputs.salt-version }}" @@ -2131,6 +2142,7 @@ jobs: - prepare-workflow - pre-commit - lint + - nsis-tests - build-docs - build-deps-onedir - build-salt-onedir diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 93c4c09f55e8..18e390d8ab7f 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -176,6 +176,9 @@ jobs: - pkg/** - *pkg_requirements - *salt_added_modified + nsis_tests: + - added|modified: &nsis_tests + - pkg/windows/nsis/** testrun: - added|modified: - *pkg_requirements @@ -303,6 +306,14 @@ jobs: - prepare-workflow with: changed-files: ${{ needs.prepare-workflow.outputs.changed-files }} + nsis-tests: + name: NSIS Tests + if: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} + uses: ./.github/workflows/nsis-tests.yml + needs: + - prepare-workflow + with: + changed-files: ${{ needs.prepare-workflow.outputs.changed-files }} prepare-release: name: "Prepare Release: ${{ needs.prepare-workflow.outputs.salt-version }}" @@ -3026,6 +3037,7 @@ jobs: - prepare-workflow - pre-commit - lint + - nsis-tests - build-docs - build-deps-onedir - build-salt-onedir diff --git a/.github/workflows/nsis-tests.yml b/.github/workflows/nsis-tests.yml new file mode 100644 index 000000000000..958f3a253038 --- /dev/null +++ b/.github/workflows/nsis-tests.yml @@ -0,0 +1,66 @@ +--- +name: Test NSIS Installer + +on: + workflow_call: + inputs: + changed-files: + required: true + type: string + description: JSON string containing information about changed files + +jobs: + Test-NSIS-Logic: + name: Logic Tests + runs-on: + - windows-latest + + steps: + + - name: Checkout Salt + uses: actions/checkout@v4 + + - name: Set Up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install NSIS + run: .\pkg\windows\install_nsis.cmd -CICD + shell: cmd + + - name: Build Test Installer + run: .\pkg\windows\nsis\tests\setup.cmd -CICD + shell: cmd + + - name: Run Config Tests + run: .\pkg\windows\nsis\tests\test.cmd -CICD .\config_tests + shell: cmd + + Test-NSIS-Stress: + name: Stress Tests + runs-on: + - windows-latest + if: ${{ contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) || fromJSON(inputs.changed-files)['nsis_tests'] }} + + steps: + + - name: Checkout Salt + uses: actions/checkout@v4 + + - name: Set Up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install NSIS + run: .\pkg\windows\install_nsis.cmd -CICD + shell: cmd + + - name: Build Test Installer + run: .\pkg\windows\nsis\tests\setup.cmd -CICD + shell: cmd + + - name: Run Stress Test + run: .\pkg\windows\nsis\tests\test.cmd -CICD .\stress_tests + shell: cmd diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 2ab7dc11b6d1..767b8c17f46d 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -166,6 +166,9 @@ jobs: - pkg/** - *pkg_requirements - *salt_added_modified + nsis_tests: + - added|modified: &nsis_tests + - pkg/windows/nsis/** testrun: - added|modified: - *pkg_requirements @@ -293,6 +296,14 @@ jobs: - prepare-workflow with: changed-files: ${{ needs.prepare-workflow.outputs.changed-files }} + nsis-tests: + name: NSIS Tests + if: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} + uses: ./.github/workflows/nsis-tests.yml + needs: + - prepare-workflow + with: + changed-files: ${{ needs.prepare-workflow.outputs.changed-files }} prepare-release: name: "Prepare Release: ${{ needs.prepare-workflow.outputs.salt-version }}" @@ -2172,6 +2183,7 @@ jobs: - prepare-workflow - pre-commit - lint + - nsis-tests - build-docs - build-deps-onedir - build-salt-onedir diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 616582c55eca..2cf7786fe2c8 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -157,6 +157,9 @@ jobs: - pkg/** - *pkg_requirements - *salt_added_modified + nsis_tests: + - added|modified: &nsis_tests + - pkg/windows/nsis/** testrun: - added|modified: - *pkg_requirements @@ -293,6 +296,14 @@ jobs: - prepare-workflow with: changed-files: ${{ needs.prepare-workflow.outputs.changed-files }} + nsis-tests: + name: NSIS Tests + if: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} + uses: ./.github/workflows/nsis-tests.yml + needs: + - prepare-workflow + with: + changed-files: ${{ needs.prepare-workflow.outputs.changed-files }} prepare-release: name: "Prepare Release: ${{ needs.prepare-workflow.outputs.salt-version }}" @@ -3023,6 +3034,7 @@ jobs: - prepare-workflow - pre-commit - lint + - nsis-tests - build-docs - build-deps-onedir - build-salt-onedir diff --git a/.github/workflows/templates/ci.yml.jinja b/.github/workflows/templates/ci.yml.jinja index 01b2fadc2100..91713863f187 100644 --- a/.github/workflows/templates/ci.yml.jinja +++ b/.github/workflows/templates/ci.yml.jinja @@ -39,13 +39,13 @@ <%- endif %> - <%- set job_name = "test-windows-installer" %> + <%- set job_name = "nsis-tests" %> <%- if includes.get(job_name, True) %> <{ job_name }>: <%- do conclusion_needs.append(job_name) %> - name: Installer + name: NSIS Tests if: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} - uses: ./.github/workflows/test-installer-action-windows.yml + uses: ./.github/workflows/nsis-tests.yml needs: - prepare-workflow with: diff --git a/.github/workflows/templates/layout.yml.jinja b/.github/workflows/templates/layout.yml.jinja index 0f0e795c0c63..2259e26bff44 100644 --- a/.github/workflows/templates/layout.yml.jinja +++ b/.github/workflows/templates/layout.yml.jinja @@ -175,6 +175,9 @@ jobs: - pkg/** - *pkg_requirements - *salt_added_modified + nsis_tests: + - added|modified: &nsis_tests + - pkg/windows/nsis/** testrun: - added|modified: - *pkg_requirements diff --git a/.github/workflows/test-installer-action-windows.yml b/.github/workflows/test-installer-action-windows.yml deleted file mode 100644 index e3f91661f05a..000000000000 --- a/.github/workflows/test-installer-action-windows.yml +++ /dev/null @@ -1,42 +0,0 @@ ---- -name: Test Windows Installer - -on: - workflow_call: - inputs: - changed-files: - required: true - type: string - description: JSON string containing information about changed files - -jobs: - Test-Windows-Installer: - name: Test Windows Installer logic - runs-on: - - windows-latest - - steps: - - - name: Checkout Salt - uses: actions/checkout@v4 - - - name: Set Up Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: Install NSIS - run: .\pkg\windows\install_nsis.cmd -CICD - shell: cmd - - - name: Build Test Installer - run: .\pkg\windows\nsis\tests\setup.cmd -CICD - shell: cmd - - - name: Run Stress Test - run: .\pkg\windows\nsis\tests\test.cmd -CICD .\stress_tests - shell: cmd - - - name: Run Config Tests - run: .\pkg\windows\nsis\tests\test.cmd -CICD .\config_tests - shell: cmd From 676945eec7eaca3e22a960ddd7477d43b2b91994 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 4 Sep 2024 11:50:44 -0600 Subject: [PATCH 21/44] Only run tests when nsis files are modified --- .github/workflows/nsis-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/nsis-tests.yml b/.github/workflows/nsis-tests.yml index 958f3a253038..e80ed43f4099 100644 --- a/.github/workflows/nsis-tests.yml +++ b/.github/workflows/nsis-tests.yml @@ -14,6 +14,7 @@ jobs: name: Logic Tests runs-on: - windows-latest + if: ${{ contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) || fromJSON(inputs.changed-files)['nsis_tests'] }} steps: From 8b16003cabe02ed137879f902b0a9284028f19a9 Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 26 Aug 2024 15:05:12 -0600 Subject: [PATCH 22/44] Fix parsing of IPv4 mapped IPv6 addresses --- changelog/66837.fixed.md | 3 +++ salt/utils/network.py | 8 +++++++- tests/pytests/unit/utils/test_network.py | 12 +++++++++--- 3 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 changelog/66837.fixed.md diff --git a/changelog/66837.fixed.md b/changelog/66837.fixed.md new file mode 100644 index 000000000000..ccbe4a1155fd --- /dev/null +++ b/changelog/66837.fixed.md @@ -0,0 +1,3 @@ +Issue 66837: Fixes an issue with the `network.local_port_tcp` function +where it was not parsing the IPv4 mapped IPv6 address correctly. The +``::ffff:`` is now removed and only the IP address is returned. diff --git a/salt/utils/network.py b/salt/utils/network.py index f964e25fa54b..bfa38e4fe9ff 100644 --- a/salt/utils/network.py +++ b/salt/utils/network.py @@ -1740,7 +1740,13 @@ def _netlink_tool_remote_on(port, which_end): continue if which_end == "local_port" and int(local_port) != int(port): continue - remotes.add(remote_host.strip("[]")) + + # Interpret IPv4-mapped IPv6 addresses as IPv4 (strip prefix) + remote_host = remote_host.strip("[]").lower() + if remote_host.startswith("::ffff:"): + remote_host = remote_host[7:] + + remotes.add(remote_host) if valid is False: remotes = None diff --git a/tests/pytests/unit/utils/test_network.py b/tests/pytests/unit/utils/test_network.py index 12d545b01545..cbc7fd3f14ae 100644 --- a/tests/pytests/unit/utils/test_network.py +++ b/tests/pytests/unit/utils/test_network.py @@ -11,7 +11,7 @@ from tests.support.mock import MagicMock, create_autospec, mock_open, patch pytestmark = [ - pytest.mark.skip_on_windows, + pytest.mark.windows_whitelisted, ] @@ -722,13 +722,13 @@ def test_netlink_tool_remote_on_a(): with patch("salt.utils.platform.is_linux", return_value=True): with patch("subprocess.check_output", return_value=LINUX_NETLINK_SS_OUTPUT): remotes = network._netlink_tool_remote_on("4506", "local_port") - assert remotes == {"192.168.122.177", "::ffff:127.0.0.1"} + assert remotes == {"192.168.122.177", "127.0.0.1"} def test_netlink_tool_remote_on_b(): with patch("subprocess.check_output", return_value=LINUX_NETLINK_SS_OUTPUT): remotes = network._netlink_tool_remote_on("4505", "remote_port") - assert remotes == {"127.0.0.1", "::ffff:1.2.3.4"} + assert remotes == {"127.0.0.1", "1.2.3.4"} def test_openbsd_remotes_on(): @@ -1430,6 +1430,7 @@ def test_isportopen_false(): assert ret is False +@pytest.mark.skip_on_windows(reason="Do not run on Windows") def test_isportopen(): ret = network.isportopen("127.0.0.1", "22") assert ret == 0 @@ -1445,6 +1446,7 @@ def test_get_socket(): assert ret.type == socket.SOCK_STREAM +@pytest.mark.skip_on_windows(reason="Do not run on Windows") def test_ip_to_host(grains): ret = network.ip_to_host("127.0.0.1") if grains["oscodename"] == "Photon": @@ -1506,6 +1508,7 @@ def test_rpad_ipv4_network(addr, expected): assert network.rpad_ipv4_network(addr) == expected +@pytest.mark.skip_on_windows(reason="Do not run on Windows") def test_hw_addr(linux_interfaces_dict, freebsd_interfaces_dict): with patch( @@ -1531,6 +1534,7 @@ def test_hw_addr(linux_interfaces_dict, freebsd_interfaces_dict): ) +@pytest.mark.skip_on_windows(reason="Do not run on Windows") def test_interface_and_ip(linux_interfaces_dict): with patch( @@ -1557,6 +1561,7 @@ def test_interface_and_ip(linux_interfaces_dict): assert ret == 'Interface "dog" not in available interfaces: "eth0", "lo"' +@pytest.mark.skip_on_windows(reason="Do not run on Windows") def test_subnets(linux_interfaces_dict): with patch( @@ -1581,6 +1586,7 @@ def test_in_subnet(caplog): assert not ret +@pytest.mark.skip_on_windows(reason="Do not run on Windows") def test_ip_addrs(linux_interfaces_dict): with patch( "salt.utils.network.linux_interfaces", From cae94bca24479fbf0e50fcc1789725ba63613a14 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 28 Aug 2024 14:23:13 -0600 Subject: [PATCH 23/44] Make all network unit tests run on Windows --- tests/pytests/unit/utils/test_network.py | 41 +++++++++++++----------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/tests/pytests/unit/utils/test_network.py b/tests/pytests/unit/utils/test_network.py index cbc7fd3f14ae..557f315a3372 100644 --- a/tests/pytests/unit/utils/test_network.py +++ b/tests/pytests/unit/utils/test_network.py @@ -7,6 +7,7 @@ import salt.exceptions import salt.utils.network import salt.utils.network as network +import salt.utils.platform from salt._compat import ipaddress from tests.support.mock import MagicMock, create_autospec, mock_open, patch @@ -1430,9 +1431,12 @@ def test_isportopen_false(): assert ret is False -@pytest.mark.skip_on_windows(reason="Do not run on Windows") def test_isportopen(): - ret = network.isportopen("127.0.0.1", "22") + if salt.utils.platform.is_windows(): + port = "135" + else: + port = "22" + ret = network.isportopen("127.0.0.1", port) assert ret == 0 @@ -1446,14 +1450,19 @@ def test_get_socket(): assert ret.type == socket.SOCK_STREAM -@pytest.mark.skip_on_windows(reason="Do not run on Windows") +# @pytest.mark.skip_on_windows(reason="Do not run on Windows") def test_ip_to_host(grains): + if salt.utils.platform.is_windows(): + hostname = socket.gethostname() + else: + hostname = "localhost" + ret = network.ip_to_host("127.0.0.1") - if grains["oscodename"] == "Photon": + if grains.get("oscodename") == "Photon": # Photon returns this for IPv4 assert ret == "ipv6-localhost" else: - assert ret == "localhost" + assert ret == hostname ret = network.ip_to_host("2001:a71::1") assert ret is None @@ -1463,22 +1472,22 @@ def test_ip_to_host(grains): assert ret == "localhost6" elif grains["os_family"] == "Debian": if grains["osmajorrelease"] == 12: - assert ret == "localhost" + assert ret == hostname else: assert ret == "ip6-localhost" elif grains["os_family"] == "RedHat": if grains["oscodename"] == "Photon": assert ret == "ipv6-localhost" else: - assert ret == "localhost" + assert ret == hostname elif grains["os_family"] == "Arch": if grains.get("osmajorrelease", None) is None: # running doesn't have osmajorrelease grains - assert ret == "localhost" + assert ret == hostname else: assert ret == "ip6-localhost" else: - assert ret == "localhost" + assert ret == hostname @pytest.mark.parametrize( @@ -1508,11 +1517,10 @@ def test_rpad_ipv4_network(addr, expected): assert network.rpad_ipv4_network(addr) == expected -@pytest.mark.skip_on_windows(reason="Do not run on Windows") def test_hw_addr(linux_interfaces_dict, freebsd_interfaces_dict): with patch( - "salt.utils.network.linux_interfaces", + "salt.utils.network.interfaces", MagicMock(return_value=linux_interfaces_dict), ): hw_addrs = network.hw_addr("eth0") @@ -1534,11 +1542,10 @@ def test_hw_addr(linux_interfaces_dict, freebsd_interfaces_dict): ) -@pytest.mark.skip_on_windows(reason="Do not run on Windows") def test_interface_and_ip(linux_interfaces_dict): with patch( - "salt.utils.network.linux_interfaces", + "salt.utils.network.interfaces", MagicMock(return_value=linux_interfaces_dict), ): expected = [ @@ -1561,11 +1568,10 @@ def test_interface_and_ip(linux_interfaces_dict): assert ret == 'Interface "dog" not in available interfaces: "eth0", "lo"' -@pytest.mark.skip_on_windows(reason="Do not run on Windows") def test_subnets(linux_interfaces_dict): with patch( - "salt.utils.network.linux_interfaces", + "salt.utils.network.interfaces", MagicMock(return_value=linux_interfaces_dict), ): ret = network.subnets() @@ -1586,17 +1592,16 @@ def test_in_subnet(caplog): assert not ret -@pytest.mark.skip_on_windows(reason="Do not run on Windows") def test_ip_addrs(linux_interfaces_dict): with patch( - "salt.utils.network.linux_interfaces", + "salt.utils.network.interfaces", MagicMock(return_value=linux_interfaces_dict), ): ret = network.ip_addrs("eth0") assert ret == ["10.10.10.56"] with patch( - "salt.utils.network.linux_interfaces", + "salt.utils.network.interfaces", MagicMock(return_value=linux_interfaces_dict), ): ret = network.ip_addrs6("eth0") From 63b9da8bde36e79be0fc5cbcfc0545deaa71dcd1 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 15 Aug 2024 08:32:22 -0600 Subject: [PATCH 24/44] Fix pkg.removed with multiple versions on Windows --- changelog/61001.fixed.md | 2 + salt/states/pkg.py | 4 +- tests/pytests/functional/conftest.py | 13 +++++++ tests/pytests/functional/states/test_pkg.py | 43 +++++++++++++++++++-- 4 files changed, 56 insertions(+), 6 deletions(-) create mode 100644 changelog/61001.fixed.md diff --git a/changelog/61001.fixed.md b/changelog/61001.fixed.md new file mode 100644 index 000000000000..f9e6acf934d2 --- /dev/null +++ b/changelog/61001.fixed.md @@ -0,0 +1,2 @@ +Fixed an issue uninstalling packages on Windows using pkg.removed where there +are multiple versions of the same software installed diff --git a/salt/states/pkg.py b/salt/states/pkg.py index b0ab5f7ef736..1d56b545dc69 100644 --- a/salt/states/pkg.py +++ b/salt/states/pkg.py @@ -2884,7 +2884,7 @@ def _uninstall( try: pkg_params = __salt__["pkg_resource.parse_targets"]( - name, pkgs, normalize=normalize + name, pkgs, normalize=normalize, version=version, **kwargs )[0] except MinionError as exc: return { @@ -2951,7 +2951,7 @@ def _uninstall( new = __salt__["pkg.list_pkgs"](versions_as_list=True, **kwargs) failed = [] for param in pkg_params: - if __grains__["os_family"] in ["Suse", "RedHat"]: + if __grains__["os_family"] in ["Suse", "RedHat", "Windows"]: # Check if the package version set to be removed is actually removed: if param in new and not pkg_params[param]: failed.append(param) diff --git a/tests/pytests/functional/conftest.py b/tests/pytests/functional/conftest.py index 2fb2246b6338..5274ce23d356 100644 --- a/tests/pytests/functional/conftest.py +++ b/tests/pytests/functional/conftest.py @@ -4,6 +4,8 @@ import pytest from saltfactories.utils.functional import Loaders +import salt.utils.platform + log = logging.getLogger(__name__) @@ -70,6 +72,17 @@ def minion_opts( }, } ) + + if salt.utils.platform.is_windows(): + # We need to setup winrepo on Windows + minion_config_overrides.update( + { + "winrepo_source_dir": "salt://winrepo_ng", + "winrepo_dir_ng": str(state_tree / "winrepo_ng"), + "winrepo_dir": str(state_tree / "winrepo"), + } + ) + factory = salt_factories.salt_minion_daemon( minion_id, defaults=minion_config_defaults or None, diff --git a/tests/pytests/functional/states/test_pkg.py b/tests/pytests/functional/states/test_pkg.py index 8bfac2589667..f4cea3e0e97e 100644 --- a/tests/pytests/functional/states/test_pkg.py +++ b/tests/pytests/functional/states/test_pkg.py @@ -20,12 +20,17 @@ pytest.mark.slow_test, pytest.mark.skip_if_not_root, pytest.mark.destructive_test, + pytest.mark.windows_whitelisted, pytest.mark.timeout_unless_on_windows(240), ] @pytest.fixture(scope="module", autouse=True) def refresh_db(grains, modules): + + if salt.utils.platform.is_windows(): + modules.winrepo.update_git_repos() + modules.pkg.refresh_db() # If this is Arch Linux, check if pacman is in use by another process @@ -43,7 +48,7 @@ def refresh_db(grains, modules): def refresh_keys(grains, modules): if grains["os_family"] == "Arch": # We should be running this periodically when building new test runner - # images, otherwise this could take several minuets to complete. + # images, otherwise this could take several minutes to complete. proc = subprocess.run(["pacman-key", "--refresh-keys"], check=False) if proc.returncode != 0: pytest.fail("pacman-key --refresh-keys command failed.") @@ -53,7 +58,7 @@ def refresh_keys(grains, modules): def PKG_TARGETS(grains): _PKG_TARGETS = ["figlet", "sl"] if grains["os"] == "Windows": - _PKG_TARGETS = ["vlc", "putty"] + _PKG_TARGETS = ["7zip", "putty"] elif grains["os"] == "Amazon": if grains["osfinger"] == "Amazon Linux-2023": _PKG_TARGETS = ["lynx", "gnuplot-minimal"] @@ -106,7 +111,9 @@ def PKG_CAP_TARGETS(grains): @pytest.fixture def PKG_32_TARGETS(grains): _PKG_32_TARGETS = [] - if grains["os_family"] == "RedHat" and grains["oscodename"] != "Photon": + if grains["os"] == "Windows": + _PKG_32_TARGETS = ["npp", "nsis"] + elif grains["os_family"] == "RedHat" and grains["oscodename"] != "Photon": if grains["os"] == "CentOS": if grains["osmajorrelease"] == 5: _PKG_32_TARGETS = ["xz-devel.i386"] @@ -205,6 +212,19 @@ def run_command(*names): return run_command +@pytest.fixture(scope="function") +def install_7zip(modules): + try: + modules.pkg.install(name="7zip", version="22.01.00.0") + modules.pkg.install(name="7zip", version="19.00.00.0") + assert modules.pkg.version("7zip") == "19.00.00.0,22.01.00.0" + yield + finally: + modules.pkg.remove(name="7zip", version="19.00.00.0") + modules.pkg.remove(name="7zip", version="22.01.00.0") + assert modules.pkg.version("7zip") == "" + + @pytest.mark.requires_salt_modules("pkg.version") @pytest.mark.requires_salt_states("pkg.installed", "pkg.removed") @pytest.mark.slow_test @@ -268,7 +288,8 @@ def test_pkg_003_installed_multipkg(caplog, PKG_TARGETS, modules, states, grains try: ret = states.pkg.installed(name=None, pkgs=PKG_TARGETS, refresh=False) assert ret.result is True - assert "WARNING" not in caplog.text + if not salt.utils.platform.is_windows(): + assert "WARNING" not in caplog.text finally: ret = states.pkg.removed(name=None, pkgs=PKG_TARGETS) assert ret.result is True @@ -1091,3 +1112,17 @@ def test_pkg_purged_with_removed_pkg(grains, PKG_TARGETS, states, modules): "installed": {}, "removed": {target: {"new": "", "old": version}}, } + + +@pytest.mark.skip_unless_on_windows() +def test_pkg_removed_with_version_multiple(install_7zip, modules, states): + """ + This tests removing a specific version of a package when multiple versions + are installed. This is specific to Windows. The only version I could find + that allowed multiple installs of differing versions was 7zip, so we'll use + that. + """ + ret = states.pkg.removed(name="7zip", version="19.00.00.0") + assert ret.result is True + current = modules.pkg.version("7zip") + assert current == "22.01.00.0" From de33a4010721c2479d226d1ea53b1a0aea166405 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 21 Aug 2024 14:36:05 -0600 Subject: [PATCH 25/44] Fix test_win_pkg with real winrepo location --- tests/pytests/functional/modules/test_win_pkg.py | 2 +- tests/pytests/functional/states/test_pkg.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/pytests/functional/modules/test_win_pkg.py b/tests/pytests/functional/modules/test_win_pkg.py index b68895ef6253..6bcfaa9bd849 100644 --- a/tests/pytests/functional/modules/test_win_pkg.py +++ b/tests/pytests/functional/modules/test_win_pkg.py @@ -29,7 +29,7 @@ def pkg(modules): def test_refresh_db(pkg, pkg_def_contents, state_tree, minion_opts): assert len(pkg.get_package_info("my-software")) == 0 - repo_dir = state_tree / "win" / "repo-ng" + repo_dir = state_tree / "winrepo_ng" with pytest.helpers.temp_file("my-software.sls", pkg_def_contents, repo_dir): pkg.refresh_db() assert len(pkg.get_package_info("my-software")) == 1 diff --git a/tests/pytests/functional/states/test_pkg.py b/tests/pytests/functional/states/test_pkg.py index f4cea3e0e97e..a56f06667e44 100644 --- a/tests/pytests/functional/states/test_pkg.py +++ b/tests/pytests/functional/states/test_pkg.py @@ -111,14 +111,14 @@ def PKG_CAP_TARGETS(grains): @pytest.fixture def PKG_32_TARGETS(grains): _PKG_32_TARGETS = [] - if grains["os"] == "Windows": - _PKG_32_TARGETS = ["npp", "nsis"] - elif grains["os_family"] == "RedHat" and grains["oscodename"] != "Photon": + if grains["os_family"] == "RedHat" and grains["oscodename"] != "Photon": if grains["os"] == "CentOS": if grains["osmajorrelease"] == 5: _PKG_32_TARGETS = ["xz-devel.i386"] else: _PKG_32_TARGETS.append("xz-devel.i686") + elif grains["os"] == "Windows": + _PKG_32_TARGETS = ["npp", "nsis"] if not _PKG_32_TARGETS: pytest.skip("No 32 bit packages have been specified for testing") return _PKG_32_TARGETS From 7437fe9230bee4477eb6cef261bcf552bc2311eb Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 27 Aug 2024 09:32:09 -0600 Subject: [PATCH 26/44] Gate passing version for Windows --- salt/states/pkg.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/salt/states/pkg.py b/salt/states/pkg.py index 1d56b545dc69..71b0cd30febb 100644 --- a/salt/states/pkg.py +++ b/salt/states/pkg.py @@ -2883,9 +2883,14 @@ def _uninstall( } try: - pkg_params = __salt__["pkg_resource.parse_targets"]( - name, pkgs, normalize=normalize, version=version, **kwargs - )[0] + if salt.utils.platform.is_windows(): + pkg_params = __salt__["pkg_resource.parse_targets"]( + name, pkgs, normalize=normalize, version=version, **kwargs + )[0] + else: + pkg_params = __salt__["pkg_resource.parse_targets"]( + name, pkgs, normalize=normalize + )[0] except MinionError as exc: return { "name": name, From d3c0f996a4a84e91a1c7326605061e2235cb38cb Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 29 Aug 2024 11:21:14 -0600 Subject: [PATCH 27/44] Revert gating for Windows, use sys instead of salt.utils.platform --- salt/states/pkg.py | 11 +++-------- tests/pytests/functional/conftest.py | 7 +++---- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/salt/states/pkg.py b/salt/states/pkg.py index 71b0cd30febb..1d56b545dc69 100644 --- a/salt/states/pkg.py +++ b/salt/states/pkg.py @@ -2883,14 +2883,9 @@ def _uninstall( } try: - if salt.utils.platform.is_windows(): - pkg_params = __salt__["pkg_resource.parse_targets"]( - name, pkgs, normalize=normalize, version=version, **kwargs - )[0] - else: - pkg_params = __salt__["pkg_resource.parse_targets"]( - name, pkgs, normalize=normalize - )[0] + pkg_params = __salt__["pkg_resource.parse_targets"]( + name, pkgs, normalize=normalize, version=version, **kwargs + )[0] except MinionError as exc: return { "name": name, diff --git a/tests/pytests/functional/conftest.py b/tests/pytests/functional/conftest.py index 5274ce23d356..0a8219b8f717 100644 --- a/tests/pytests/functional/conftest.py +++ b/tests/pytests/functional/conftest.py @@ -1,11 +1,10 @@ import logging import shutil +import sys import pytest from saltfactories.utils.functional import Loaders -import salt.utils.platform - log = logging.getLogger(__name__) @@ -73,8 +72,8 @@ def minion_opts( } ) - if salt.utils.platform.is_windows(): - # We need to setup winrepo on Windows + if sys.platform.startswith("win"): + # We need to set up winrepo on Windows minion_config_overrides.update( { "winrepo_source_dir": "salt://winrepo_ng", From 36c65391a1aa63c6466c8105439f4f5a20156bf7 Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 30 Aug 2024 09:27:20 -0600 Subject: [PATCH 28/44] Add some SSH settings to fix the SSH Broken Pipe with Amazon Linux --- tests/conftest.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 9d673a3092c9..3fc062c73366 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1381,7 +1381,9 @@ def sshd_server(salt_factories, sshd_config_dir, salt_master, grains): "X11DisplayOffset": "10", "PrintMotd": "no", "PrintLastLog": "yes", - "TCPKeepAlive": "yes", + # https://unix.stackexchange.com/a/616355 + "ServerAliveInterval": "20", + "TCPKeepAlive": "no", "AcceptEnv": "LANG LC_*", "UsePAM": "yes", } From 72bb50cb279902b1151a4c3f21ac1d102cfda37d Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 4 Sep 2024 09:38:45 -0600 Subject: [PATCH 29/44] Revert SSH settings --- tests/conftest.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 3fc062c73366..9d673a3092c9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1381,9 +1381,7 @@ def sshd_server(salt_factories, sshd_config_dir, salt_master, grains): "X11DisplayOffset": "10", "PrintMotd": "no", "PrintLastLog": "yes", - # https://unix.stackexchange.com/a/616355 - "ServerAliveInterval": "20", - "TCPKeepAlive": "no", + "TCPKeepAlive": "yes", "AcceptEnv": "LANG LC_*", "UsePAM": "yes", } From f0e2cf4689770ba4688a269610fcb06aa1ca7b2c Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 26 Aug 2024 11:16:33 -0600 Subject: [PATCH 30/44] Remove salt.utils.data from fileserver The salt.utils.data util was used to support python 2. Since we no longer support python 2, we can start removing this. This gives us some great speed increases as we're removing a lot of overhead and unnecessary function calls --- salt/fileserver/__init__.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/salt/fileserver/__init__.py b/salt/fileserver/__init__.py index fe4b3b8e4968..b71af97b12b9 100644 --- a/salt/fileserver/__init__.py +++ b/salt/fileserver/__init__.py @@ -11,7 +11,6 @@ from collections.abc import Sequence import salt.loader -import salt.utils.data import salt.utils.files import salt.utils.path import salt.utils.url @@ -147,13 +146,7 @@ def check_file_list_cache(opts, form, list_cache, w_lock): opts.get("fileserver_list_cache_time", 20), list_cache, ) - return ( - salt.utils.data.decode( - salt.payload.load(fp_).get(form, []) - ), - False, - False, - ) + return salt.payload.load(fp_).get(form, []), False, False elif _lock_cache(w_lock): # Set the w_lock and go refresh_cache = True @@ -189,7 +182,7 @@ def check_env_cache(opts, env_cache): try: with salt.utils.files.fopen(env_cache, "rb") as fp_: log.trace("Returning env cache data from %s", env_cache) - return salt.utils.data.decode(salt.payload.load(fp_)) + return salt.payload.load(fp_) except OSError: pass return None From 50187433cfe693e0883b24ba5c8d53e290312fb1 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 28 Aug 2024 13:42:03 -0600 Subject: [PATCH 31/44] Add changelog for 66835 --- changelog/66835.fixed.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changelog/66835.fixed.md diff --git a/changelog/66835.fixed.md b/changelog/66835.fixed.md new file mode 100644 index 000000000000..07420788c5f6 --- /dev/null +++ b/changelog/66835.fixed.md @@ -0,0 +1,2 @@ +Removed ``salt.utils.data.decode`` usage from the fileserver. This function was +necessary to support Python 2. This speeds up loading the list cache by 80-90x. \ No newline at end of file From f9c9787251285d77703ba39802f74d0c90a16d59 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 29 Aug 2024 09:14:40 -0600 Subject: [PATCH 32/44] Fix pre-commit --- changelog/66835.fixed.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/66835.fixed.md b/changelog/66835.fixed.md index 07420788c5f6..33d932b7fdf1 100644 --- a/changelog/66835.fixed.md +++ b/changelog/66835.fixed.md @@ -1,2 +1,2 @@ Removed ``salt.utils.data.decode`` usage from the fileserver. This function was -necessary to support Python 2. This speeds up loading the list cache by 80-90x. \ No newline at end of file +necessary to support Python 2. This speeds up loading the list cache by 80-90x. From 468b26e91867f114a4891a3d38f7313aac1227d7 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Sun, 18 Aug 2024 15:18:32 -0700 Subject: [PATCH 33/44] initial work on log once option --- salt/_logging/impl.py | 10 ++++++++++ salt/grains/core.py | 1 + 2 files changed, 11 insertions(+) diff --git a/salt/_logging/impl.py b/salt/_logging/impl.py index 832c72b4769a..2a12ddf3f941 100644 --- a/salt/_logging/impl.py +++ b/salt/_logging/impl.py @@ -157,6 +157,9 @@ def set_log_record_factory(factory): class SaltLoggingClass(LOGGING_LOGGER_CLASS, metaclass=LoggingMixinMeta): + + ONCECACHE = set() + def __new__(cls, *args): """ We override `__new__` in our logging logger class in order to provide @@ -233,10 +236,16 @@ def _log( stack_info=False, stacklevel=1, exc_info_on_loglevel=None, + once=False, ): if extra is None: extra = {} + if once: + if str(args) in self.ONCECACHE: + return + self.ONCECACHE.add(str(args)) + # pylint: disable=no-member current_jid = RequestContext.current.get("data", {}).get("jid", None) log_fmt_jid = RequestContext.current.get("opts", {}).get("log_fmt_jid", None) @@ -265,6 +274,7 @@ def _log( exc_info_on_loglevel ) ) + # XXX: extra is never None if extra is None: extra = {"exc_info_on_loglevel": exc_info_on_loglevel} else: diff --git a/salt/grains/core.py b/salt/grains/core.py index 51646f6f9791..119e78c8e9bb 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -1275,6 +1275,7 @@ def _virtual(osdata): "cannot execute it. Grains output might not be " "accurate.", command, + once=True, ) return grains From 3140aecd2ba4cd9564000b1c5c54ea8d4dd510fa Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Wed, 6 Dec 2023 16:59:38 -0700 Subject: [PATCH 34/44] Add more loader dunders to cp module. Using cp module to dip our feet into more explicit loader dunders. Most all of the existing dunders can be imported already. The one outlier being used by the cp module is __opts__. --- salt/loader/dunder.py | 3 +++ salt/modules/cp.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/loader/dunder.py b/salt/loader/dunder.py index d3027098b5ae..321e6f9a69db 100644 --- a/salt/loader/dunder.py +++ b/salt/loader/dunder.py @@ -8,3 +8,6 @@ __file_client__ = loader_context.named_context("__file_client__", default=None) +__context__ = loader_context.named_context("__context__") +__pillar__ = loader_context.named_context("__pillar__") +__grains__ = loader_context.named_context("__grains__") diff --git a/salt/modules/cp.py b/salt/modules/cp.py index 25b0fcbae911..c9c72a4d026b 100644 --- a/salt/modules/cp.py +++ b/salt/modules/cp.py @@ -20,7 +20,7 @@ import salt.utils.templates import salt.utils.url from salt.exceptions import CommandExecutionError -from salt.loader.dunder import __file_client__ +from salt.loader.dunder import __context__, __file_client__, __grains__, __pillar__ log = logging.getLogger(__name__) From 96c59e07fc9a9ab5f971283361e1d0d13f2b5a72 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Thu, 7 Dec 2023 15:43:57 -0700 Subject: [PATCH 35/44] Add tests for opts dunder --- salt/loader/dunder.py | 1 + .../pytests/functional/loader/test_dunder.py | 49 +++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 tests/pytests/functional/loader/test_dunder.py diff --git a/salt/loader/dunder.py b/salt/loader/dunder.py index 321e6f9a69db..3b198b1497f8 100644 --- a/salt/loader/dunder.py +++ b/salt/loader/dunder.py @@ -8,6 +8,7 @@ __file_client__ = loader_context.named_context("__file_client__", default=None) +__opts__ = loader_context.named_context("__opts__") __context__ = loader_context.named_context("__context__") __pillar__ = loader_context.named_context("__pillar__") __grains__ = loader_context.named_context("__grains__") diff --git a/tests/pytests/functional/loader/test_dunder.py b/tests/pytests/functional/loader/test_dunder.py new file mode 100644 index 000000000000..4cd0d629cdcc --- /dev/null +++ b/tests/pytests/functional/loader/test_dunder.py @@ -0,0 +1,49 @@ +import pathlib + +import salt.loader.context +import salt.loader.lazy +import salt.utils.files +import tests.support.helpers + + +def test_opts_dunder_opts_without_import(tempdir): + """ + Test __opts__ without being imported. + + When a loaded module uses __opts__ but does not import it from + salt.loader.dunder the __opts__ object will be a dictionary. + """ + opts = {"optimization_order": [0, 1, 2]} + with salt.utils.files.fopen(pathlib.Path(tempdir.tempdir) / "mymod.py", "w") as fp: + fp.write( + tests.support.helpers.dedent( + """ + def mymethod(): + return type(__opts__) + """ + ) + ) + loader = salt.loader.lazy.LazyLoader([tempdir.tempdir], opts) + assert loader["mymod.mymethod"]() == dict + + +def test_opts_dunder_opts_with_import(tempdir): + """ + Test __opts__ when imported. + + When a loaded module uses __opts__ by importing it from + salt.loader.dunder the __opts__ object will be a NamedLoaderContext. + """ + opts = {"optimization_order": [0, 1, 2]} + with salt.utils.files.fopen(pathlib.Path(tempdir.tempdir) / "mymod.py", "w") as fp: + fp.write( + tests.support.helpers.dedent( + """ + from salt.loader.dunder import __opts__ + def mymethod(): + return type(__opts__) + """ + ) + ) + loader = salt.loader.lazy.LazyLoader([tempdir.tempdir], opts) + assert loader["mymod.mymethod"]() == salt.loader.context.NamedLoaderContext From fc06de8db1382a1d228e3a5470272e24055c34f6 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Thu, 7 Dec 2023 15:48:48 -0700 Subject: [PATCH 36/44] Try importing opts dunder --- salt/modules/cp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/cp.py b/salt/modules/cp.py index c9c72a4d026b..3349a083ecd2 100644 --- a/salt/modules/cp.py +++ b/salt/modules/cp.py @@ -20,7 +20,7 @@ import salt.utils.templates import salt.utils.url from salt.exceptions import CommandExecutionError -from salt.loader.dunder import __context__, __file_client__, __grains__, __pillar__ +from salt.loader.dunder import __context__, __file_client__, __grains__, __pillar__, __opts__ log = logging.getLogger(__name__) From b36f2760078f28c50bb5352372c68b53fcc4de4f Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Thu, 7 Dec 2023 18:31:52 -0700 Subject: [PATCH 37/44] Better handling of loader returns - Detect when a loader is returning a NamedLoaderContext and return the value. This will cut down on how often we need to call value() in loaded modules. - Add some tests to show newly defined behavior --- salt/loader/context.py | 14 ++++++++++++-- salt/modules/cp.py | 10 ++++++++-- tests/pytests/functional/loader/test_dunder.py | 13 ++++++++----- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/salt/loader/context.py b/salt/loader/context.py index 88a6472a8f3c..04e3bf094ebc 100644 --- a/salt/loader/context.py +++ b/salt/loader/context.py @@ -68,9 +68,19 @@ def value(self): loader = self.loader() if loader is None: return self.default - if self.name == "__context__": + elif self.name == "__context__": return loader.pack[self.name] - if self.name == loader.pack_self: + elif self.name == "__opts__": + # XXX: This behaviour tires to mimick what the loader does when + # __opts__ was not imported from dunder. It would be nice to just + # pass the value of __opts__ here. However, un-winding this + # behavior will be tricky. + opts = {} + if self.default: + opts.update(copy.deepcopy(self.default)) + opts.update(copy.deepcopy(loader.opts)) + return opts + elif self.name == loader.pack_self: return loader try: return loader.pack[self.name] diff --git a/salt/modules/cp.py b/salt/modules/cp.py index 3349a083ecd2..503b9976dbe1 100644 --- a/salt/modules/cp.py +++ b/salt/modules/cp.py @@ -20,7 +20,13 @@ import salt.utils.templates import salt.utils.url from salt.exceptions import CommandExecutionError -from salt.loader.dunder import __context__, __file_client__, __grains__, __pillar__, __opts__ +from salt.loader.dunder import ( + __context__, + __file_client__, + __grains__, + __opts__, + __pillar__, +) log = logging.getLogger(__name__) @@ -167,7 +173,7 @@ def _client(): """ if __file_client__: return __file_client__.value() - return salt.fileclient.get_file_client(__opts__) + return salt.fileclient.get_file_client(__opts__.value()) def _render_filenames(path, dest, saltenv, template, **kw): diff --git a/tests/pytests/functional/loader/test_dunder.py b/tests/pytests/functional/loader/test_dunder.py index 4cd0d629cdcc..76770784fbc3 100644 --- a/tests/pytests/functional/loader/test_dunder.py +++ b/tests/pytests/functional/loader/test_dunder.py @@ -6,7 +6,7 @@ import tests.support.helpers -def test_opts_dunder_opts_without_import(tempdir): +def xtest_opts_dunder_opts_without_import(tempdir): """ Test __opts__ without being imported. @@ -19,12 +19,12 @@ def test_opts_dunder_opts_without_import(tempdir): tests.support.helpers.dedent( """ def mymethod(): - return type(__opts__) + return __opts__ """ ) ) loader = salt.loader.lazy.LazyLoader([tempdir.tempdir], opts) - assert loader["mymod.mymethod"]() == dict + assert type(loader["mymod.mymethod"]()) == dict def test_opts_dunder_opts_with_import(tempdir): @@ -40,10 +40,13 @@ def test_opts_dunder_opts_with_import(tempdir): tests.support.helpers.dedent( """ from salt.loader.dunder import __opts__ - def mymethod(): + def optstype(): return type(__opts__) + def opts(): + return __opts__ """ ) ) loader = salt.loader.lazy.LazyLoader([tempdir.tempdir], opts) - assert loader["mymod.mymethod"]() == salt.loader.context.NamedLoaderContext + assert loader["mymod.optstype"]() == salt.loader.context.NamedLoaderContext + assert loader["mymod.opts"]() == opts From 96bc89f7217d30998292f7682a002e5fc5806122 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Fri, 8 Dec 2023 14:58:11 -0700 Subject: [PATCH 38/44] Fix broken tests --- salt/loader/context.py | 3 +++ tests/pytests/functional/loader/test_dunder.py | 14 ++++++-------- tests/pytests/unit/states/test_pkg.py | 3 ++- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/salt/loader/context.py b/salt/loader/context.py index 04e3bf094ebc..a94e81d00a16 100644 --- a/salt/loader/context.py +++ b/salt/loader/context.py @@ -43,6 +43,9 @@ def __init__(self, name, loader_context, default=None): self.loader_context = loader_context self.default = default + def with_default(self, default): + return NamedLoaderContext(self.name, self.loader_context, default=default) + def loader(self): """ The LazyLoader in the current context. This will return None if there diff --git a/tests/pytests/functional/loader/test_dunder.py b/tests/pytests/functional/loader/test_dunder.py index 76770784fbc3..581d669a12f3 100644 --- a/tests/pytests/functional/loader/test_dunder.py +++ b/tests/pytests/functional/loader/test_dunder.py @@ -1,12 +1,10 @@ -import pathlib - import salt.loader.context import salt.loader.lazy import salt.utils.files import tests.support.helpers -def xtest_opts_dunder_opts_without_import(tempdir): +def test_opts_dunder_opts_without_import(tmp_path): """ Test __opts__ without being imported. @@ -14,7 +12,7 @@ def xtest_opts_dunder_opts_without_import(tempdir): salt.loader.dunder the __opts__ object will be a dictionary. """ opts = {"optimization_order": [0, 1, 2]} - with salt.utils.files.fopen(pathlib.Path(tempdir.tempdir) / "mymod.py", "w") as fp: + with salt.utils.files.fopen(tmp_path / "mymod.py", "w") as fp: fp.write( tests.support.helpers.dedent( """ @@ -23,11 +21,11 @@ def mymethod(): """ ) ) - loader = salt.loader.lazy.LazyLoader([tempdir.tempdir], opts) + loader = salt.loader.lazy.LazyLoader([tmp_path], opts) assert type(loader["mymod.mymethod"]()) == dict -def test_opts_dunder_opts_with_import(tempdir): +def test_opts_dunder_opts_with_import(tmp_path): """ Test __opts__ when imported. @@ -35,7 +33,7 @@ def test_opts_dunder_opts_with_import(tempdir): salt.loader.dunder the __opts__ object will be a NamedLoaderContext. """ opts = {"optimization_order": [0, 1, 2]} - with salt.utils.files.fopen(pathlib.Path(tempdir.tempdir) / "mymod.py", "w") as fp: + with salt.utils.files.fopen(tmp_path / "mymod.py", "w") as fp: fp.write( tests.support.helpers.dedent( """ @@ -47,6 +45,6 @@ def opts(): """ ) ) - loader = salt.loader.lazy.LazyLoader([tempdir.tempdir], opts) + loader = salt.loader.lazy.LazyLoader([tmp_path], opts) assert loader["mymod.optstype"]() == salt.loader.context.NamedLoaderContext assert loader["mymod.opts"]() == opts diff --git a/tests/pytests/unit/states/test_pkg.py b/tests/pytests/unit/states/test_pkg.py index 0255175005af..f9c566524df7 100644 --- a/tests/pytests/unit/states/test_pkg.py +++ b/tests/pytests/unit/states/test_pkg.py @@ -11,6 +11,7 @@ import salt.states.beacon as beaconstate import salt.states.pkg as pkg import salt.utils.state as state_utils +from salt.loader.dunder import __opts__ from salt.utils.event import SaltEvent from tests.support.mock import MagicMock, patch @@ -21,7 +22,7 @@ def configure_loader_modules(minion_opts): return { cp: { - "__opts__": minion_opts, + "__opts__": __opts__.with_default(minion_opts), }, pkg: { "__env__": "base", From 5ee6d3c2bcff85e8a98edc587c4a0aa2fe11a0e8 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Sat, 9 Dec 2023 14:54:18 -0700 Subject: [PATCH 39/44] Fix another test using __opts__ --- tests/pytests/functional/modules/test_aptpkg.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/pytests/functional/modules/test_aptpkg.py b/tests/pytests/functional/modules/test_aptpkg.py index 41ade2889de3..60b134966feb 100644 --- a/tests/pytests/functional/modules/test_aptpkg.py +++ b/tests/pytests/functional/modules/test_aptpkg.py @@ -13,6 +13,7 @@ import salt.modules.gpg as gpg import salt.utils.files import salt.utils.stringutils +from salt.loader.dunder import __opts__ from tests.support.mock import Mock, patch pytestmark = [ @@ -76,7 +77,7 @@ def configure_loader_modules(minion_opts): }, gpg: {}, cp: { - "__opts__": minion_opts, + "__opts__": __opts__.with_default(minion_opts), }, config: { "__opts__": minion_opts, From d4ee87338b37947be1b84b881d45a082c06727c4 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Sun, 10 Dec 2023 02:38:53 -0700 Subject: [PATCH 40/44] Fix windows pkg tests --- tests/pytests/unit/modules/test_win_pkg.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/pytests/unit/modules/test_win_pkg.py b/tests/pytests/unit/modules/test_win_pkg.py index 3ae8f24f8dd6..a976b6d6083b 100644 --- a/tests/pytests/unit/modules/test_win_pkg.py +++ b/tests/pytests/unit/modules/test_win_pkg.py @@ -6,6 +6,7 @@ import pytest +import salt.loader.dunder import salt.modules.config as config import salt.modules.cp as cp import salt.modules.pkg_resource as pkg_resource @@ -57,7 +58,7 @@ def configure_loader_modules(minion_opts): opts = minion_opts opts["master_uri"] = "localhost" return { - cp: {"__opts__": opts}, + cp: {"__opts__": salt.loader.dunder.__opts__.with_default(opts)}, win_pkg: { "_get_latest_package_version": MagicMock(return_value="3.03"), "_get_package_info": MagicMock(return_value=pkg_info), From 0897afcc21b634472e40ad658d64cb139e7072e1 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Thu, 12 Sep 2024 15:22:35 -0700 Subject: [PATCH 41/44] Retrun opts because they get modified often --- salt/loader/context.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/salt/loader/context.py b/salt/loader/context.py index a94e81d00a16..1b330f18c851 100644 --- a/salt/loader/context.py +++ b/salt/loader/context.py @@ -5,6 +5,8 @@ import collections.abc import contextlib import copy +import logging +import traceback try: # Try the stdlib C extension first @@ -15,6 +17,8 @@ import salt.exceptions +log = logging.getLogger(__name__) + DEFAULT_CTX_VAR = "loader_ctxvar" loader_ctxvar = contextvars.ContextVar(DEFAULT_CTX_VAR) @@ -71,20 +75,12 @@ def value(self): loader = self.loader() if loader is None: return self.default + if self.name == loader.pack_self: + return loader elif self.name == "__context__": return loader.pack[self.name] elif self.name == "__opts__": - # XXX: This behaviour tires to mimick what the loader does when - # __opts__ was not imported from dunder. It would be nice to just - # pass the value of __opts__ here. However, un-winding this - # behavior will be tricky. - opts = {} - if self.default: - opts.update(copy.deepcopy(self.default)) - opts.update(copy.deepcopy(loader.opts)) - return opts - elif self.name == loader.pack_self: - return loader + return loader.pack[self.name] try: return loader.pack[self.name] except KeyError: From 0dc964fb507cdda2272ed0581dab08aa35330c5e Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Fri, 13 Sep 2024 00:17:58 -0700 Subject: [PATCH 42/44] Clean up imports --- salt/loader/context.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/salt/loader/context.py b/salt/loader/context.py index 1b330f18c851..38d0093a8baf 100644 --- a/salt/loader/context.py +++ b/salt/loader/context.py @@ -5,8 +5,6 @@ import collections.abc import contextlib import copy -import logging -import traceback try: # Try the stdlib C extension first @@ -17,8 +15,6 @@ import salt.exceptions -log = logging.getLogger(__name__) - DEFAULT_CTX_VAR = "loader_ctxvar" loader_ctxvar = contextvars.ContextVar(DEFAULT_CTX_VAR) From 77260acfd512d6620907749aec2a0b1357f1e703 Mon Sep 17 00:00:00 2001 From: ScriptAutomate Date: Fri, 13 Sep 2024 15:17:27 -0500 Subject: [PATCH 43/44] Drop Arch Linux; update latest golden images --- .github/workflows/ci.yml | 23 ----- .github/workflows/nightly.yml | 23 ----- .github/workflows/scheduled.yml | 23 ----- .github/workflows/staging.yml | 22 ----- .gitignore | 1 + changelog/66886.deprecated.md | 1 + cicd/golden-images.json | 122 ++++++++++++--------------- cicd/shared-gh-workflows-context.yml | 1 - tools/ci.py | 1 - tools/precommit/workflows.py | 1 - 10 files changed, 58 insertions(+), 160 deletions(-) create mode 100644 changelog/66886.deprecated.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b6de9cff239..4e88f05b4a24 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1527,27 +1527,6 @@ jobs: workflow-slug: ci default-timeout: 180 - archlinux-lts: - name: Arch Linux LTS Test - if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} - needs: - - prepare-workflow - - build-ci-deps - uses: ./.github/workflows/test-action-linux.yml - with: - distro-slug: archlinux-lts - nox-session: ci-test-onedir - platform: linux - arch: x86_64 - nox-version: 2022.8.7 - gh-actions-python-version: "3.10" - testrun: ${{ needs.prepare-workflow.outputs.testrun }} - salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" - cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.14 - skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }} - workflow-slug: ci - default-timeout: 180 - debian-11: name: Debian 11 Test if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] && contains(fromJSON(needs.prepare-workflow.outputs.os-labels), 'debian-11') }} @@ -1995,7 +1974,6 @@ jobs: - amazonlinux-2-arm64 - amazonlinux-2023 - amazonlinux-2023-arm64 - - archlinux-lts - debian-11 - debian-11-arm64 - debian-12 @@ -2163,7 +2141,6 @@ jobs: - amazonlinux-2-arm64 - amazonlinux-2023 - amazonlinux-2023-arm64 - - archlinux-lts - debian-11 - debian-11-arm64 - debian-12 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 18e390d8ab7f..d7bb49730019 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1589,27 +1589,6 @@ jobs: workflow-slug: nightly default-timeout: 360 - archlinux-lts: - name: Arch Linux LTS Test - if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} - needs: - - prepare-workflow - - build-ci-deps - uses: ./.github/workflows/test-action-linux.yml - with: - distro-slug: archlinux-lts - nox-session: ci-test-onedir - platform: linux - arch: x86_64 - nox-version: 2022.8.7 - gh-actions-python-version: "3.10" - testrun: ${{ needs.prepare-workflow.outputs.testrun }} - salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" - cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.14 - skip-code-coverage: false - workflow-slug: nightly - default-timeout: 360 - debian-11: name: Debian 11 Test if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} @@ -2057,7 +2036,6 @@ jobs: - amazonlinux-2-arm64 - amazonlinux-2023 - amazonlinux-2023-arm64 - - archlinux-lts - debian-11 - debian-11-arm64 - debian-12 @@ -2964,7 +2942,6 @@ jobs: - amazonlinux-2-arm64 - amazonlinux-2023 - amazonlinux-2023-arm64 - - archlinux-lts - debian-11 - debian-11-arm64 - debian-12 diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 767b8c17f46d..86e57fb71a42 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -1566,27 +1566,6 @@ jobs: workflow-slug: scheduled default-timeout: 360 - archlinux-lts: - name: Arch Linux LTS Test - if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} - needs: - - prepare-workflow - - build-ci-deps - uses: ./.github/workflows/test-action-linux.yml - with: - distro-slug: archlinux-lts - nox-session: ci-test-onedir - platform: linux - arch: x86_64 - nox-version: 2022.8.7 - gh-actions-python-version: "3.10" - testrun: ${{ needs.prepare-workflow.outputs.testrun }} - salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" - cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.14 - skip-code-coverage: false - workflow-slug: scheduled - default-timeout: 360 - debian-11: name: Debian 11 Test if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} @@ -2034,7 +2013,6 @@ jobs: - amazonlinux-2-arm64 - amazonlinux-2023 - amazonlinux-2023-arm64 - - archlinux-lts - debian-11 - debian-11-arm64 - debian-12 @@ -2204,7 +2182,6 @@ jobs: - amazonlinux-2-arm64 - amazonlinux-2023 - amazonlinux-2023-arm64 - - archlinux-lts - debian-11 - debian-11-arm64 - debian-12 diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 2cf7786fe2c8..1c2546357e01 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -1574,27 +1574,6 @@ jobs: workflow-slug: staging default-timeout: 180 - archlinux-lts: - name: Arch Linux LTS Test - if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} - needs: - - prepare-workflow - - build-ci-deps - uses: ./.github/workflows/test-action-linux.yml - with: - distro-slug: archlinux-lts - nox-session: ci-test-onedir - platform: linux - arch: x86_64 - nox-version: 2022.8.7 - gh-actions-python-version: "3.10" - testrun: ${{ needs.prepare-workflow.outputs.testrun }} - salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" - cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.14 - skip-code-coverage: true - workflow-slug: staging - default-timeout: 180 - debian-11: name: Debian 11 Test if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} @@ -2917,7 +2896,6 @@ jobs: - amazonlinux-2-arm64 - amazonlinux-2023 - amazonlinux-2023-arm64 - - archlinux-lts - debian-11 - debian-11-arm64 - debian-12 diff --git a/.gitignore b/.gitignore index 63b49a644872..73cc51c3d1ce 100644 --- a/.gitignore +++ b/.gitignore @@ -89,6 +89,7 @@ tests/unit/templates/roots # Pycharm .idea venv/ +.venv/ # VS Code .vscode diff --git a/changelog/66886.deprecated.md b/changelog/66886.deprecated.md new file mode 100644 index 000000000000..597c0aee10a4 --- /dev/null +++ b/changelog/66886.deprecated.md @@ -0,0 +1 @@ +Drop Arch Linux support diff --git a/cicd/golden-images.json b/cicd/golden-images.json index ca7818fdd6ba..b0504ad777a7 100644 --- a/cicd/golden-images.json +++ b/cicd/golden-images.json @@ -1,8 +1,8 @@ { "amazonlinux-2-arm64": { - "ami": "ami-0c98c023fba59d522", + "ami": "ami-0aab00f54b6cddde6", "ami_description": "CI Image of AmazonLinux 2 arm64", - "ami_name": "salt-project/ci/amazonlinux/2/arm64/20240509.1530", + "ami_name": "salt-project/ci/amazonlinux/2/arm64/20240912.2135", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -10,9 +10,9 @@ "ssh_username": "ec2-user" }, "amazonlinux-2": { - "ami": "ami-02cba95cfd7074794", + "ami": "ami-0fd6cec7bbcf52d36", "ami_description": "CI Image of AmazonLinux 2 x86_64", - "ami_name": "salt-project/ci/amazonlinux/2/x86_64/20240509.1530", + "ami_name": "salt-project/ci/amazonlinux/2/x86_64/20240912.2135", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -20,9 +20,9 @@ "ssh_username": "ec2-user" }, "amazonlinux-2023-arm64": { - "ami": "ami-0609f0e98f5a6b73d", + "ami": "ami-095e9e4757b5fca1a", "ami_description": "CI Image of AmazonLinux 2023 arm64", - "ami_name": "salt-project/ci/amazonlinux/2023/arm64/20240509.1529", + "ami_name": "salt-project/ci/amazonlinux/2023/arm64/20240912.2136", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -30,29 +30,19 @@ "ssh_username": "ec2-user" }, "amazonlinux-2023": { - "ami": "ami-0554a801eb6dcc42c", + "ami": "ami-002d043f1a36bf06e", "ami_description": "CI Image of AmazonLinux 2023 x86_64", - "ami_name": "salt-project/ci/amazonlinux/2023/x86_64/20240509.1529", + "ami_name": "salt-project/ci/amazonlinux/2023/x86_64/20240912.2136", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", "is_windows": "false", "ssh_username": "ec2-user" }, - "archlinux-lts": { - "ami": "ami-01ad78f19930b9747", - "ami_description": "CI Image of ArchLinux lts x86_64", - "ami_name": "salt-project/ci/archlinux/lts/x86_64/20240509.1530", - "arch": "x86_64", - "cloudwatch-agent-available": "false", - "instance_type": "t3a.large", - "is_windows": "false", - "ssh_username": "arch" - }, "debian-11-arm64": { - "ami": "ami-0eff227d9a94d8692", + "ami": "ami-0ff63235fce7bea1d", "ami_description": "CI Image of Debian 11 arm64", - "ami_name": "salt-project/ci/debian/11/arm64/20240509.1529", + "ami_name": "salt-project/ci/debian/11/arm64/20240912.2135", "arch": "arm64", "cloudwatch-agent-available": "false", "instance_type": "m6g.large", @@ -60,9 +50,9 @@ "ssh_username": "admin" }, "debian-11": { - "ami": "ami-099b2a5a1fb995166", + "ami": "ami-08685bfca48beeb67", "ami_description": "CI Image of Debian 11 x86_64", - "ami_name": "salt-project/ci/debian/11/x86_64/20240509.1529", + "ami_name": "salt-project/ci/debian/11/x86_64/20240912.2135", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -70,9 +60,9 @@ "ssh_username": "admin" }, "debian-12-arm64": { - "ami": "ami-0ab6b0cc8488f8880", + "ami": "ami-07d383138f04b32ba", "ami_description": "CI Image of Debian 12 arm64", - "ami_name": "salt-project/ci/debian/12/arm64/20240509.1529", + "ami_name": "salt-project/ci/debian/12/arm64/20240912.2135", "arch": "arm64", "cloudwatch-agent-available": "false", "instance_type": "m6g.large", @@ -80,9 +70,9 @@ "ssh_username": "admin" }, "debian-12": { - "ami": "ami-0e1f5b55325249c4e", + "ami": "ami-0867ec74072fd97a0", "ami_description": "CI Image of Debian 12 x86_64", - "ami_name": "salt-project/ci/debian/12/x86_64/20240509.1530", + "ami_name": "salt-project/ci/debian/12/x86_64/20240912.2135", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -90,9 +80,9 @@ "ssh_username": "admin" }, "fedora-40-arm64": { - "ami": "ami-064df327a55f83953", + "ami": "ami-03be8e03c17f1abeb", "ami_description": "CI Image of Fedora 40 arm64", - "ami_name": "salt-project/ci/fedora/40/arm64/20240509.1530", + "ami_name": "salt-project/ci/fedora/40/arm64/20240912.2136", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -100,9 +90,9 @@ "ssh_username": "fedora" }, "fedora-40": { - "ami": "ami-08d8dbd4f063788de", + "ami": "ami-060a59b30809758b2", "ami_description": "CI Image of Fedora 40 x86_64", - "ami_name": "salt-project/ci/fedora/40/x86_64/20240509.1530", + "ami_name": "salt-project/ci/fedora/40/x86_64/20240912.2136", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -110,9 +100,9 @@ "ssh_username": "fedora" }, "opensuse-15": { - "ami": "ami-0f82d5ab3015af6ad", + "ami": "ami-0aaf63315ada5365b", "ami_description": "CI Image of Opensuse 15 x86_64", - "ami_name": "salt-project/ci/opensuse/15/x86_64/20240509.1529", + "ami_name": "salt-project/ci/opensuse/15/x86_64/20240912.2135", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -120,9 +110,9 @@ "ssh_username": "ec2-user" }, "photonos-4-arm64": { - "ami": "ami-0ea152c346cb8e13b", + "ami": "ami-0d425acec9d0d78a5", "ami_description": "CI Image of PhotonOS 4 arm64", - "ami_name": "salt-project/ci/photonos/4/arm64/20240509.1530", + "ami_name": "salt-project/ci/photonos/4/arm64/20240912.2136", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -130,9 +120,9 @@ "ssh_username": "root" }, "photonos-4": { - "ami": "ami-09b55d0bf3a1aa7e5", + "ami": "ami-056d988807f8b586d", "ami_description": "CI Image of PhotonOS 4 x86_64", - "ami_name": "salt-project/ci/photonos/4/x86_64/20240509.1530", + "ami_name": "salt-project/ci/photonos/4/x86_64/20240912.2136", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -140,9 +130,9 @@ "ssh_username": "root" }, "photonos-5-arm64": { - "ami": "ami-09de4952bc9fc068a", + "ami": "ami-059f47b459d04544a", "ami_description": "CI Image of PhotonOS 5 arm64", - "ami_name": "salt-project/ci/photonos/5/arm64/20240509.1530", + "ami_name": "salt-project/ci/photonos/5/arm64/20240912.2136", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -150,9 +140,9 @@ "ssh_username": "root" }, "photonos-5": { - "ami": "ami-0c3375a583643fc77", + "ami": "ami-06424daf7c85ffff0", "ami_description": "CI Image of PhotonOS 5 x86_64", - "ami_name": "salt-project/ci/photonos/5/x86_64/20240509.1530", + "ami_name": "salt-project/ci/photonos/5/x86_64/20240912.2136", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -160,9 +150,9 @@ "ssh_username": "root" }, "rockylinux-8-arm64": { - "ami": "ami-0662cc201cada14b8", + "ami": "ami-0a21b175629f1a793", "ami_description": "CI Image of RockyLinux 8 arm64", - "ami_name": "salt-project/ci/rockylinux/8/arm64/20240509.1530", + "ami_name": "salt-project/ci/rockylinux/8/arm64/20240912.2136", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -170,9 +160,9 @@ "ssh_username": "rocky" }, "rockylinux-8": { - "ami": "ami-071ca70a907d79e05", + "ami": "ami-01032695e18f0fe85", "ami_description": "CI Image of RockyLinux 8 x86_64", - "ami_name": "salt-project/ci/rockylinux/8/x86_64/20240509.1530", + "ami_name": "salt-project/ci/rockylinux/8/x86_64/20240912.2136", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -180,9 +170,9 @@ "ssh_username": "rocky" }, "rockylinux-9-arm64": { - "ami": "ami-065842dfdf03a1a03", + "ami": "ami-0c9147ca5f07effc6", "ami_description": "CI Image of RockyLinux 9 arm64", - "ami_name": "salt-project/ci/rockylinux/9/arm64/20240509.1530", + "ami_name": "salt-project/ci/rockylinux/9/arm64/20240912.2136", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -190,9 +180,9 @@ "ssh_username": "rocky" }, "rockylinux-9": { - "ami": "ami-09f5d6df00e99ba16", + "ami": "ami-01a72f34d198efc4a", "ami_description": "CI Image of RockyLinux 9 x86_64", - "ami_name": "salt-project/ci/rockylinux/9/x86_64/20240509.1530", + "ami_name": "salt-project/ci/rockylinux/9/x86_64/20240912.2136", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -200,9 +190,9 @@ "ssh_username": "rocky" }, "ubuntu-20.04-arm64": { - "ami": "ami-00171fa604b826054", + "ami": "ami-0bf8ea4c07a88d6c5", "ami_description": "CI Image of Ubuntu 20.04 arm64", - "ami_name": "salt-project/ci/ubuntu/20.04/arm64/20240509.1530", + "ami_name": "salt-project/ci/ubuntu/20.04/arm64/20240912.2136", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -210,9 +200,9 @@ "ssh_username": "ubuntu" }, "ubuntu-20.04": { - "ami": "ami-07ddfbdc489064022", + "ami": "ami-08a84f7455622c3d5", "ami_description": "CI Image of Ubuntu 20.04 x86_64", - "ami_name": "salt-project/ci/ubuntu/20.04/x86_64/20240509.1530", + "ami_name": "salt-project/ci/ubuntu/20.04/x86_64/20240912.2136", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -220,9 +210,9 @@ "ssh_username": "ubuntu" }, "ubuntu-22.04-arm64": { - "ami": "ami-0e6b6fc1dd298e055", + "ami": "ami-0415a2d2279277d61", "ami_description": "CI Image of Ubuntu 22.04 arm64", - "ami_name": "salt-project/ci/ubuntu/22.04/arm64/20240509.1530", + "ami_name": "salt-project/ci/ubuntu/22.04/arm64/20240912.2136", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -230,9 +220,9 @@ "ssh_username": "ubuntu" }, "ubuntu-22.04": { - "ami": "ami-0736289579c0d01ba", + "ami": "ami-055513129ce06397c", "ami_description": "CI Image of Ubuntu 22.04 x86_64", - "ami_name": "salt-project/ci/ubuntu/22.04/x86_64/20240509.1530", + "ami_name": "salt-project/ci/ubuntu/22.04/x86_64/20240912.2136", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -240,9 +230,9 @@ "ssh_username": "ubuntu" }, "ubuntu-24.04-arm64": { - "ami": "ami-015058823f69446b3", + "ami": "ami-035ef6d54ec25b0fa", "ami_description": "CI Image of Ubuntu 24.04 arm64", - "ami_name": "salt-project/ci/ubuntu/24.04/arm64/20240509.1530", + "ami_name": "salt-project/ci/ubuntu/24.04/arm64/20240912.2136", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -250,9 +240,9 @@ "ssh_username": "ubuntu" }, "ubuntu-24.04": { - "ami": "ami-0eb04152e7cafaaf9", + "ami": "ami-0a287b781a487ec65", "ami_description": "CI Image of Ubuntu 24.04 x86_64", - "ami_name": "salt-project/ci/ubuntu/24.04/x86_64/20240509.1530", + "ami_name": "salt-project/ci/ubuntu/24.04/x86_64/20240912.2136", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -260,9 +250,9 @@ "ssh_username": "ubuntu" }, "windows-2016": { - "ami": "ami-06026cb4d83072df5", + "ami": "ami-030cdb60764141f56", "ami_description": "CI Image of Windows 2016 x86_64", - "ami_name": "salt-project/ci/windows/2016/x86_64/20240509.1530", + "ami_name": "salt-project/ci/windows/2016/x86_64/20240913.1756", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.xlarge", @@ -270,9 +260,9 @@ "ssh_username": "Administrator" }, "windows-2019": { - "ami": "ami-095a9256ec0e8261c", + "ami": "ami-08f10b0d4914572de", "ami_description": "CI Image of Windows 2019 x86_64", - "ami_name": "salt-project/ci/windows/2019/x86_64/20240509.1530", + "ami_name": "salt-project/ci/windows/2019/x86_64/20240913.1756", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.xlarge", @@ -280,9 +270,9 @@ "ssh_username": "Administrator" }, "windows-2022": { - "ami": "ami-0d295c0711e513c05", + "ami": "ami-07eda52ffbd76a4c6", "ami_description": "CI Image of Windows 2022 x86_64", - "ami_name": "salt-project/ci/windows/2022/x86_64/20240509.1530", + "ami_name": "salt-project/ci/windows/2022/x86_64/20240913.1756", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.xlarge", diff --git a/cicd/shared-gh-workflows-context.yml b/cicd/shared-gh-workflows-context.yml index 9cd0641c739b..038071329d9e 100644 --- a/cicd/shared-gh-workflows-context.yml +++ b/cicd/shared-gh-workflows-context.yml @@ -4,7 +4,6 @@ relenv_version: "0.17.0" mandatory_os_slugs: - rockylinux-9 - amazonlinux-2023-arm64 - - archlinux-lts - photonos-5-arm64 - macos-12 - ubuntu-24.04-arm64 diff --git a/tools/ci.py b/tools/ci.py index 391214e69a38..0791fabe90bb 100644 --- a/tools/ci.py +++ b/tools/ci.py @@ -1010,7 +1010,6 @@ def get_pkg_downloads_matrix(ctx: Context): "photon", ) linux_skip_pkg_download_tests = ( - "archlinux-lts", "opensuse-15", "windows", ) diff --git a/tools/precommit/workflows.py b/tools/precommit/workflows.py index aa844b904ccc..b60371aa80cb 100644 --- a/tools/precommit/workflows.py +++ b/tools/precommit/workflows.py @@ -51,7 +51,6 @@ display_name="Amazon Linux 2023 Arm64", arch="arm64", ), - Linux(slug="archlinux-lts", display_name="Arch Linux LTS", arch="x86_64"), Linux(slug="debian-11", display_name="Debian 11", arch="x86_64"), Linux(slug="debian-11-arm64", display_name="Debian 11 Arm64", arch="arm64"), Linux(slug="debian-12", display_name="Debian 12", arch="x86_64"), From c54a8ced4fd4eeab482af3e76267659418770353 Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 16 Sep 2024 13:00:52 -0600 Subject: [PATCH 44/44] Test on systems where 7zip exists --- tests/pytests/functional/states/test_pkg.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/pytests/functional/states/test_pkg.py b/tests/pytests/functional/states/test_pkg.py index a56f06667e44..230491d77c60 100644 --- a/tests/pytests/functional/states/test_pkg.py +++ b/tests/pytests/functional/states/test_pkg.py @@ -58,7 +58,7 @@ def refresh_keys(grains, modules): def PKG_TARGETS(grains): _PKG_TARGETS = ["figlet", "sl"] if grains["os"] == "Windows": - _PKG_TARGETS = ["7zip", "putty"] + _PKG_TARGETS = ["npp_x64", "winrar"] elif grains["os"] == "Amazon": if grains["osfinger"] == "Amazon Linux-2023": _PKG_TARGETS = ["lynx", "gnuplot-minimal"] @@ -118,7 +118,7 @@ def PKG_32_TARGETS(grains): else: _PKG_32_TARGETS.append("xz-devel.i686") elif grains["os"] == "Windows": - _PKG_32_TARGETS = ["npp", "nsis"] + _PKG_32_TARGETS = ["npp", "putty"] if not _PKG_32_TARGETS: pytest.skip("No 32 bit packages have been specified for testing") return _PKG_32_TARGETS @@ -217,12 +217,16 @@ def install_7zip(modules): try: modules.pkg.install(name="7zip", version="22.01.00.0") modules.pkg.install(name="7zip", version="19.00.00.0") - assert modules.pkg.version("7zip") == "19.00.00.0,22.01.00.0" + versions = modules.pkg.version("7zip") + assert "19.00.00.0" in versions + assert "22.01.00.0" in versions yield finally: modules.pkg.remove(name="7zip", version="19.00.00.0") modules.pkg.remove(name="7zip", version="22.01.00.0") - assert modules.pkg.version("7zip") == "" + versions = modules.pkg.version("7zip") + assert "19.00.00.0" not in versions + assert "22.01.00.0" not in versions @pytest.mark.requires_salt_modules("pkg.version") @@ -1125,4 +1129,4 @@ def test_pkg_removed_with_version_multiple(install_7zip, modules, states): ret = states.pkg.removed(name="7zip", version="19.00.00.0") assert ret.result is True current = modules.pkg.version("7zip") - assert current == "22.01.00.0" + assert "22.01.00.0" in current