Merge pull request #1164 from moreati/release-v0.3.14

Release v0.3.14

.ci/README.md

@@ -28,14 +28,15 @@ for doing `setup.py install` while pulling a Docker container, for example.
 
 ### Environment Variables
 
-* `DISTRO`: the `mitogen_` tests need a target Docker container distro. This
-  name comes from the Docker Hub `mitogen` user, i.e. `mitogen/$DISTRO-test`
-* `DISTROS`: the `ansible_` tests can run against multiple targets
-  simultaneously, which speeds things up. This is a space-separated list of
-  DISTRO names, but additionally, supports:
+* `MITOGEN_TEST_DISTRO_SPECS`: a space delimited list of distro specs to run
+  the tests against. (e.g. `centos6 ubuntu2004-py3*4`). Each spec determines
+  the Linux distribution, target Python interepreter & number of instances.
+  Only distributions with a pre-built Linux container image can be used.
     * `debian-py3`: when generating Ansible inventory file, set
       `ansible_python_interpreter` to `python3`, i.e. run a test where the
       target interpreter is Python 3.
     * `debian*16`: generate 16 Docker containers running Debian. Also works
       with -py3.
 
+* `MITOGEN_TEST_IMAGE_TEMPLATE`: specifies the Linux container image name,
+  and hence the container registry used for test targets.

.ci/ansible_tests.py

@@ -35,7 +35,7 @@ def pause_if_interactive():
 
 
 with ci_lib.Fold('docker_setup'):
-    containers = ci_lib.container_specs(ci_lib.DISTROS)
+    containers = ci_lib.container_specs(ci_lib.DISTRO_SPECS.split())
     ci_lib.start_containers(containers)
 
 

.ci/ci_lib.py

@@ -28,6 +28,10 @@
 )
 
 
+DISTRO_SPECS = os.environ.get(
+    'MITOGEN_TEST_DISTRO_SPECS',
+    'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004',
+)
 IMAGE_TEMPLATE = os.environ.get(
     'MITOGEN_TEST_IMAGE_TEMPLATE',
     'public.ecr.aws/n5z0e8q9/%(distro)s-test',
@@ -196,10 +200,6 @@ def __exit__(self, _1, _2, _3): pass
 
 
 GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
-# Used only when MODE=mitogen
-DISTRO = os.environ.get('DISTRO', 'debian9')
-# Used only when MODE=ansible
-DISTROS = os.environ.get('DISTROS', 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004').split()
 TMP = TempDir().path
 
 

.ci/localhost_ansible_tests.py

@@ -3,8 +3,6 @@
 
 from __future__ import print_function
 
-import getpass
-import io
 import os
 import subprocess
 import sys
@@ -57,18 +55,6 @@
         os.chdir(IMAGE_PREP_DIR)
         ci_lib.run("ansible-playbook -c local -i localhost, _user_accounts.yml")
 
-    # FIXME Don't hardcode https://github.com/mitogen-hq/mitogen/issues/1022
-    #       and os.environ['USER'] is not populated on Azure macOS runners.
-    os.chdir(HOSTS_DIR)
-    with io.open('default.hosts', 'r+', encoding='utf-8') as f:
-        user = getpass.getuser()
-        content = f.read()
-        content = content.replace("{{ lookup('pipe', 'whoami') }}", user)
-        f.seek(0)
-        f.write(content)
-        f.truncate()
-    ci_lib.dump_file('default.hosts')
-
     cmd = ';'.join([
         'from __future__ import print_function',
         'import os, sys',

.ci/mitogen_py24_tests.py

@@ -8,8 +8,6 @@
 os.environ.update({
     'NOCOVERAGE': '1',
     'UNIT2': '/usr/local/python2.4.6/bin/unit2',
-
-    'MITOGEN_TEST_DISTRO': ci_lib.DISTRO,
     'MITOGEN_LOG_LEVEL': 'debug',
     'SKIP_ANSIBLE': '1',
 })

.ci/mitogen_tests.py

@@ -6,7 +6,6 @@
 import ci_lib
 
 os.environ.update({
-    'MITOGEN_TEST_DISTRO': ci_lib.DISTRO,
     'MITOGEN_LOG_LEVEL': 'debug',
     'SKIP_ANSIBLE': '1',
 })

.github/workflows/tests.yml

@@ -67,80 +67,14 @@ jobs:
             python_version: '3.13'
             tox_env: py313-mode_ansible-ansible10-strategy_linear
 
-          - name: Mito_27_centos6
-            tox_env: py27-mode_mitogen-distro_centos6
-          - name: Mito_27_centos7
-            tox_env: py27-mode_mitogen-distro_centos7
-          - name: Mito_27_centos8
-            tox_env: py27-mode_mitogen-distro_centos8
-          - name: Mito_27_debian9
-            tox_env: py27-mode_mitogen-distro_debian9
-          - name: Mito_27_debian10
-            tox_env: py27-mode_mitogen-distro_debian10
-          - name: Mito_27_debian11
-            tox_env: py27-mode_mitogen-distro_debian11
-          - name: Mito_27_ubuntu1604
-            tox_env: py27-mode_mitogen-distro_ubuntu1604
-          - name: Mito_27_ubuntu1804
-            tox_env: py27-mode_mitogen-distro_ubuntu1804
-          - name: Mito_27_ubuntu2004
-            tox_env: py27-mode_mitogen-distro_ubuntu2004
-
-          - name: Mito_36_centos6
-            python_version: '3.6'
-            tox_env: py36-mode_mitogen-distro_centos6
-          - name: Mito_36_centos7
-            python_version: '3.6'
-            tox_env: py36-mode_mitogen-distro_centos7
-          - name: Mito_36_centos8
-            python_version: '3.6'
-            tox_env: py36-mode_mitogen-distro_centos8
-          - name: Mito_36_debian9
-            python_version: '3.6'
-            tox_env: py36-mode_mitogen-distro_debian9
-          - name: Mito_36_debian10
-            python_version: '3.6'
-            tox_env: py36-mode_mitogen-distro_debian10
-          - name: Mito_36_debian11
-            python_version: '3.6'
-            tox_env: py36-mode_mitogen-distro_debian11
-          - name: Mito_36_ubuntu1604
-            python_version: '3.6'
-            tox_env: py36-mode_mitogen-distro_ubuntu1604
-          - name: Mito_36_ubuntu1804
-            python_version: '3.6'
-            tox_env: py36-mode_mitogen-distro_ubuntu1804
-          - name: Mito_36_ubuntu2004
+          - name: Mito_27
+            tox_env: py27-mode_mitogen
+          - name: Mito_36
             python_version: '3.6'
-            tox_env: py36-mode_mitogen-distro_ubuntu2004
-
-          - name: Mito_313_centos6
-            python_version: '3.13'
-            tox_env: py313-mode_mitogen-distro_centos6
-          - name: Mito_313_centos7
-            python_version: '3.13'
-            tox_env: py313-mode_mitogen-distro_centos7
-          - name: Mito_313_centos8
-            python_version: '3.13'
-            tox_env: py313-mode_mitogen-distro_centos8
-          - name: Mito_313_debian9
-            python_version: '3.13'
-            tox_env: py313-mode_mitogen-distro_debian9
-          - name: Mito_313_debian10
-            python_version: '3.13'
-            tox_env: py313-mode_mitogen-distro_debian10
-          - name: Mito_313_debian11
-            python_version: '3.13'
-            tox_env: py313-mode_mitogen-distro_debian11
-          - name: Mito_313_ubuntu1604
-            python_version: '3.13'
-            tox_env: py313-mode_mitogen-distro_ubuntu1604
-          - name: Mito_313_ubuntu1804
-            python_version: '3.13'
-            tox_env: py313-mode_mitogen-distro_ubuntu1804
-          - name: Mito_313_ubuntu2004
+            tox_env: py36-mode_mitogen
+          - name: Mito_313
             python_version: '3.13'
-            tox_env: py313-mode_mitogen-distro_ubuntu2004
+            tox_env: py313-mode_mitogen
 
     steps:
       - uses: actions/checkout@v4

ansible_mitogen/connection.py

@@ -814,7 +814,7 @@ def _connect_stack(self, stack):
 
         self.context = dct['context']
         self.chain = CallChain(self, self.context, pipelined=True)
-        if self._play_context.become:
+        if self.become:
             self.login_context = dct['via']
         else:
             self.login_context = self.context
@@ -926,7 +926,7 @@ def reset(self):
         self.close()
 
         inventory_name, stack = self._build_stack()
-        if self._play_context.become:
+        if self.become:
             stack = stack[:-1]
 
         worker_model = ansible_mitogen.process.get_worker_model()

ansible_mitogen/mixins.py

@@ -294,7 +294,7 @@ def _remote_expand_user(self, path, sudoable=True):
         if not path.startswith('~'):
             # /home/foo -> /home/foo
             return path
-        if sudoable or not self._play_context.become:
+        if sudoable or not self._connection.become:
             if path == '~':
                 # ~ -> /home/dmw
                 return self._connection.homedir

ansible_mitogen/transport_config.py

@@ -417,6 +417,10 @@ def __init__(self, connection, play_context, transport, inventory_name):
         # used to run interpreter discovery
         self._action = connection._action
 
+    def _become_option(self, name):
+        plugin = self._connection.become
+        return plugin.get_option(name, self._task_vars, self._play_context)
+
     def _connection_option(self, name):
         try:
             return self._connection.get_option(name, hostvars=self._task_vars)
@@ -437,13 +441,13 @@ def remote_user(self):
         return self._connection_option('remote_user')
 
     def become(self):
-        return self._play_context.become
+        return self._connection.become
 
     def become_method(self):
         return self._play_context.become_method
 
     def become_user(self):
-        return self._play_context.become_user
+        return self._become_option('become_user')
 
     def become_pass(self):
         # become_pass is owned/provided by the active become plugin. However

docs/changelog.rst

@@ -18,6 +18,14 @@ To avail of fixes in an unreleased version, please download a ZIP file
 `directly from GitHub <https://github.com/mitogen-hq/mitogen/>`_.
 
 
+v0.3.14 (2024-10-16)
+--------------------
+
+* :gh:issue:`1159` CI: Reduce number of Jobs by parameterizing Mitogen Docker
+  SSH tests
+* :gh:issue:`1083` :mod:`ansible_mitogen`: Support templated become username.
+
+
 v0.3.13 (2024-10-09)
 --------------------
 

docs/contributors.rst

@@ -134,6 +134,7 @@ sponsorship and outstanding future-thinking of its early adopters.
     <li>luto</li>
     <li><a href="https://mayeu.me/">Mayeu a.k.a Matthieu Maury</a></li>
     <li><a href="https://github.com/madsi1m">Michael D'Silva</a></li>
+    <li><a href="https://github.com/mordekasg">mordek</a></li>
     <li><a href="https://twitter.com/nathanhruby">@nathanhruby</a></li>
     <li><a href="https://github.com/opoplawski">Orion Poplawski</a></li>
     <li><a href="https://github.com/philfry">Philippe Kueck</a></li>

mitogen/__init__.py

@@ -35,7 +35,7 @@
 
 
 #: Library version as a tuple.
-__version__ = (0, 3, 13)
+__version__ = (0, 3, 14)
 
 
 #: This is :data:`False` in slave contexts. Previously it was used to prevent

tests/README.md

@@ -30,11 +30,19 @@ and run the tests there.
 1. Run ``test``
 
 
-# Selecting a target distribution
+# Selecting target distributions
 
-Docker target images exist for testing against CentOS and Debian, with the
-default being Debian. To select CentOS, specify `MITOGEN_TEST_DISTRO=centos` in
-the environment.
+Linux container images for testing are available at
+
+- https://github.com/orgs/mitogen-hq/packages
+- https://public.ecr.aws/n5z0e8q9
+
+The images used are determined by two environment variables
+
+- `MITOGEN_TEST_DISTRO_SPECS`
+- `MITOGEN_TEST_IMAGE_TEMPLATE`
+
+Defaults for these can be found in `.ci/ci_lib.py` & `tests/testlib.py`
 
 
 # User Accounts

tests/ansible/hosts/default.hosts

@@ -4,8 +4,7 @@
 # When running the tests outside CI, make a single 'target' host which is the
 # local machine. The ansible_user override is necessary since some tests want a
 # fixed ansible.cfg remote_user setting to test against.
-# FIXME Hardcoded by replacement in some CI runs https://github.com/mitogen-hq/mitogen/issues/1022
-#       and os.environ['USER'] is not populated on Azure macOS runners.
+# os.environ['USER'] is an empty string on GitHub Actions macOS runners.
 target ansible_host=localhost ansible_user="{{ lookup('pipe', 'whoami') }}"
 
 [test-targets]
@@ -26,6 +25,20 @@ tt-bare
 [tt_targets_bare:vars]
 ansible_host=localhost
 
+[tt_become_bare]
+tt-become-bare
+
+[tt_become_bare:vars]
+ansible_host=localhost
+ansible_user="{{ lookup('pipe', 'whoami') }}"
+
+[tt_become_by_inv]
+tt-become-user              ansible_become=true ansible_become_user="{{ 'root' | trim }}"
+
+[tt_become_by_inv:vars]
+ansible_host=localhost
+ansible_user="{{ lookup('pipe', 'whoami') }}"
+
 [tt_targets_inventory]
 tt-password                 ansible_password="{{ 'has_sudo_nopw_password' | trim }}" ansible_user=mitogen__has_sudo_nopw
 tt-port                     ansible_password=has_sudo_nopw_password ansible_port="{{ 22 | int }}" ansible_user=mitogen__has_sudo_nopw

tests/ansible/integration/become/all.yml

@@ -5,3 +5,7 @@
 - import_playbook: sudo_nopassword.yml
 - import_playbook: sudo_password.yml
 - import_playbook: sudo_requiretty.yml
+- import_playbook: templated_by_inv.yml
+- import_playbook: templated_by_play_keywords.yml
+- import_playbook: templated_by_play_vars.yml
+- import_playbook: templated_by_task_keywords.yml

tests/ansible/integration/become/templated_by_inv.yml

@@ -0,0 +1,14 @@
+- name: integration/become/templated_by_inv.yml
+  hosts: tt_become_by_inv
+  gather_facts: false
+  tasks:
+    - meta: reset_connection
+    - name: Templated become in inventory
+      command:
+        cmd: whoami
+      changed_when: false
+      check_mode: false
+      register: become_templated_by_inv_whoami
+      failed_when:
+        - become_templated_by_inv_whoami is failed
+          or become_templated_by_inv_whoami.stdout != 'root'

tests/ansible/integration/become/templated_by_play_keywords.yml

@@ -0,0 +1,16 @@
+- name: integration/become/templated_by_play_keywords.yml
+  hosts: tt_become_bare
+  gather_facts: false
+  become: true
+  become_user: "{{ 'root' | trim }}"
+  tasks:
+    - meta: reset_connection
+    - name: Templated become by play keywords
+      command:
+        cmd: whoami
+      changed_when: false
+      check_mode: false
+      register: become_templated_by_play_keywords_whoami
+      failed_when:
+        - become_templated_by_play_keywords_whoami is failed
+          or become_templated_by_play_keywords_whoami.stdout != 'root'

tests/ansible/integration/become/templated_by_play_vars.yml

@@ -0,0 +1,16 @@
+- name: integration/become/templated_by_play_vars.yml
+  hosts: tt_become_bare
+  gather_facts: false
+  vars:
+    ansible_become: true
+    ansible_become_user: "{{ 'root' | trim }}"
+  tasks:
+    - name: Templated become by play vars
+      command:
+        cmd: whoami
+      changed_when: false
+      check_mode: false
+      register: become_templated_by_play_vars_whoami
+      failed_when:
+        - become_templated_by_play_vars_whoami is failed
+          or become_templated_by_play_vars_whoami.stdout != 'root'