diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 24b70c64b..9a275a00c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -162,7 +162,7 @@ jobs: macos: # https://github.com/actions/runner-images/blob/main/images/macos/macos-13-Readme.md runs-on: macos-13 - timeout-minutes: 120 + timeout-minutes: 15 strategy: fail-fast: false diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index 1231acb5b..5053a5f5b 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -145,7 +145,7 @@ def _connect_ssh(spec): 'identity_file': private_key_file, 'identities_only': False, 'ssh_path': spec.ssh_executable(), - 'connect_timeout': spec.ansible_ssh_timeout(), + 'connect_timeout': spec.timeout(), 'ssh_args': spec.ssh_args(), 'ssh_debug_level': spec.mitogen_ssh_debug_level(), 'remote_name': get_remote_name(spec), @@ -169,7 +169,7 @@ def _connect_buildah(spec): 'username': spec.remote_user(), 'container': spec.remote_addr(), 'python_path': spec.python_path(), - 'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(), + 'connect_timeout': spec.timeout(), 'remote_name': get_remote_name(spec), } } @@ -185,7 +185,7 @@ def _connect_docker(spec): 'username': spec.remote_user(), 'container': spec.remote_addr(), 'python_path': spec.python_path(rediscover_python=True), - 'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(), + 'connect_timeout': spec.timeout(), 'remote_name': get_remote_name(spec), } } @@ -200,7 +200,7 @@ def _connect_kubectl(spec): 'kwargs': { 'pod': spec.remote_addr(), 'python_path': spec.python_path(), - 'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(), + 'connect_timeout': spec.timeout(), 'kubectl_path': spec.mitogen_kubectl_path(), 'kubectl_args': spec.extra_args(), 'remote_name': get_remote_name(spec), @@ -218,7 +218,7 @@ def _connect_jail(spec): 'username': spec.remote_user(), 'container': spec.remote_addr(), 'python_path': spec.python_path(), - 'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(), + 'connect_timeout': spec.timeout(), 'remote_name': get_remote_name(spec), } } @@ -234,7 +234,7 @@ def _connect_lxc(spec): 'container': spec.remote_addr(), 'python_path': spec.python_path(), 'lxc_attach_path': spec.mitogen_lxc_attach_path(), - 'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(), + 'connect_timeout': spec.timeout(), 'remote_name': get_remote_name(spec), } } @@ -250,7 +250,7 @@ def _connect_lxd(spec): 'container': spec.remote_addr(), 'python_path': spec.python_path(), 'lxc_path': spec.mitogen_lxc_path(), - 'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(), + 'connect_timeout': spec.timeout(), 'remote_name': get_remote_name(spec), } } @@ -273,7 +273,7 @@ def _connect_podman(spec): 'username': spec.remote_user(), 'container': spec.remote_addr(), 'python_path': spec.python_path(rediscover_python=True), - 'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(), + 'connect_timeout': spec.timeout(), 'remote_name': get_remote_name(spec), } } @@ -933,31 +933,39 @@ def reset(self): self.reset_compat_msg ) - # Strategy's _execute_meta doesn't have an action obj but we'll need one for - # running interpreter_discovery - # will create a new temporary action obj for this purpose - self._action = ansible_mitogen.mixins.ActionModuleMixin( - task=0, - connection=self, - play_context=self._play_context, - loader=0, - templar=0, - shared_loader_obj=0 - ) - - # Workaround for https://github.com/ansible/ansible/issues/84238 + # Handle templated connection variables during `meta: reset_connection`. + # Many bugs/implementation details of Mitogen & Ansible collide here. + # See #1079, #1096, #1132, ansible/ansible#84238, ... try: task, templar = self._play_context.vars.pop( '_mitogen.smuggled.reset_connection', ) except KeyError: - pass + self._action_monkey_patched_by_mitogen = False else: + # LOG.info('%r.reset(): remote_addr=%r', self, self._play_context.remote_addr) + # ansible.plugins.strategy.StrategyBase._execute_meta() doesn't + # have an action object, which we need for interpreter_discovery. + # Create a temporary action object for this purpose. + self._action = ansible_mitogen.mixins.ActionModuleMixin( + task=task, + connection=self, + play_context=self._play_context, + loader=templar._loader, + templar=templar, + shared_loader_obj=0, + ) + self._action_monkey_patched_by_mitogen = True + + # Workaround for https://github.com/ansible/ansible/issues/84238 self.set_options( task_keys=task.dump_attrs(), var_options=self._mitogen_var_options(templar), ) + del task + del templar + # Clear out state in case we were ever connected. self.close() @@ -977,6 +985,11 @@ def reset(self): finally: binding.close() + # Cleanup any monkey patching we did for `meta: reset_connection` + if self._action_monkey_patched_by_mitogen: + del self._action + del self._action_monkey_patched_by_mitogen + # Compatibility with Ansible 2.4 wait_for_connection plug-in. _reset = reset diff --git a/ansible_mitogen/strategy.py b/ansible_mitogen/strategy.py index c319f3e11..440e58112 100644 --- a/ansible_mitogen/strategy.py +++ b/ansible_mitogen/strategy.py @@ -328,8 +328,12 @@ def run(self, iterator, play_context, result=0): finally: ansible_mitogen.process.set_worker_model(None) - def _smuggle_to_connction_reset(self, task, play_context, iterator, target_host): - # Workaround for https://github.com/ansible/ansible/issues/84238 + def _smuggle_to_connection_reset(self, task, play_context, iterator, target_host): + """ + Create a templar and make it available for use in Connection.reset(). + This allows templated connection variables to be used when Mitogen + reconstructs its connection stack. + """ variables = self._variable_manager.get_vars( play=iterator._play, host=target_host, task=task, _hosts=self._hosts_cache, _hosts_all=self._hosts_cache_all, @@ -337,13 +341,29 @@ def _smuggle_to_connction_reset(self, task, play_context, iterator, target_host) templar = ansible.template.Templar( loader=self._loader, variables=variables, ) + + # Required for remote_user option set by variable (e.g. ansible_user). + # Without it remote_user in ansible.cfg gets used. + play_context = play_context.set_task_and_variable_override( + task=task, variables=variables, templar=templar, + ) + play_context.post_validate(templar=templar) + + # Required for timeout option set by variable (e.g. ansible_timeout). + # Without it the task timeout keyword (default: 0) gets used. + play_context.update_vars(variables) + + # Stash the task and templar somewhere Connection.reset() can find it play_context.vars.update({ '_mitogen.smuggled.reset_connection': (task, templar), }) + return play_context def _execute_meta(self, task, play_context, iterator, target_host): if task.args['_raw_params'] == 'reset_connection': - self._smuggle_to_connction_reset(task, play_context, iterator, target_host) + play_context = self._smuggle_to_connection_reset( + task, play_context, iterator, target_host, + ) return super(StrategyMixin, self)._execute_meta( task, play_context, iterator, target_host, diff --git a/ansible_mitogen/transport_config.py b/ansible_mitogen/transport_config.py index 2218a7fa7..effb4d62e 100644 --- a/ansible_mitogen/transport_config.py +++ b/ansible_mitogen/transport_config.py @@ -493,12 +493,24 @@ def port(self): return self._connection_option('port') def python_path(self, rediscover_python=False): - s = self._connection.get_task_var('ansible_python_interpreter') - # #511, #536: executor/module_common.py::_get_shebang() hard-wires - # "/usr/bin/python" as the default interpreter path if no other - # interpreter is specified. + # See also + # - ansible_mitogen.connecton.Connection.get_task_var() + try: + delegated_vars = self._task_vars['ansible_delegated_vars'] + variables = delegated_vars[self._connection.delegate_to_hostname] + except KeyError: + variables = self._task_vars + + interpreter_python = C.config.get_config_value( + 'INTERPRETER_PYTHON', variables=variables, + ) + + if '{{' in interpreter_python or '{%' in interpreter_python: + templar = self._connection.templar + interpreter_python = templar.template(interpreter_python) + return parse_python_path( - s, + interpreter_python, task_vars=self._task_vars, action=self._action, rediscover_python=rediscover_python) @@ -513,14 +525,10 @@ def ssh_executable(self): return self._connection_option('ssh_executable') def timeout(self): - return self._play_context.timeout + return self._connection_option('timeout') def ansible_ssh_timeout(self): - return ( - self._connection.get_task_var('ansible_timeout') or - self._connection.get_task_var('ansible_ssh_timeout') or - self.timeout() - ) + return self.timeout() def ssh_args(self): return [ diff --git a/docs/changelog.rst b/docs/changelog.rst index c92b69db9..f4a70ed0c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -18,6 +18,19 @@ To avail of fixes in an unreleased version, please download a ZIP file `directly from GitHub `_. +v0.3.20 (2025-01-07) +-------------------- + +* :gh:issue:`1079` :mod:`ansible_mitogen`: Fix :ans:mod:`wait_for_connection` + timeout with templated ``ansible_python_interpreter`` +* :gh:issue:`1079` :mod:`ansible_mitogen`: Fix templated python interpreter + with `meta: reset_connection` +* :gh:issue:`1083` :mod:`ansible_mitogen`: Templated connection timeout + (e.g. ``ansible_timeout``). +* :gh:issue:`740` :mod:`ansible_mitogen`: Respect ``interpreter_python`` + in ``ansible.cfg`` and ``ANSIBLE_PYTHON_INTERPRETER`` environment variable. + + v0.3.19 (2024-12-02) -------------------- diff --git a/mitogen/__init__.py b/mitogen/__init__.py index fa9b176de..08a3bc58a 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ #: Library version as a tuple. -__version__ = (0, 3, 19) +__version__ = (0, 3, 20) #: This is :data:`False` in slave contexts. Previously it was used to prevent diff --git a/tests/ansible/hosts/default.hosts b/tests/ansible/hosts/default.hosts index 17d1fd6d6..a99352b36 100644 --- a/tests/ansible/hosts/default.hosts +++ b/tests/ansible/hosts/default.hosts @@ -19,6 +19,9 @@ ssh-common-args ansible_host=localhost ansible_user="{{ lookup('pipe', 'whoami') ansible_ssh_common_args=-o PermitLocalCommand=yes -o LocalCommand="touch {{ ssh_args_canary_file }}" ssh_args_canary_file=/tmp/ssh_args_by_inv_{{ inventory_hostname }} +[issue1079] +wait-for-connection ansible_host=localhost ansible_user="{{ lookup('pipe', 'whoami') }}" + [tt_targets_bare] tt-bare @@ -47,5 +50,7 @@ tt-host-key-checking ansible_host=localhost ansible_host_key_checking= tt-password ansible_host=localhost ansible_password="{{ 'has_sudo_nopw_password' | trim }}" ansible_user=mitogen__has_sudo_nopw tt-port ansible_host=localhost ansible_password=has_sudo_nopw_password ansible_port="{{ 22 | int }}" ansible_user=mitogen__has_sudo_nopw tt-private-key-file ansible_host=localhost ansible_private_key_file="{{ git_basedir }}/tests/data/docker/mitogen__has_sudo_pubkey.key" ansible_user=mitogen__has_sudo_pubkey +tt-python-interpreter ansible_host=localhost ansible_password=has_sudo_nopw_password ansible_python_interpreter="{{ ansible_playbook_python | trim }}" ansible_user=mitogen__has_sudo_nopw tt-remote-user ansible_host=localhost ansible_password=has_sudo_nopw_password ansible_user="{{ 'mitogen__has_sudo_nopw' | trim }}" tt-ssh-executable ansible_host=localhost ansible_password=has_sudo_nopw_password ansible_ssh_executable="{{ 'ssh' | trim }}" ansible_user=mitogen__has_sudo_nopw +tt-timeout ansible_host=localhost ansible_password=has_sudo_nopw_password ansible_timeout="{{ 5 | int }}" ansible_user=mitogen__has_sudo_nopw diff --git a/tests/ansible/integration/ssh/templated_by_play_taskvar.yml b/tests/ansible/integration/ssh/templated_by_play_taskvar.yml index 4d7e318e9..c5c2e5443 100644 --- a/tests/ansible/integration/ssh/templated_by_play_taskvar.yml +++ b/tests/ansible/integration/ssh/templated_by_play_taskvar.yml @@ -7,6 +7,7 @@ ansible_password: "{{ 'has_sudo_nopw_password' | trim }}" ansible_port: "{{ hostvars[groups['test-targets'][0]].ansible_port | default(22) }}" ansible_ssh_executable: "{{ 'ssh' | trim }}" + ansible_timeout: "{{ 5 | int }}" ansible_user: "{{ 'mitogen__has_sudo_nopw' | trim }}" tasks: @@ -23,6 +24,7 @@ ansible_private_key_file: "{{ git_basedir }}/tests/data/docker/mitogen__has_sudo_pubkey.key" ansible_port: "{{ hostvars[groups['test-targets'][0]].ansible_port | default(22) }}" ansible_ssh_executable: "{{ 'ssh' | trim }}" + ansible_timeout: "{{ 5 | int }}" ansible_user: "{{ 'mitogen__has_sudo_pubkey' | trim }}" tasks: diff --git a/tests/ansible/regression/all.yml b/tests/ansible/regression/all.yml index a4272805f..a7c8033e4 100644 --- a/tests/ansible/regression/all.yml +++ b/tests/ansible/regression/all.yml @@ -16,4 +16,5 @@ - import_playbook: issue_776__load_plugins_called_twice.yml - import_playbook: issue_952__ask_become_pass.yml - import_playbook: issue_1066__add_host__host_key_checking.yml +- import_playbook: issue_1079__wait_for_connection_timeout.yml - import_playbook: issue_1087__template_streamerror.yml diff --git a/tests/ansible/regression/issue_1079__wait_for_connection_timeout.yml b/tests/ansible/regression/issue_1079__wait_for_connection_timeout.yml new file mode 100644 index 000000000..6ded03ea2 --- /dev/null +++ b/tests/ansible/regression/issue_1079__wait_for_connection_timeout.yml @@ -0,0 +1,22 @@ +- name: regression/issue_1079__wait_for_connection_timeout.yml + hosts: issue1079 + gather_facts: false + tasks: + - name: Wait for connection at start of play + wait_for_connection: + timeout: 5 + tags: + - issue_1079 + - wait_for_connection + +- hosts: issue1079 + gather_facts: false + tasks: + - meta: reset_connection + - name: Wait for connection after reset_connection + wait_for_connection: + timeout: 5 + tags: + - issue_1079 + - reset_connection + - wait_for_connection diff --git a/tests/ansible/regression/issue_109__target_has_old_ansible_installed.yml b/tests/ansible/regression/issue_109__target_has_old_ansible_installed.yml index 064832ece..a7ae0908c 100644 --- a/tests/ansible/regression/issue_109__target_has_old_ansible_installed.yml +++ b/tests/ansible/regression/issue_109__target_has_old_ansible_installed.yml @@ -26,6 +26,7 @@ - env.cwd == ansible_user_dir - (not env.mitogen_loaded) or (env.python_path.count("") == 1) fail_msg: | + ansible_user_dir={{ ansible_user_dir }} env={{ env }} - name: Run some new-style from ansible.module_utils... modules diff --git a/tests/ansible/templates/test-targets.j2 b/tests/ansible/templates/test-targets.j2 index bb0d85ece..063847ccb 100644 --- a/tests/ansible/templates/test-targets.j2 +++ b/tests/ansible/templates/test-targets.j2 @@ -40,6 +40,13 @@ ssh_args_canary_file=/tmp/ssh_args_by_inv_{{ '{{' }} inventory_hostname {{ '}}' {% set tt = containers[0] %} +[issue1079] +wait-for-connection ansible_host={{ tt.hostname }} ansible_port={{ tt.port }} ansible_python_interpreter="{{ '{{' }} '{{ tt.python_path }}' | trim {{ '}}' }}" + +[issue1079:vars] +ansible_user=mitogen__has_sudo_nopw +ansible_password=has_sudo_nopw_password + [tt_targets_bare] tt-bare @@ -77,5 +84,7 @@ tt-host-key-checking ansible_host={{ tt.hostname }} ansible_host_key_c tt-password ansible_host={{ tt.hostname }} ansible_password="{{ '{{' }} 'has_sudo_nopw_password' | trim {{ '}}' }}" ansible_port={{ tt.port }} ansible_python_interpreter={{ tt.python_path }} ansible_user=mitogen__has_sudo_nopw tt-port ansible_host={{ tt.hostname }} ansible_password=has_sudo_nopw_password ansible_port="{{ '{{' }} {{ tt.port }} | int {{ '}}' }}" ansible_python_interpreter={{ tt.python_path }} ansible_user=mitogen__has_sudo_nopw tt-private-key-file ansible_host={{ tt.hostname }} ansible_port={{ tt.port }} ansible_private_key_file="{{ '{{' }} git_basedir {{ '}}' }}/tests/data/docker/mitogen__has_sudo_pubkey.key" ansible_python_interpreter={{ tt.python_path }} ansible_user=mitogen__has_sudo_pubkey +tt-python-interpreter ansible_host={{ tt.hostname }} ansible_port={{ tt.port }} ansible_password=has_sudo_nopw_password ansible_python_interpreter="{{ '{{' }} '{{ tt.python_path }}' | trim {{ '}}' }}" ansible_user=mitogen__has_sudo_nopw tt-remote-user ansible_host={{ tt.hostname }} ansible_password=has_sudo_nopw_password ansible_port={{ tt.port }} ansible_python_interpreter={{ tt.python_path }} ansible_user="{{ '{{' }} 'mitogen__has_sudo_nopw' | trim {{ '}}' }}" tt-ssh-executable ansible_host={{ tt.hostname }} ansible_password=has_sudo_nopw_password ansible_port={{ tt.port }} ansible_python_interpreter={{ tt.python_path }} ansible_ssh_executable="{{ '{{' }} 'ssh' | trim {{ '}}' }}" ansible_user=mitogen__has_sudo_nopw +tt-timeout ansible_host={{ tt.hostname }} ansible_password=has_sudo_nopw_password ansible_port={{ tt.port }} ansible_python_interpreter={{ tt.python_path }} ansible_timeout="{{ '{{' }} 5 | int {{ '}}' }}" ansible_user=mitogen__has_sudo_nopw