From 85b3cf90263cad645e9785d0c651f56c8f3836c2 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 5 Aug 2022 23:15:15 +0200 Subject: [PATCH] u-u: implement plugin system with `postrun()` functionality This commit implements a new plugin system. Plugins are python classes that start with `UnattendedUpgradesPlugin` and that are locates in one of the following directories: ``` /etc/unattended-upgrades/plugins/ /usr/share/unattended-upgrades/plugins ``` Plugins are also searched in the `UNATTENDED_UPGRADES_PLUGIN_PATH` environment path and in any of the directories in the apt configration: `Unattended-Upgrade::Dirs::Plugins::`. The `postrun()` function is called with a PluginDataPostrun python object that looks like a python dictionary. This dictionary contains the following elements: * "plugin_api": the API version as string of the form "1.0" * "hostname": The hostname of the machine that run u-u. * "success": A boolean that indicates if the run was successful * "result": A string with a human readable (and translated) status message * "packages_upgraded": A list of packages that got upgraded. * "packages_kept_back": A list of packages kept back. * "packages_kept_installed": A list of packages not auto-removed. * "reboot_required": Indicates a reboot is required. * "log_dpkg": The full dpkg log output. * "log_unattended_upgrades": The full unattended-upgrades log. This new code can be tested with: ``` $ sudo PYTHONPATH=. UNATTENDED_UPGRADES_PLUGIN_PATH=./examples/plugins/ ./unattended-upgrade ``` --- README.md | 6 + examples/plugins/simple.py | 31 +++++ test/root.unused-deps/usr/bin/dpkg | 3 + test/test_plugins.py | 90 ++++++++++++++ unattended-upgrade | 192 ++++++++++++++++++++++++++++- 5 files changed, 319 insertions(+), 3 deletions(-) create mode 100644 examples/plugins/simple.py create mode 100644 test/test_plugins.py diff --git a/README.md b/README.md index ad1da3a1..12253b2a 100644 --- a/README.md +++ b/README.md @@ -106,6 +106,12 @@ your needs. If you do not have this file, just create it or create/edit /etc/apt/apt.conf - you can check your configuration by running "apt-config dump". +Plugin support +-------------- + +Plugin support to integrate with e.g. webhooks or custom supporting is +available, check the [example plugin](https://github.com/mvo5/unattended-upgrades/blob/master/examples/plugins/simple.py) in the git repository. + Supported Options Reference --------------------------- diff --git a/examples/plugins/simple.py b/examples/plugins/simple.py new file mode 100644 index 00000000..115f733f --- /dev/null +++ b/examples/plugins/simple.py @@ -0,0 +1,31 @@ +import json + + +# Note the plugin name must start with UnattendedUpgradesPlugin* +# Copy the file into any of: +# +# /etc/unattended-upgrades/plugins/ +# /usr/share/unattended-upgrades/plugins +# +# or modify UNATTENDED_UPGRADES_PLUGIN_PATH to use a custom location. +class UnattendedUpgradesPluginExample: + """Example plugin for unattended-upgrades""" + + def postrun(self, result): + """ + The postrun function is run after an unattended-upgrades run + that generated some result. The result is a dict that is + kept backward compatible. + """ + # The data in result is a python class called PluginDataPostrun. + # It can be viewed via "pydoc3 /usr/bin/unattended-upgrades" + # and then searching for PluginDataPostrun. + # + # It also acts as a simple python dict that can easily + # be serialized to json. It contains information like what + # packages got upgraded, removed and kept back. Also the + # full u-u log and the dpkg log (if any). + # + # Here an example that serialized the data as json to a file + with open("simple-example-postrun-res.json", "w") as fp: + json.dump(result, fp) diff --git a/test/root.unused-deps/usr/bin/dpkg b/test/root.unused-deps/usr/bin/dpkg index 38a26c2c..b5fb5251 100755 --- a/test/root.unused-deps/usr/bin/dpkg +++ b/test/root.unused-deps/usr/bin/dpkg @@ -4,6 +4,7 @@ import os import sys import subprocess + if __name__ == "__main__": if "--unpack" in sys.argv: dpkg_status = os.path.join( @@ -14,3 +15,5 @@ if __name__ == "__main__": ["sed", "-i", "/Depends:\ test-package-dependency/d", dpkg_status]) subprocess.check_call( ["sed", "-i", "s/1.0.test.pkg/2.0.test.pkg/", dpkg_status]) + with open(dpkg_status, "r") as fp: + print(fp.read()) diff --git a/test/test_plugins.py b/test/test_plugins.py new file mode 100644 index 00000000..af04327a --- /dev/null +++ b/test/test_plugins.py @@ -0,0 +1,90 @@ +#!/usr/bin/python3 + +import json +import os +import shutil + +from unittest.mock import patch + +import unattended_upgrade +from test.test_base import TestBase, MockOptions + + +class TestPlugins(TestBase): + + def setUp(self): + TestBase.setUp(self) + # XXX: copy/clean to root.plugins + self.rootdir = self.make_fake_aptroot( + template=os.path.join(self.testdir, "root.unused-deps"), + fake_pkgs=[("test-package", "1.0.test.pkg", {})], + ) + # create a fake plugin dir, our fake plugin just writes a json + # dump of it's args + fake_plugin_dir = os.path.join(self.tempdir, "plugins") + os.makedirs(fake_plugin_dir, 0o755) + example_plugin = os.path.join( + os.path.dirname(__file__), "../examples/plugins/simple.py") + shutil.copy(example_plugin, fake_plugin_dir) + os.environ["UNATTENDED_UPGRADES_PLUGIN_PATH"] = fake_plugin_dir + self.addCleanup(lambda: os.environ.pop("UNATTENDED_UPGRADES_PLUGIN_PATH")) + # go to a tempdir as the simple.py plugin above will just write a + # file into cwd + os.chdir(self.tempdir) + + @patch("unattended_upgrade.reboot_required") + @patch("unattended_upgrade.host") + def test_plugin_happy(self, mock_host, mock_reboot_required): + mock_reboot_required.return_value = True + mock_host.return_value = "some-host" + options = MockOptions() + options.debug = False + unattended_upgrade.main(options, rootdir=self.rootdir) + with open("simple-example-postrun-res.json") as fp: + arg1 = json.load(fp) + # the log text needs extra processing + log_dpkg = arg1.pop("log_dpkg") + log_uu = arg1.pop("log_unattended_upgrades") + self.assertEqual(arg1, { + "plugin_api": "1.0", + "hostname": "some-host", + "success": True, + "result": "All upgrades installed", + "packages_upgraded": ["test-package"], + # XXX: add test for pkgs_kept_back + "packages_kept_back": [], + # XXX: add test for pkgs_kept_installed + "packages_kept_installed": [], + # XXX: add test for pkgs_removed + "packages_removed": [], + "reboot_required": True, + }) + self.assertTrue(log_dpkg.startswith("Log started:")) + self.assertTrue(log_uu.startswith("Starting unattended upgrades script")) + self.assertIn("Packages that will be upgraded: test-package", log_uu) + + def test_plugin_data_postrun(self): + res = unattended_upgrade.PluginDataPostrun({ + "plugin_api": "1.0", + "hostname": "some-host", + "success": True, + "result": "some result str", + "packages_upgraded": ["upgrade-pkg1", "upgrade-pkg2"], + "packages_kept_back": ["kept-pkg1"], + "packages_removed": ["rm-pkg1"], + "packages_kept_installed": ["kept-installed-pkg1"], + "reboot_required": False, + "log_dpkg": "a very long dpkg log", + }) + # ensure properties keep working + self.assertEqual(res.plugin_api, "1.0") + self.assertEqual(res.hostname, "some-host") + self.assertEqual(res.success, True) + self.assertEqual(res.result, "some result str") + self.assertEqual( + res.packages_upgraded, ["upgrade-pkg1", "upgrade-pkg2"]) + self.assertEqual(res.packages_kept_back, ["kept-pkg1"]) + self.assertEqual(res.packages_removed, ["rm-pkg1"]) + self.assertEqual(res.packages_kept_installed, ["kept-installed-pkg1"]) + self.assertEqual(res.reboot_required, False) + self.assertEqual(res.log_dpkg, "a very long dpkg log") diff --git a/unattended-upgrade b/unattended-upgrade index f4b4decf..b08601e1 100755 --- a/unattended-upgrade +++ b/unattended-upgrade @@ -31,12 +31,16 @@ import email.charset import fcntl import fnmatch import gettext +import glob try: from gi.repository.Gio import NetworkMonitor except ImportError: pass import grp +import inspect import io +from importlib.abc import Loader +import importlib.util import locale import logging import logging.handlers @@ -163,6 +167,176 @@ PkgPin = namedtuple('PkgPin', ['pkg', 'priority']) PkgFilePin = namedtuple('PkgFilePin', ['id', 'priority']) +class PluginDataPostrun(dict): + """PluginResultData represents the result of the unattended upgrade + operation for a plugin. + + It can be used as a dict or as a dataclass like object. + """ + + def __init__(self, dict): + # type: (dict) -> None + self.update(dict) + + @property + def plugin_api(self): + # type: () -> str + """The API for the plugin interface. + + Major versions break compatiblity, minor versions only add fields. + """ + return self["plugin_api"] + + @property + def hostname(self): + # type: () -> Optional[str] + """The hostname of the system that u-u ran on""" + return self["hostname"] + + @property + def success(self): + # type: () -> bool + """Boolean state if u-u ran successfully""" + return self["success"] + + @property + def result(self): + # type: () -> str + """The result of the operation as a human readable string""" + return self["result"] + + @property + def packages_upgraded(self): + # type () -> List[str] + """List of strings with the package names that got upgraded""" + return self.get("packages_upgraded") + + @property + def packages_kept_back(self): + # type () -> List[str] + """List of strings with the package names that were held back""" + return self.get("packages_kept_back") + + @property + def packages_removed(self): + # type () -> List[str] + """List of strings with the package names that got removed""" + return self.get("packages_removed") + + @property + def packages_kept_installed(self): + # type () -> List[str] + """List of strings with the package names that are kept on hold""" + return self.get("packages_kept_installed") + + @property + def reboot_required(self): + # type () -> boolean + """Boolean value if a reboot is required""" + return self.get("reboot_required") + + @property + def log_dpkg(self): + # type () -> str + """The full dpkg output as a string""" + return self.get("log_dpkg") + + @property + def log_unattended_upgrades(self): + # type () -> Optional[str] + """The full unattended-upgrades log output as a string""" + return self.get("log_unattended_upgrades") + + +class PluginManager: + def __init__(self): + # type: () -> None + _plugin_dirs = [ + "/etc/unattended-upgrades/plugins/", + "/usr/share/unattended-upgrades/plugins", + ] + cfg_plugin_paths = apt_pkg.config.value_list( + "Unattended-Upgrade::Dirs::Plugins") + if cfg_plugin_paths: + _plugin_dirs.extend(cfg_plugin_paths) + env_plugin_path = os.getenv("UNATTENDED_UPGRADES_PLUGIN_PATH") + if env_plugin_path: + _plugin_dirs.extend(env_plugin_path.split(":")) + self._plugins = self._load_plugins(_plugin_dirs) + + def _load_plugins(self, paths): + # type: (List[str]) -> AbstractSet[object] + logging.debug("loading plugins from {}".format(paths)) + plugins = set() + for path in paths: + for p in glob.glob(path + "/*.py"): + module_name = os.path.splitext(os.path.basename(p))[0] + try: + plugin_spec = importlib.util.spec_from_file_location(module_name, p) + if plugin_spec is None: + raise ValueError( + "plugin_spec for %s returned None" % module_name) + mod = importlib.util.module_from_spec(plugin_spec) + assert isinstance(plugin_spec.loader, Loader) + plugin_spec.loader.exec_module(mod) + except Exception as e: + logging.warn("cannot load plugin %s", e) + continue + for __, member in inspect.getmembers(mod): + # required class name prefix + prefix = "UnattendedUpgradesPlugin" + if inspect.isclass(member) and member.__name__.startswith(prefix): + logging.debug("adding plugin %s %s", mod, member) + plugins.add(member()) + return plugins + + def _call_plugin_func(self, func_name, *args): + # FIXME: add type annotations + for plugin in self._plugins: + plugin_func = getattr(plugin, func_name, None) + if plugin_func is not None: + try: + plugin_func(*args) + except Exception as e: + logging.warning('cannot run "%s" in plugin %s: %s' % ( + func_name, plugin, e)) + + def postrun(self, res, uu_log, dpkg_log): + # type: (UnattendedUpgradesResult, Optional[StringIO], str) -> None + + # this gets the current function name (e.g. "postrun") + func_name = sys._getframe().f_code.co_name + kept_back = set() + # XXX: add "package-kept-back-details" only if someone asks (YAGNI) + kept_back_details = [] + for origin, origin_pkgs in res.pkgs_kept_back.items(): + for pkg in origin_pkgs: + kept_back.add(pkg) + kept_back_details.append({"package": pkg, "origin": origin}) + # This is a custom dict because the expected main-use case for + # this is webhook support so json serialization should be + # trivial. straightforward. The properties are there mostly + # for documentation purposes. + # + # Keep in sync with the PluginResult properties + result = PluginDataPostrun({ + # increment minor version very time something is added + "plugin_api": "1.0", + "hostname": host(), + "success": res.success, + "result": res.result_str, + "packages_upgraded": res.pkgs, + "packages_kept_back": sorted(list(kept_back)), + "packages_removed": sorted(res.pkgs_removed), + "packages_kept_installed": sorted(res.pkgs_kept_installed), + "reboot_required": reboot_required(), + "log_dpkg": dpkg_log, + }) + if uu_log is not None: + result["log_unattended_upgrades"] = uu_log.getvalue() + self._call_plugin_func(func_name, result) + + class UnattendedUpgradesCache(apt.Cache): def __init__(self, rootdir): @@ -675,6 +849,11 @@ def is_dpkg_journal_dirty(): return False +def reboot_required(): + # type: () -> bool + return os.path.isfile(REBOOT_REQUIRED_FILE) + + def signal_handler(signal, frame): # type: (int, object) -> None logging.warning("SIGTERM received, will stop") @@ -1490,7 +1669,7 @@ def send_summary_mail(pkgs, # type: List[str] # Check if reboot-required flag is present reboot_flag_str = _( - "[reboot required]") if os.path.isfile(REBOOT_REQUIRED_FILE) else "" + "[reboot required]") if reboot_required() else "" # Check if packages are kept on hold hold_flag_str = (_("[package on hold]") if pkgs_kept_back or pkgs_kept_installed else "") @@ -1502,7 +1681,7 @@ def send_summary_mail(pkgs, # type: List[str] machine=host(), result="SUCCESS" if res else "FAILURE").strip() body = wrap_indent(_("Unattended upgrade result: %s") % result_str) body += "\n\n" - if os.path.isfile(REBOOT_REQUIRED_FILE): + if reboot_required(): body += _( "Warning: A reboot is required to complete this upgrade, " "or a previous one.\n\n") @@ -1650,7 +1829,9 @@ def _setup_logging(options): logger.setLevel(logging.INFO) stdout_handler = logging.StreamHandler(sys.stdout) logger.addHandler(stdout_handler) - if apt_pkg.config.find("Unattended-Upgrade::Mail", ""): + # XXX: we always need memory logging if we use plugins + # if apt_pkg.config.find("Unattended-Upgrade::Mail", ""): + if True: mem_log_handler = logging.StreamHandler(mem_log) logger.addHandler(mem_log_handler) # Configure syslog if necessary @@ -2041,6 +2222,8 @@ def main(options, rootdir="/"): logging.error("Lock file is already taken, exiting") return 1 + plugin_manager = PluginManager() + try: res = run(options, rootdir, mem_log, logfile_dpkg, install_start_time) @@ -2058,6 +2241,9 @@ def main(options, rootdir="/"): res.pkgs_kept_back, res.pkgs_removed, res.pkgs_kept_installed, mem_log, log_content) + # report results to any plugin + plugin_manager.postrun(res, mem_log, log_content) + if res.update_stamp: # write timestamp file write_stamp_file()