u-u: implement plugin system with postrun() functionality

This commit implements a new plugin system. Plugins are python
classes that start with `UnattendedUpgradesPlugin` and that are
locates in one of the following directories:
```
/etc/unattended-upgrades/plugins/
/usr/share/unattended-upgrades/plugins
```
Plugins are also searched in the `UNATTENDED_UPGRADES_PLUGIN_PATH`
environment path and in any of the directories in the apt configration:
`Unattended-Upgrade::Dirs::Plugins::`.

The `postrun()` function is called with a PluginDataPostrun python
object that looks like a python dictionary.
This dictionary contains the following elements:
* "plugin_api": the API version as string of the form "1.0"
* "hostname": The hostname of the machine that run u-u.
* "success": A boolean that indicates if the run was successful
* "result": A string with a human readable (and translated) status message
* "packages_upgraded": A list of packages that got upgraded.
* "packages_kept_back": A list of packages kept back.
* "packages_kept_installed": A list of packages not auto-removed.
* "reboot_required": Indicates a reboot is required.
* "log_dpkg": The full dpkg log output.
* "log_unattended_upgrades": The full unattended-upgrades log.

This new code can be tested with:
```
$ sudo PYTHONPATH=. UNATTENDED_UPGRADES_PLUGIN_PATH=./examples/plugins/ ./unattended-upgrade
```

README.md

@@ -106,6 +106,13 @@ your needs. If you do not have this file, just create it or
 create/edit /etc/apt/apt.conf - you can check your configuration by
 running "apt-config dump".
 
+Plugin support
+--------------
+
+Plugin support is available to integrate with webhooks or other custom
+tools that need to read u-u run results. Check the [example plugin](https://github.com/mvo5/unattended-upgrades/blob/master/examples/plugins/simple.py)
+in the git repository for more details.
+
 
 Supported Options Reference
 ---------------------------

examples/plugins/simple.py

@@ -0,0 +1,31 @@
+import json
+
+
+# Note the plugin name must start with UnattendedUpgradesPlugin*
+# Copy the file into any of:
+#
+#     /etc/unattended-upgrades/plugins/
+#     /usr/share/unattended-upgrades/plugins
+#
+# or modify UNATTENDED_UPGRADES_PLUGIN_PATH to use a custom location.
+class UnattendedUpgradesPluginExample:
+    """Example plugin for unattended-upgrades"""
+
+    def postrun(self, result):
+        """
+        The postrun function is run after an unattended-upgrades run
+        that generated some result. The result is a dict that is
+        kept backward compatible.
+        """
+        # The data in result is a python class called PluginDataPostrun.
+        # It can be viewed via "pydoc3 /usr/bin/unattended-upgrades"
+        # and then searching for PluginDataPostrun.
+        #
+        # It also acts as a simple python dict that can easily
+        # be serialized to json. It contains information like what
+        # packages got upgraded, removed and kept back. Also the
+        # full u-u log and the dpkg log (if any).
+        #
+        # Here an example that serialized the data as json to a file
+        with open("simple-example-postrun-res.json", "w") as fp:
+            json.dump(result, fp)

test/root.unused-deps/usr/bin/dpkg

@@ -4,6 +4,7 @@ import os
 import sys
 import subprocess
 
+
 if __name__ == "__main__":
    if "--unpack" in sys.argv:
       dpkg_status = os.path.join(
@@ -14,3 +15,5 @@ if __name__ == "__main__":
           ["sed", "-i", "/Depends:\ test-package-dependency/d", dpkg_status])
       subprocess.check_call(
           ["sed", "-i", "s/1.0.test.pkg/2.0.test.pkg/", dpkg_status])
+      with open(dpkg_status, "r") as fp:
+         print(fp.read())

test/test_plugins.py

@@ -0,0 +1,90 @@
+#!/usr/bin/python3
+
+import json
+import os
+import shutil
+
+from unittest.mock import patch
+
+import unattended_upgrade
+from test.test_base import TestBase, MockOptions
+
+
+class TestPlugins(TestBase):
+
+    def setUp(self):
+        TestBase.setUp(self)
+        # XXX: copy/clean to root.plugins
+        self.rootdir = self.make_fake_aptroot(
+            template=os.path.join(self.testdir, "root.unused-deps"),
+            fake_pkgs=[("test-package", "1.0.test.pkg", {})],
+        )
+        # create a fake plugin dir, our fake plugin just writes a json
+        # dump of it's args
+        fake_plugin_dir = os.path.join(self.tempdir, "plugins")
+        os.makedirs(fake_plugin_dir, 0o755)
+        example_plugin = os.path.join(
+            os.path.dirname(__file__), "../examples/plugins/simple.py")
+        shutil.copy(example_plugin, fake_plugin_dir)
+        os.environ["UNATTENDED_UPGRADES_PLUGIN_PATH"] = fake_plugin_dir
+        self.addCleanup(lambda: os.environ.pop("UNATTENDED_UPGRADES_PLUGIN_PATH"))
+        # go to a tempdir as the simple.py plugin above will just write a
+        # file into cwd
+        os.chdir(self.tempdir)
+
+    @patch("unattended_upgrade.reboot_required")
+    @patch("unattended_upgrade.host")
+    def test_plugin_happy(self, mock_host, mock_reboot_required):
+        mock_reboot_required.return_value = True
+        mock_host.return_value = "some-host"
+        options = MockOptions()
+        options.debug = False
+        unattended_upgrade.main(options, rootdir=self.rootdir)
+        with open("simple-example-postrun-res.json") as fp:
+            arg1 = json.load(fp)
+        # the log text needs extra processing
+        log_dpkg = arg1.pop("log_dpkg")
+        log_uu = arg1.pop("log_unattended_upgrades")
+        self.assertEqual(arg1, {
+            "plugin_api": "1.0",
+            "hostname": "some-host",
+            "success": True,
+            "result": "All upgrades installed",
+            "packages_upgraded": ["test-package"],
+            # XXX: add test for pkgs_kept_back
+            "packages_kept_back": [],
+            # XXX: add test for pkgs_kept_installed
+            "packages_kept_installed": [],
+            # XXX: add test for pkgs_removed
+            "packages_removed": [],
+            "reboot_required": True,
+        })
+        self.assertTrue(log_dpkg.startswith("Log started:"))
+        self.assertTrue(log_uu.startswith("Starting unattended upgrades script"))
+        self.assertIn("Packages that will be upgraded: test-package", log_uu)
+
+    def test_plugin_data_postrun(self):
+        res = unattended_upgrade.PluginDataPostrun({
+            "plugin_api": "1.0",
+            "hostname": "some-host",
+            "success": True,
+            "result": "some result str",
+            "packages_upgraded": ["upgrade-pkg1", "upgrade-pkg2"],
+            "packages_kept_back": ["kept-pkg1"],
+            "packages_removed": ["rm-pkg1"],
+            "packages_kept_installed": ["kept-installed-pkg1"],
+            "reboot_required": False,
+            "log_dpkg": "a very long dpkg log",
+        })
+        # ensure properties keep working
+        self.assertEqual(res.plugin_api, "1.0")
+        self.assertEqual(res.hostname, "some-host")
+        self.assertEqual(res.success, True)
+        self.assertEqual(res.result, "some result str")
+        self.assertEqual(
+            res.packages_upgraded, ["upgrade-pkg1", "upgrade-pkg2"])
+        self.assertEqual(res.packages_kept_back, ["kept-pkg1"])
+        self.assertEqual(res.packages_removed, ["rm-pkg1"])
+        self.assertEqual(res.packages_kept_installed, ["kept-installed-pkg1"])
+        self.assertEqual(res.reboot_required, False)
+        self.assertEqual(res.log_dpkg, "a very long dpkg log")