Standalone report versions and available updates (#1603)

* simpler

* simpler

* final version

* better naming

* Simpler

* Simpler

* simpler

* add cachetools dependency

* add types

* Increase coverage

* Fix

* simpler

* rename

* Add short timeout to pypi

* simpler naming

e2e/test_e2e.py

@@ -64,6 +64,17 @@ def test_nodes_ready():
             tries += 1
 
 
+def test_versions():
+    resp = requests.get(os.path.join(BASE_URL, "api/v1/versions"))
+    resp.raise_for_status()
+    data = resp.json()
+    print(f"Versions: {data}")
+    assert data["nucliadb"]["installed"]
+    assert "latest" in data["nucliadb"]
+    assert data["nucliadb-admin-assets"]["installed"]
+    assert "latest" in data["nucliadb-admin-assets"]
+
+
 def test_config_check(kbid: str):
     resp = requests.get(
         os.path.join(BASE_URL, f"api/v1/config-check"),

nucliadb/nucliadb/common/http_clients/pypi.py

@@ -0,0 +1,44 @@
+# Copyright (C) 2021 Bosutech XXI S.L.
+#
+# nucliadb is offered under the AGPL v3.0 and as commercial software.
+# For commercial licensing, contact us at [email protected].
+#
+# AGPL:
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import httpx
+
+PYPI_JSON_API = "https://pypi.org/pypi/{package_name}/json"
+
+
+class PyPi:
+    def __init__(self, timeout_seconds: int = 2):
+        self.session = httpx.AsyncClient(timeout=timeout_seconds)
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, *exc):
+        await self.close()
+
+    async def close(self):
+        await self.session.aclose()
+
+    async def get_latest_version(self, package_name: str) -> str:
+        response = await self.session.get(
+            PYPI_JSON_API.format(package_name=package_name),
+            headers={"Accept": "application/json"},
+        )
+        response.raise_for_status()
+        return response.json()["info"]["version"]

nucliadb/nucliadb/standalone/api_router.py

@@ -29,6 +29,7 @@
 
 from nucliadb.common.cluster import manager
 from nucliadb.common.http_clients.processing import ProcessingHTTPClient
+from nucliadb.standalone import versions
 from nucliadb_models.resource import NucliaDBRoles
 from nucliadb_utils.authentication import requires
 from nucliadb_utils.settings import nuclia_settings
@@ -123,3 +124,16 @@ async def ready(request: Request) -> JSONResponse:
     if len(manager.get_index_nodes()) == 0:
         return JSONResponse({"status": "not ready"}, status_code=503)
     return JSONResponse({"status": "ok"})
+
+
+@standalone_api_router.get("/versions")
+async def versions_endpoint(request: Request) -> JSONResponse:
+    return JSONResponse(
+        {
+            package: {
+                "installed": versions.get_installed_version(package),
+                "latest": await versions.get_latest_version(package),
+            }
+            for package in versions.WatchedPackages
+        }
+    )

nucliadb/nucliadb/standalone/run.py

@@ -17,17 +17,18 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
+import asyncio
 import logging
 import os
 import sys
 
-import pkg_resources
 import pydantic_argparse
 import uvicorn  # type: ignore
 from fastapi import FastAPI
 
 from nucliadb.common.cluster.settings import settings as cluster_settings
 from nucliadb.ingest.settings import settings as ingest_settings
+from nucliadb.standalone import versions
 from nucliadb.standalone.config import config_nucliadb
 from nucliadb.standalone.settings import Settings
 from nucliadb_telemetry import errors
@@ -40,7 +41,7 @@
 
 
 def setup() -> Settings:
-    errors.setup_error_handling(pkg_resources.get_distribution("nucliadb").version)
+    errors.setup_error_handling(versions.get_installed_version("nucliadb"))
     parser = pydantic_argparse.ArgumentParser(
         model=Settings,
         prog="NucliaDB",
@@ -110,11 +111,23 @@ def run():
         ]
     )
 
+    installed_version = versions.installed_nucliadb()
+    loop = asyncio.get_event_loop()
+    latest_version = loop.run_until_complete(versions.latest_nucliadb())
+    if latest_version is None:
+        version_info_fmted = f"{installed_version} (Update check failed)"
+    elif versions.nucliadb_updates_available(installed_version, latest_version):
+        version_info_fmted = f"{installed_version} (Update available: {latest_version})"
+    else:
+        version_info_fmted = installed_version
+
     sys.stdout.write(
         f"""=================================================
 ||
 ||   NucliaDB Standalone Server Running!
 ||
+||   Version: {version_info_fmted}
+||
 ||   Configuration:
 {settings_to_output_fmted}
 =================================================

nucliadb/nucliadb/standalone/tests/unit/test_versions.py

@@ -0,0 +1,68 @@
+# Copyright (C) 2021 Bosutech XXI S.L.
+#
+# nucliadb is offered under the AGPL v3.0 and as commercial software.
+# For commercial licensing, contact us at [email protected].
+#
+# AGPL:
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+from unittest import mock
+
+import pkg_resources
+import pytest
+
+from nucliadb.standalone.versions import (
+    get_latest_version,
+    installed_nucliadb,
+    is_newer_release,
+    latest_nucliadb,
+)
+
+
+@pytest.mark.parametrize(
+    "installed,latest,expected",
+    [
+        ("1.1.1", "1.1.1.post1", False),
+        ("1.1.1", "1.1.1", False),
+        ("1.1.1", "1.1.0", False),
+        ("1.1.1", "1.0.1", False),
+        ("1.1.1", "0.1.1", False),
+        ("1.1.1", "1.1.2", True),
+        ("1.1.1", "1.2.1", True),
+        ("1.1.1", "2.1.1", True),
+    ],
+)
+def test_is_newer_release(installed, latest, expected):
+    assert is_newer_release(installed, latest) is expected
+
+
+def test_installed_nucliadb():
+    pkg_resources.parse_version(installed_nucliadb())
+
+
+@pytest.fixture()
+def pypi_mock():
+    version = "1.0.0"
+    with mock.patch(
+        "nucliadb.standalone.versions._get_latest_version", return_value=version
+    ):
+        yield
+
+
+async def test_latest_nucliadb(pypi_mock):
+    assert await latest_nucliadb() == "1.0.0"
+
+
+async def test_get_latest_version(pypi_mock):
+    assert await get_latest_version("foobar") == "1.0.0"

nucliadb/nucliadb/standalone/versions.py

@@ -0,0 +1,103 @@
+# Copyright (C) 2021 Bosutech XXI S.L.
+#
+# nucliadb is offered under the AGPL v3.0 and as commercial software.
+# For commercial licensing, contact us at [email protected].
+#
+# AGPL:
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import enum
+import logging
+from typing import Optional
+
+import pkg_resources
+from cachetools import TTLCache
+
+from nucliadb.common.http_clients.pypi import PyPi
+
+logger = logging.getLogger(__name__)
+
+
+CACHE_TTL_SECONDS = 30 * 60  # 30 minutes
+CACHE = TTLCache(maxsize=128, ttl=CACHE_TTL_SECONDS)  # type: ignore
+
+
+class StandalonePackages(enum.Enum):
+    NUCLIADB = "nucliadb"
+    NUCLIADB_ADMIN_ASSETS = "nucliadb-admin-assets"
+
+
+WatchedPackages = [pkg.value for pkg in StandalonePackages]
+
+
+def installed_nucliadb() -> str:
+    return get_installed_version(StandalonePackages.NUCLIADB.value)
+
+
+async def latest_nucliadb() -> Optional[str]:
+    return await get_latest_version(StandalonePackages.NUCLIADB.value)
+
+
+def nucliadb_updates_available(installed: str, latest: Optional[str]) -> bool:
+    if latest is None:
+        return False
+    return is_newer_release(installed, latest)
+
+
+def is_newer_release(installed: str, latest: str) -> bool:
+    """
+    Returns true if the latest version is newer than the installed version.
+    >>> is_newer_release("1.2.3", "1.2.4")
+    True
+    >>> is_newer_release("1.2.3", "1.2.3")
+    False
+    >>> is_newer_release("1.2.3", "1.2.3.post1")
+    False
+    """
+    parsed_installed = pkg_resources.parse_version(_release(installed))
+    parsed_latest = pkg_resources.parse_version(_release(latest))
+    return parsed_latest > parsed_installed
+
+
+def _release(version: str) -> str:
+    """
+    Strips the .postX part of the version so that wecan compare major.minor.patch only.
+
+    >>> _release("1.2.3")
+    '1.2.3'
+    >>> _release("1.2.3.post1")
+    '1.2.3'
+    """
+    return version.split(".post")[0]
+
+
+def get_installed_version(package_name: str) -> str:
+    return pkg_resources.get_distribution(package_name).version
+
+
+async def get_latest_version(package: str) -> Optional[str]:
+    result = CACHE.get(package, None)
+    if result is None:
+        try:
+            result = await _get_latest_version(package)
+        except Exception as exc:
+            logger.warning(f"Error getting latest {package} version", exc_info=exc)
+            return None
+        CACHE[package] = result
+    return result
+
+
+async def _get_latest_version(package_name: str) -> str:
+    async with PyPi() as pypi:
+        return await pypi.get_latest_version(package_name)

nucliadb/nucliadb/tests/unit/http_clients/test_pypi.py

@@ -0,0 +1,52 @@
+# Copyright (C) 2021 Bosutech XXI S.L.
+#
+# nucliadb is offered under the AGPL v3.0 and as commercial software.
+# For commercial licensing, contact us at [email protected].
+#
+# AGPL:
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from unittest import mock
+
+import pytest
+
+from nucliadb.common.http_clients import pypi
+
+
+class TestPyPi:
+    @pytest.fixture()
+    def response(self):
+        resp = mock.Mock()
+        resp.status = 200
+        resp.json.return_value = {"info": {"version": "1.0.0"}}
+        yield resp
+
+    @pytest.fixture()
+    def client(self, response):
+        cl = pypi.PyPi()
+        cl.session = mock.MagicMock()
+        cl.session.get = mock.AsyncMock()
+        cl.session.aclose = mock.AsyncMock()
+        cl.session.get.return_value = response
+        yield cl
+
+    @pytest.mark.asyncio
+    async def test_get_latest_version(self, client):
+        assert await client.get_latest_version("foo") == "1.0.0"
+
+    @pytest.mark.asyncio
+    async def test_context_manager(self, client):
+        async with client:
+            pass
+        client.session.aclose.assert_awaited_once()

nucliadb/nucliadb/train/servicer.py

@@ -52,7 +52,7 @@ async def initialize(self):
 
     async def finalize(self):
         try:
-            self.session.close()
+            await self.session.close()
         except Exception:
             pass
 

nucliadb/requirements.txt

@@ -59,4 +59,6 @@ idna>=3.3
 sniffio>=1.2.0
 
 async_lru==2.0.2
+cachetools==5.3.2
+types-cachetools==5.3.0.5
 kubernetes_asyncio