From fdb68e3d2835630ff404ab3f61044b32d8bfb73c Mon Sep 17 00:00:00 2001 From: Ferran Llamas Date: Tue, 28 Nov 2023 17:40:53 +0100 Subject: [PATCH] Standalone report versions and available updates (#1603) * simpler * simpler * final version * better naming * Simpler * Simpler * simpler * add cachetools dependency * add types * Increase coverage * Fix * simpler * rename * Add short timeout to pypi * simpler naming --- e2e/test_e2e.py | 11 ++ nucliadb/nucliadb/common/http_clients/pypi.py | 44 ++++++++ nucliadb/nucliadb/standalone/api_router.py | 14 +++ nucliadb/nucliadb/standalone/run.py | 17 ++- .../standalone/tests/unit/test_versions.py | 68 ++++++++++++ nucliadb/nucliadb/standalone/versions.py | 103 ++++++++++++++++++ .../tests/unit/http_clients/test_pypi.py | 52 +++++++++ nucliadb/nucliadb/train/servicer.py | 2 +- nucliadb/requirements.txt | 2 + 9 files changed, 310 insertions(+), 3 deletions(-) create mode 100644 nucliadb/nucliadb/common/http_clients/pypi.py create mode 100644 nucliadb/nucliadb/standalone/tests/unit/test_versions.py create mode 100644 nucliadb/nucliadb/standalone/versions.py create mode 100644 nucliadb/nucliadb/tests/unit/http_clients/test_pypi.py diff --git a/e2e/test_e2e.py b/e2e/test_e2e.py index 81be34b778..eed1984d16 100644 --- a/e2e/test_e2e.py +++ b/e2e/test_e2e.py @@ -64,6 +64,17 @@ def test_nodes_ready(): tries += 1 +def test_versions(): + resp = requests.get(os.path.join(BASE_URL, "api/v1/versions")) + resp.raise_for_status() + data = resp.json() + print(f"Versions: {data}") + assert data["nucliadb"]["installed"] + assert "latest" in data["nucliadb"] + assert data["nucliadb-admin-assets"]["installed"] + assert "latest" in data["nucliadb-admin-assets"] + + def test_config_check(kbid: str): resp = requests.get( os.path.join(BASE_URL, f"api/v1/config-check"), diff --git a/nucliadb/nucliadb/common/http_clients/pypi.py b/nucliadb/nucliadb/common/http_clients/pypi.py new file mode 100644 index 0000000000..de80839180 --- /dev/null +++ b/nucliadb/nucliadb/common/http_clients/pypi.py @@ -0,0 +1,44 @@ +# Copyright (C) 2021 Bosutech XXI S.L. +# +# nucliadb is offered under the AGPL v3.0 and as commercial software. +# For commercial licensing, contact us at info@nuclia.com. +# +# AGPL: +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +import httpx + +PYPI_JSON_API = "https://pypi.org/pypi/{package_name}/json" + + +class PyPi: + def __init__(self, timeout_seconds: int = 2): + self.session = httpx.AsyncClient(timeout=timeout_seconds) + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + await self.close() + + async def close(self): + await self.session.aclose() + + async def get_latest_version(self, package_name: str) -> str: + response = await self.session.get( + PYPI_JSON_API.format(package_name=package_name), + headers={"Accept": "application/json"}, + ) + response.raise_for_status() + return response.json()["info"]["version"] diff --git a/nucliadb/nucliadb/standalone/api_router.py b/nucliadb/nucliadb/standalone/api_router.py index 9b4fd90222..df7e2fbc10 100644 --- a/nucliadb/nucliadb/standalone/api_router.py +++ b/nucliadb/nucliadb/standalone/api_router.py @@ -29,6 +29,7 @@ from nucliadb.common.cluster import manager from nucliadb.common.http_clients.processing import ProcessingHTTPClient +from nucliadb.standalone import versions from nucliadb_models.resource import NucliaDBRoles from nucliadb_utils.authentication import requires from nucliadb_utils.settings import nuclia_settings @@ -123,3 +124,16 @@ async def ready(request: Request) -> JSONResponse: if len(manager.get_index_nodes()) == 0: return JSONResponse({"status": "not ready"}, status_code=503) return JSONResponse({"status": "ok"}) + + +@standalone_api_router.get("/versions") +async def versions_endpoint(request: Request) -> JSONResponse: + return JSONResponse( + { + package: { + "installed": versions.get_installed_version(package), + "latest": await versions.get_latest_version(package), + } + for package in versions.WatchedPackages + } + ) diff --git a/nucliadb/nucliadb/standalone/run.py b/nucliadb/nucliadb/standalone/run.py index 3c963ca976..0cf5c03625 100644 --- a/nucliadb/nucliadb/standalone/run.py +++ b/nucliadb/nucliadb/standalone/run.py @@ -17,17 +17,18 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . # +import asyncio import logging import os import sys -import pkg_resources import pydantic_argparse import uvicorn # type: ignore from fastapi import FastAPI from nucliadb.common.cluster.settings import settings as cluster_settings from nucliadb.ingest.settings import settings as ingest_settings +from nucliadb.standalone import versions from nucliadb.standalone.config import config_nucliadb from nucliadb.standalone.settings import Settings from nucliadb_telemetry import errors @@ -40,7 +41,7 @@ def setup() -> Settings: - errors.setup_error_handling(pkg_resources.get_distribution("nucliadb").version) + errors.setup_error_handling(versions.get_installed_version("nucliadb")) parser = pydantic_argparse.ArgumentParser( model=Settings, prog="NucliaDB", @@ -110,11 +111,23 @@ def run(): ] ) + installed_version = versions.installed_nucliadb() + loop = asyncio.get_event_loop() + latest_version = loop.run_until_complete(versions.latest_nucliadb()) + if latest_version is None: + version_info_fmted = f"{installed_version} (Update check failed)" + elif versions.nucliadb_updates_available(installed_version, latest_version): + version_info_fmted = f"{installed_version} (Update available: {latest_version})" + else: + version_info_fmted = installed_version + sys.stdout.write( f"""================================================= || || NucliaDB Standalone Server Running! || +|| Version: {version_info_fmted} +|| || Configuration: {settings_to_output_fmted} ================================================= diff --git a/nucliadb/nucliadb/standalone/tests/unit/test_versions.py b/nucliadb/nucliadb/standalone/tests/unit/test_versions.py new file mode 100644 index 0000000000..1073fe3945 --- /dev/null +++ b/nucliadb/nucliadb/standalone/tests/unit/test_versions.py @@ -0,0 +1,68 @@ +# Copyright (C) 2021 Bosutech XXI S.L. +# +# nucliadb is offered under the AGPL v3.0 and as commercial software. +# For commercial licensing, contact us at info@nuclia.com. +# +# AGPL: +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from unittest import mock + +import pkg_resources +import pytest + +from nucliadb.standalone.versions import ( + get_latest_version, + installed_nucliadb, + is_newer_release, + latest_nucliadb, +) + + +@pytest.mark.parametrize( + "installed,latest,expected", + [ + ("1.1.1", "1.1.1.post1", False), + ("1.1.1", "1.1.1", False), + ("1.1.1", "1.1.0", False), + ("1.1.1", "1.0.1", False), + ("1.1.1", "0.1.1", False), + ("1.1.1", "1.1.2", True), + ("1.1.1", "1.2.1", True), + ("1.1.1", "2.1.1", True), + ], +) +def test_is_newer_release(installed, latest, expected): + assert is_newer_release(installed, latest) is expected + + +def test_installed_nucliadb(): + pkg_resources.parse_version(installed_nucliadb()) + + +@pytest.fixture() +def pypi_mock(): + version = "1.0.0" + with mock.patch( + "nucliadb.standalone.versions._get_latest_version", return_value=version + ): + yield + + +async def test_latest_nucliadb(pypi_mock): + assert await latest_nucliadb() == "1.0.0" + + +async def test_get_latest_version(pypi_mock): + assert await get_latest_version("foobar") == "1.0.0" diff --git a/nucliadb/nucliadb/standalone/versions.py b/nucliadb/nucliadb/standalone/versions.py new file mode 100644 index 0000000000..c7f9a27e78 --- /dev/null +++ b/nucliadb/nucliadb/standalone/versions.py @@ -0,0 +1,103 @@ +# Copyright (C) 2021 Bosutech XXI S.L. +# +# nucliadb is offered under the AGPL v3.0 and as commercial software. +# For commercial licensing, contact us at info@nuclia.com. +# +# AGPL: +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +import enum +import logging +from typing import Optional + +import pkg_resources +from cachetools import TTLCache + +from nucliadb.common.http_clients.pypi import PyPi + +logger = logging.getLogger(__name__) + + +CACHE_TTL_SECONDS = 30 * 60 # 30 minutes +CACHE = TTLCache(maxsize=128, ttl=CACHE_TTL_SECONDS) # type: ignore + + +class StandalonePackages(enum.Enum): + NUCLIADB = "nucliadb" + NUCLIADB_ADMIN_ASSETS = "nucliadb-admin-assets" + + +WatchedPackages = [pkg.value for pkg in StandalonePackages] + + +def installed_nucliadb() -> str: + return get_installed_version(StandalonePackages.NUCLIADB.value) + + +async def latest_nucliadb() -> Optional[str]: + return await get_latest_version(StandalonePackages.NUCLIADB.value) + + +def nucliadb_updates_available(installed: str, latest: Optional[str]) -> bool: + if latest is None: + return False + return is_newer_release(installed, latest) + + +def is_newer_release(installed: str, latest: str) -> bool: + """ + Returns true if the latest version is newer than the installed version. + >>> is_newer_release("1.2.3", "1.2.4") + True + >>> is_newer_release("1.2.3", "1.2.3") + False + >>> is_newer_release("1.2.3", "1.2.3.post1") + False + """ + parsed_installed = pkg_resources.parse_version(_release(installed)) + parsed_latest = pkg_resources.parse_version(_release(latest)) + return parsed_latest > parsed_installed + + +def _release(version: str) -> str: + """ + Strips the .postX part of the version so that wecan compare major.minor.patch only. + + >>> _release("1.2.3") + '1.2.3' + >>> _release("1.2.3.post1") + '1.2.3' + """ + return version.split(".post")[0] + + +def get_installed_version(package_name: str) -> str: + return pkg_resources.get_distribution(package_name).version + + +async def get_latest_version(package: str) -> Optional[str]: + result = CACHE.get(package, None) + if result is None: + try: + result = await _get_latest_version(package) + except Exception as exc: + logger.warning(f"Error getting latest {package} version", exc_info=exc) + return None + CACHE[package] = result + return result + + +async def _get_latest_version(package_name: str) -> str: + async with PyPi() as pypi: + return await pypi.get_latest_version(package_name) diff --git a/nucliadb/nucliadb/tests/unit/http_clients/test_pypi.py b/nucliadb/nucliadb/tests/unit/http_clients/test_pypi.py new file mode 100644 index 0000000000..1f37ca6687 --- /dev/null +++ b/nucliadb/nucliadb/tests/unit/http_clients/test_pypi.py @@ -0,0 +1,52 @@ +# Copyright (C) 2021 Bosutech XXI S.L. +# +# nucliadb is offered under the AGPL v3.0 and as commercial software. +# For commercial licensing, contact us at info@nuclia.com. +# +# AGPL: +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +from unittest import mock + +import pytest + +from nucliadb.common.http_clients import pypi + + +class TestPyPi: + @pytest.fixture() + def response(self): + resp = mock.Mock() + resp.status = 200 + resp.json.return_value = {"info": {"version": "1.0.0"}} + yield resp + + @pytest.fixture() + def client(self, response): + cl = pypi.PyPi() + cl.session = mock.MagicMock() + cl.session.get = mock.AsyncMock() + cl.session.aclose = mock.AsyncMock() + cl.session.get.return_value = response + yield cl + + @pytest.mark.asyncio + async def test_get_latest_version(self, client): + assert await client.get_latest_version("foo") == "1.0.0" + + @pytest.mark.asyncio + async def test_context_manager(self, client): + async with client: + pass + client.session.aclose.assert_awaited_once() diff --git a/nucliadb/nucliadb/train/servicer.py b/nucliadb/nucliadb/train/servicer.py index 56ba8b480f..95d69dce2d 100644 --- a/nucliadb/nucliadb/train/servicer.py +++ b/nucliadb/nucliadb/train/servicer.py @@ -52,7 +52,7 @@ async def initialize(self): async def finalize(self): try: - self.session.close() + await self.session.close() except Exception: pass diff --git a/nucliadb/requirements.txt b/nucliadb/requirements.txt index 35e812afc4..cfc7bb2732 100644 --- a/nucliadb/requirements.txt +++ b/nucliadb/requirements.txt @@ -59,4 +59,6 @@ idna>=3.3 sniffio>=1.2.0 async_lru==2.0.2 +cachetools==5.3.2 +types-cachetools==5.3.0.5 kubernetes_asyncio