From d9341267eb556e953bf30697f13f0f4d677caccf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20L=C3=B6sche?= Date: Wed, 17 Jul 2024 10:49:55 +0200 Subject: [PATCH 1/4] Bump libs --- requirements-test.txt | 70 +++++++++++++++++++++---------------------- requirements.txt | 10 +++---- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/requirements-test.txt b/requirements-test.txt index 2b8e084..17e79cb 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,16 +1,16 @@ -astroid==3.0.0 +astroid==3.2.3 # via pylint -attrs==23.1.0 +attrs==23.2.0 # via hypothesis -black==23.9.1 +black==24.4.2 # via fixbackup (pyproject.toml) -boto3==1.28.63 +boto3==1.34.144 # via fixbackup (pyproject.toml) -botocore==1.31.63 +botocore==1.34.144 # via # boto3 # s3transfer -cachetools==5.3.1 +cachetools==5.4.0 # via tox chardet==5.2.0 # via tox @@ -18,27 +18,27 @@ click==8.1.7 # via black colorama==0.4.6 # via tox -coverage[toml]==7.3.2 +coverage[toml]==7.6.0 # via # fixbackup (pyproject.toml) # pytest-cov -dill==0.3.7 +dill==0.3.8 # via pylint -distlib==0.3.7 +distlib==0.3.8 # via virtualenv -filelock==3.12.4 +filelock==3.15.4 # via # tox # virtualenv -flake8==6.1.0 +flake8==7.1.0 # via # fixbackup (pyproject.toml) # pep8-naming -hypothesis==6.87.4 +hypothesis==6.108.2 # via fixbackup (pyproject.toml) iniconfig==2.0.0 # via pytest -isort==5.12.0 +isort==5.13.2 # via pylint jmespath==1.0.1 # via @@ -48,68 +48,68 @@ mccabe==0.7.0 # via # flake8 # pylint -mypy==1.6.0 +mypy==1.10.1 # via fixbackup (pyproject.toml) mypy-extensions==1.0.0 # via # black # mypy -packaging==23.2 +packaging==24.1 # via # black # pyproject-api # pytest # tox -pathspec==0.11.2 +pathspec==0.12.1 # via black -pep8-naming==0.13.3 +pep8-naming==0.14.1 # via fixbackup (pyproject.toml) -platformdirs==3.11.0 +platformdirs==4.2.2 # via # black # pylint # tox # virtualenv -pluggy==1.3.0 +pluggy==1.5.0 # via # pytest # tox -pycodestyle==2.11.0 +pycodestyle==2.12.0 # via flake8 -pyflakes==3.1.0 +pyflakes==3.2.0 # via flake8 -pylint==3.0.1 +pylint==3.2.5 # via fixbackup (pyproject.toml) -pyproject-api==1.6.1 +pyproject-api==1.7.1 # via tox -pytest==7.4.2 +pytest==8.2.2 # via # fixbackup (pyproject.toml) # pytest-asyncio # pytest-cov -pytest-asyncio==0.21.1 +pytest-asyncio==0.23.7 # via fixbackup (pyproject.toml) -pytest-cov==4.1.0 +pytest-cov==5.0.0 # via fixbackup (pyproject.toml) -pytest-runner==6.0.0 +pytest-runner==6.0.1 # via fixbackup (pyproject.toml) -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 # via botocore -s3transfer==0.7.0 +s3transfer==0.10.2 # via boto3 six==1.16.0 # via python-dateutil sortedcontainers==2.4.0 # via hypothesis -tomlkit==0.12.1 +tomlkit==0.13.0 # via pylint -tox==4.11.3 +tox==4.16.0 # via fixbackup (pyproject.toml) -typing-extensions==4.8.0 +typing-extensions==4.12.2 # via mypy -urllib3==2.0.7 +urllib3==2.2.2 # via botocore -virtualenv==20.24.5 +virtualenv==20.26.3 # via tox -wheel==0.41.2 +wheel==0.43.0 # via fixbackup (pyproject.toml) diff --git a/requirements.txt b/requirements.txt index 699b539..88d2235 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ -boto3==1.28.63 +boto3==1.34.144 # via fixbackup (pyproject.toml) -botocore==1.31.63 +botocore==1.34.144 # via # boto3 # s3transfer @@ -8,11 +8,11 @@ jmespath==1.0.1 # via # boto3 # botocore -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 # via botocore -s3transfer==0.7.0 +s3transfer==0.10.2 # via boto3 six==1.16.0 # via python-dateutil -urllib3==2.0.7 +urllib3==2.2.2 # via botocore From 8305ef964dcdf072a0cf738901c2f5f5487063ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20L=C3=B6sche?= Date: Wed, 17 Jul 2024 11:13:38 +0200 Subject: [PATCH 2/4] Add PostgreSQL support --- Dockerfile | 1 + fixbackup/__init__.py | 2 +- fixbackup/backup/__init__.py | 16 ++++- fixbackup/backup/postgresql.py | 109 +++++++++++++++++++++++++++++++++ pyproject.toml | 2 +- 5 files changed, 127 insertions(+), 3 deletions(-) create mode 100644 fixbackup/backup/postgresql.py diff --git a/Dockerfile b/Dockerfile index ff99036..477e95e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -68,6 +68,7 @@ RUN groupadd -g "${PGID:-0}" -o fix \ python3-pip \ redis-tools \ mysql-client \ + postgresql \ && dpkg -i /usr/local/tmp/arangodb3-client_*.deb \ && ln -s /usr/bin/busybox /usr/local/bin/vi \ && ln -s /usr/bin/busybox /usr/local/bin/wget \ diff --git a/fixbackup/__init__.py b/fixbackup/__init__.py index 16dbf2b..448516a 100644 --- a/fixbackup/__init__.py +++ b/fixbackup/__init__.py @@ -11,4 +11,4 @@ __author__ = "Some Engineering Inc." __license__ = "Apache 2.0" __copyright__ = "Copyright © 2023 Some Engineering Inc." -__version__ = "0.0.8" +__version__ = "0.0.9" diff --git a/fixbackup/backup/__init__.py b/fixbackup/backup/__init__.py index 427cd75..0be2ba5 100644 --- a/fixbackup/backup/__init__.py +++ b/fixbackup/backup/__init__.py @@ -4,10 +4,11 @@ from typing import List, Tuple from .redis import backup as redis_backup, add_args as redis_add_args from .mysql import backup as mysql_backup, add_args as mysql_add_args +from .postgresql import backup as pg_backup, add_args as postgresql_add_args from .arangodb import backup as arangodb_backup, add_args as arangodb_add_args from ..utils import valid_hostname, valid_ip, valid_dbname -add_args = [redis_add_args, mysql_add_args, arangodb_add_args] +add_args = [redis_add_args, mysql_add_args, arangodb_add_args, postgresql_add_args] def backup(args: Namespace, backup_directory: Path) -> Tuple[List[Path], bool]: @@ -39,6 +40,19 @@ def backup(args: Namespace, backup_directory: Path) -> Tuple[List[Path], bool]: else: all_success = False + if args.pg_host and (valid_hostname(args.pg_host) or valid_ip(args.pg_host)): + if args.pg_database: + db = str(args.pg_database) + if not valid_dbname(db): + raise ValueError(f"Invalid database name: {db}") + else: + db = "all" + pg_backup_file = backup_directory / f"{environment}-{date_prefix}-mysql-{args.pg_host}-{db}.sql.gz" + if pg_backup(args, pg_backup_file): + result.append(pg_backup_file) + else: + all_success = False + if args.arangodb_host and (valid_hostname(args.arangodb_host)): if args.arangodb_database: db = str(args.arangodb_database) diff --git a/fixbackup/backup/postgresql.py b/fixbackup/backup/postgresql.py new file mode 100644 index 0000000..879905a --- /dev/null +++ b/fixbackup/backup/postgresql.py @@ -0,0 +1,109 @@ +import os +import subprocess +from pathlib import Path +from argparse import ArgumentParser, Namespace +from ..utils import BackupFile +from ..logger import log + + +def add_args(arg_parser: ArgumentParser) -> None: + arg_parser.add_argument( + "--pg-host", + help="PostgreSQL host", + dest="pg_host", + type=str, + default=os.getenv("PG_HOST"), + ) + + arg_parser.add_argument( + "--pg-port", + help="PostgreSQL port", + dest="pg_port", + type=int, + default=os.getenv("PG_PORT", 5432), + ) + + arg_parser.add_argument( + "--pg-user", + help="PostgreSQL user", + dest="gp_user", + type=str, + default=os.getenv("PG_USER", "root"), + ) + + arg_parser.add_argument( + "--pg-password", + help="PostgreSQL password", + dest="pg_password", + type=str, + default=os.getenv("PG_PASSWORD"), + ) + + arg_parser.add_argument( + "--pg-database", + help="PostgreSQL database", + dest="pg_database", + type=str, + default=os.getenv("PG_DATABASE"), + ) + + arg_parser.add_argument( + "--pg-dump-args", + help="Extra arguments to pass to pg_dump", + dest="pg_dump_args", + action="append", + default=[], + ) + + +def backup(args: Namespace, backup_file_path: Path, timeout: int = 900, compress: bool = True) -> bool: + log.info("Starting PostgreSQL backup...") + + if not args.pg_host: + return False + + env = os.environ.copy() + command = [ + "pg_dump", + "-w", + "--if-exists", + "--inserts", + "-h", + str(args.pg_host), + "-p", + str(args.pg_port), + "-u", + str(args.pg_user), + *args.pg_dump_args, + ] + if args.pg_database: + command.append("-d") + command.append(args.pg_database) + else: + command[0] = "pg_dumpall" + + if args.pg_password: + env["PGPASSWORD"] = args.pg_password + + log.debug(f"Running command: {' '.join(command)}") + + try: + with BackupFile(backup_file_path, compress) as backup_fd: + process = subprocess.Popen(command, stdout=backup_fd, stderr=subprocess.PIPE, env=env) + _, stderr = process.communicate(timeout=timeout) + + if process.returncode == 0: + log.info(f"PostgreSQL backup completed successfully. Saved to {backup_file_path}") + if stderr: + log.debug(stderr.decode().strip()) + return True + else: + log.error(f"PostgreSQL backup failed with return code: {process.returncode}") + if stderr: + log.error(stderr.decode().strip()) + except subprocess.TimeoutExpired: + log.error(f"PostgreSQL backup failed with timeout after {timeout} seconds") + process.kill() + process.communicate() + + return False diff --git a/pyproject.toml b/pyproject.toml index fdeac9f..9010efe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "fixbackup" -version = "0.0.8" +version = "0.0.9" authors = [{name="Some Engineering Inc."}] description = "FIX Database Backup System" license = {file="LICENSE"} From 8e2f42af002188dd3d5267f4ca9ad2848b315054 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20L=C3=B6sche?= Date: Wed, 17 Jul 2024 11:14:33 +0200 Subject: [PATCH 3/4] black --- fixbackup/logger.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/fixbackup/logger.py b/fixbackup/logger.py index ee093ad..3703a5a 100644 --- a/fixbackup/logger.py +++ b/fixbackup/logger.py @@ -126,20 +126,15 @@ def log_to_root(message: str, *args: Any, **kwargs: Any) -> None: class FixLogger(Logger): - def debug2(self, msg: str, *args: Any, **kwargs: Any) -> None: - ... + def debug2(self, msg: str, *args: Any, **kwargs: Any) -> None: ... - def debug3(self, msg: str, *args: Any, **kwargs: Any) -> None: - ... + def debug3(self, msg: str, *args: Any, **kwargs: Any) -> None: ... - def debug4(self, msg: str, *args: Any, **kwargs: Any) -> None: - ... + def debug4(self, msg: str, *args: Any, **kwargs: Any) -> None: ... - def debug5(self, msg: str, *args: Any, **kwargs: Any) -> None: - ... + def debug5(self, msg: str, *args: Any, **kwargs: Any) -> None: ... - def trace(self, msg: str, *args: Any, **kwargs: Any) -> None: - ... + def trace(self, msg: str, *args: Any, **kwargs: Any) -> None: ... def get_fix_logger(name: Optional[str] = None) -> FixLogger: From cc17678b454b2fd7d4556ab13bf504dcbc4ae506 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20L=C3=B6sche?= Date: Wed, 17 Jul 2024 11:19:54 +0200 Subject: [PATCH 4/4] Turn off black caused E704 errors in flake8 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 4e86489..ea4c504 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ env_list = syntax, tests, black, flake8, mypy [flake8] max-line-length = 120 exclude = .git,.tox,__pycache__,.idea,.pytest_cache,venv -ignore = F401, F403, F405, F811, E722, N806, N813, E266, W503, E203 +ignore = F401, F403, F405, E704, F811, E722, N806, N813, E266, W503, E203 [pytest] testpaths = test