Merge pull request #5 from someengineering/lloesche/pgdump

Add Postgresql backup

Dockerfile

@@ -68,6 +68,7 @@ RUN groupadd -g "${PGID:-0}" -o fix \
         python3-pip \
         redis-tools \
         mysql-client \
+        postgresql \
     && dpkg -i /usr/local/tmp/arangodb3-client_*.deb \
     && ln -s /usr/bin/busybox /usr/local/bin/vi \
     && ln -s /usr/bin/busybox /usr/local/bin/wget \

fixbackup/__init__.py

@@ -11,4 +11,4 @@
 __author__ = "Some Engineering Inc."
 __license__ = "Apache 2.0"
 __copyright__ = "Copyright © 2023 Some Engineering Inc."
-__version__ = "0.0.8"
+__version__ = "0.0.9"

fixbackup/backup/__init__.py

@@ -4,10 +4,11 @@
 from typing import List, Tuple
 from .redis import backup as redis_backup, add_args as redis_add_args
 from .mysql import backup as mysql_backup, add_args as mysql_add_args
+from .postgresql import backup as pg_backup, add_args as postgresql_add_args
 from .arangodb import backup as arangodb_backup, add_args as arangodb_add_args
 from ..utils import valid_hostname, valid_ip, valid_dbname
 
-add_args = [redis_add_args, mysql_add_args, arangodb_add_args]
+add_args = [redis_add_args, mysql_add_args, arangodb_add_args, postgresql_add_args]
 
 
 def backup(args: Namespace, backup_directory: Path) -> Tuple[List[Path], bool]:
@@ -39,6 +40,19 @@ def backup(args: Namespace, backup_directory: Path) -> Tuple[List[Path], bool]:
         else:
             all_success = False
 
+    if args.pg_host and (valid_hostname(args.pg_host) or valid_ip(args.pg_host)):
+        if args.pg_database:
+            db = str(args.pg_database)
+            if not valid_dbname(db):
+                raise ValueError(f"Invalid database name: {db}")
+        else:
+            db = "all"
+        pg_backup_file = backup_directory / f"{environment}-{date_prefix}-mysql-{args.pg_host}-{db}.sql.gz"
+        if pg_backup(args, pg_backup_file):
+            result.append(pg_backup_file)
+        else:
+            all_success = False
+
     if args.arangodb_host and (valid_hostname(args.arangodb_host)):
         if args.arangodb_database:
             db = str(args.arangodb_database)

fixbackup/backup/postgresql.py

@@ -0,0 +1,109 @@
+import os
+import subprocess
+from pathlib import Path
+from argparse import ArgumentParser, Namespace
+from ..utils import BackupFile
+from ..logger import log
+
+
+def add_args(arg_parser: ArgumentParser) -> None:
+    arg_parser.add_argument(
+        "--pg-host",
+        help="PostgreSQL host",
+        dest="pg_host",
+        type=str,
+        default=os.getenv("PG_HOST"),
+    )
+
+    arg_parser.add_argument(
+        "--pg-port",
+        help="PostgreSQL port",
+        dest="pg_port",
+        type=int,
+        default=os.getenv("PG_PORT", 5432),
+    )
+
+    arg_parser.add_argument(
+        "--pg-user",
+        help="PostgreSQL user",
+        dest="gp_user",
+        type=str,
+        default=os.getenv("PG_USER", "root"),
+    )
+
+    arg_parser.add_argument(
+        "--pg-password",
+        help="PostgreSQL password",
+        dest="pg_password",
+        type=str,
+        default=os.getenv("PG_PASSWORD"),
+    )
+
+    arg_parser.add_argument(
+        "--pg-database",
+        help="PostgreSQL database",
+        dest="pg_database",
+        type=str,
+        default=os.getenv("PG_DATABASE"),
+    )
+
+    arg_parser.add_argument(
+        "--pg-dump-args",
+        help="Extra arguments to pass to pg_dump",
+        dest="pg_dump_args",
+        action="append",
+        default=[],
+    )
+
+
+def backup(args: Namespace, backup_file_path: Path, timeout: int = 900, compress: bool = True) -> bool:
+    log.info("Starting PostgreSQL backup...")
+
+    if not args.pg_host:
+        return False
+
+    env = os.environ.copy()
+    command = [
+        "pg_dump",
+        "-w",
+        "--if-exists",
+        "--inserts",
+        "-h",
+        str(args.pg_host),
+        "-p",
+        str(args.pg_port),
+        "-u",
+        str(args.pg_user),
+        *args.pg_dump_args,
+    ]
+    if args.pg_database:
+        command.append("-d")
+        command.append(args.pg_database)
+    else:
+        command[0] = "pg_dumpall"
+
+    if args.pg_password:
+        env["PGPASSWORD"] = args.pg_password
+
+    log.debug(f"Running command: {' '.join(command)}")
+
+    try:
+        with BackupFile(backup_file_path, compress) as backup_fd:
+            process = subprocess.Popen(command, stdout=backup_fd, stderr=subprocess.PIPE, env=env)
+            _, stderr = process.communicate(timeout=timeout)
+
+            if process.returncode == 0:
+                log.info(f"PostgreSQL backup completed successfully. Saved to {backup_file_path}")
+                if stderr:
+                    log.debug(stderr.decode().strip())
+                return True
+            else:
+                log.error(f"PostgreSQL backup failed with return code: {process.returncode}")
+                if stderr:
+                    log.error(stderr.decode().strip())
+    except subprocess.TimeoutExpired:
+        log.error(f"PostgreSQL backup failed with timeout after {timeout} seconds")
+        process.kill()
+        process.communicate()
+
+    return False

fixbackup/logger.py

@@ -126,20 +126,15 @@ def log_to_root(message: str, *args: Any, **kwargs: Any) -> None:
 
 
 class FixLogger(Logger):
-    def debug2(self, msg: str, *args: Any, **kwargs: Any) -> None:
-        ...
+    def debug2(self, msg: str, *args: Any, **kwargs: Any) -> None: ...
 
-    def debug3(self, msg: str, *args: Any, **kwargs: Any) -> None:
-        ...
+    def debug3(self, msg: str, *args: Any, **kwargs: Any) -> None: ...
 
-    def debug4(self, msg: str, *args: Any, **kwargs: Any) -> None:
-        ...
+    def debug4(self, msg: str, *args: Any, **kwargs: Any) -> None: ...
 
-    def debug5(self, msg: str, *args: Any, **kwargs: Any) -> None:
-        ...
+    def debug5(self, msg: str, *args: Any, **kwargs: Any) -> None: ...
 
-    def trace(self, msg: str, *args: Any, **kwargs: Any) -> None:
-        ...
+    def trace(self, msg: str, *args: Any, **kwargs: Any) -> None: ...
 
 
 def get_fix_logger(name: Optional[str] = None) -> FixLogger:

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fixbackup"
-version = "0.0.8"
+version = "0.0.9"
 authors = [{name="Some Engineering Inc."}]
 description = "FIX Database Backup System"
 license = {file="LICENSE"}

requirements-test.txt

@@ -1,44 +1,44 @@
-astroid==3.0.0
+astroid==3.2.3
     # via pylint
-attrs==23.1.0
+attrs==23.2.0
     # via hypothesis
-black==23.9.1
+black==24.4.2
     # via fixbackup (pyproject.toml)
-boto3==1.28.63
+boto3==1.34.144
     # via fixbackup (pyproject.toml)
-botocore==1.31.63
+botocore==1.34.144
     # via
     #   boto3
     #   s3transfer
-cachetools==5.3.1
+cachetools==5.4.0
     # via tox
 chardet==5.2.0
     # via tox
 click==8.1.7
     # via black
 colorama==0.4.6
     # via tox
-coverage[toml]==7.3.2
+coverage[toml]==7.6.0
     # via
     #   fixbackup (pyproject.toml)
     #   pytest-cov
-dill==0.3.7
+dill==0.3.8
     # via pylint
-distlib==0.3.7
+distlib==0.3.8
     # via virtualenv
-filelock==3.12.4
+filelock==3.15.4
     # via
     #   tox
     #   virtualenv
-flake8==6.1.0
+flake8==7.1.0
     # via
     #   fixbackup (pyproject.toml)
     #   pep8-naming
-hypothesis==6.87.4
+hypothesis==6.108.2
     # via fixbackup (pyproject.toml)
 iniconfig==2.0.0
     # via pytest
-isort==5.12.0
+isort==5.13.2
     # via pylint
 jmespath==1.0.1
     # via
@@ -48,68 +48,68 @@ mccabe==0.7.0
     # via
     #   flake8
     #   pylint
-mypy==1.6.0
+mypy==1.10.1
     # via fixbackup (pyproject.toml)
 mypy-extensions==1.0.0
     # via
     #   black
     #   mypy
-packaging==23.2
+packaging==24.1
     # via
     #   black
     #   pyproject-api
     #   pytest
     #   tox
-pathspec==0.11.2
+pathspec==0.12.1
     # via black
-pep8-naming==0.13.3
+pep8-naming==0.14.1
     # via fixbackup (pyproject.toml)
-platformdirs==3.11.0
+platformdirs==4.2.2
     # via
     #   black
     #   pylint
     #   tox
     #   virtualenv
-pluggy==1.3.0
+pluggy==1.5.0
     # via
     #   pytest
     #   tox
-pycodestyle==2.11.0
+pycodestyle==2.12.0
     # via flake8
-pyflakes==3.1.0
+pyflakes==3.2.0
     # via flake8
-pylint==3.0.1
+pylint==3.2.5
     # via fixbackup (pyproject.toml)
-pyproject-api==1.6.1
+pyproject-api==1.7.1
     # via tox
-pytest==7.4.2
+pytest==8.2.2
     # via
     #   fixbackup (pyproject.toml)
     #   pytest-asyncio
     #   pytest-cov
-pytest-asyncio==0.21.1
+pytest-asyncio==0.23.7
     # via fixbackup (pyproject.toml)
-pytest-cov==4.1.0
+pytest-cov==5.0.0
     # via fixbackup (pyproject.toml)
-pytest-runner==6.0.0
+pytest-runner==6.0.1
     # via fixbackup (pyproject.toml)
-python-dateutil==2.8.2
+python-dateutil==2.9.0.post0
     # via botocore
-s3transfer==0.7.0
+s3transfer==0.10.2
     # via boto3
 six==1.16.0
     # via python-dateutil
 sortedcontainers==2.4.0
     # via hypothesis
-tomlkit==0.12.1
+tomlkit==0.13.0
     # via pylint
-tox==4.11.3
+tox==4.16.0
     # via fixbackup (pyproject.toml)
-typing-extensions==4.8.0
+typing-extensions==4.12.2
     # via mypy
-urllib3==2.0.7
+urllib3==2.2.2
     # via botocore
-virtualenv==20.24.5
+virtualenv==20.26.3
     # via tox
-wheel==0.41.2
+wheel==0.43.0
     # via fixbackup (pyproject.toml)

requirements.txt

@@ -1,18 +1,18 @@
-boto3==1.28.63
+boto3==1.34.144
     # via fixbackup (pyproject.toml)
-botocore==1.31.63
+botocore==1.34.144
     # via
     #   boto3
     #   s3transfer
 jmespath==1.0.1
     # via
     #   boto3
     #   botocore
-python-dateutil==2.8.2
+python-dateutil==2.9.0.post0
     # via botocore
-s3transfer==0.7.0
+s3transfer==0.10.2
     # via boto3
 six==1.16.0
     # via python-dateutil
-urllib3==2.0.7
+urllib3==2.2.2
     # via botocore

tox.ini

@@ -4,7 +4,7 @@ env_list = syntax, tests, black, flake8, mypy
 [flake8]
 max-line-length = 120
 exclude = .git,.tox,__pycache__,.idea,.pytest_cache,venv
-ignore = F401, F403, F405, F811, E722, N806, N813, E266, W503, E203
+ignore = F401, F403, F405, E704, F811, E722, N806, N813, E266, W503, E203
 
 [pytest]
 testpaths = test