diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..99667bab3 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,8 @@ +# Each line is a file pattern followed by one or more owners. + +# These owners will be the default owners for everything in +# the repo. Unless a later match takes precedence, +# @global-owner1 and @global-owner2 will be requested for +# review when someone opens a pull request. +* @EnterpriseDB/barman-team + diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml deleted file mode 100644 index 74f456785..000000000 --- a/.github/workflows/black.yml +++ /dev/null @@ -1,38 +0,0 @@ ---- -name: Black - -on: - push: - branches-ignore: - - master - pull_request: - branches-ignore: - - master - workflow_dispatch: - -jobs: - lint: - name: "Black" - runs-on: ubuntu-latest - steps: - - name: Step 1 - Checkout repository - uses: actions/checkout@v2 - - name: Step 2 - Apply Black - uses: psf/black@stable - with: - options: "" - - name: Step 3 - Check if changes - id: git-status - shell: bash - run: | - if [[ -n $(git status --porcelain) ]] - then - echo "::set-output name=status::true" - fi - - name: Step 4 - Commit changes - if: contains(steps.git-status.outputs.status, 'true') - uses: EndBug/add-and-commit@v8 - with: - default_author: github_actions - message: "black: reformat source code" - push: true diff --git a/.github/workflows/blackduck-scan.yml b/.github/workflows/blackduck-scan.yml deleted file mode 100644 index 4e8de1473..000000000 --- a/.github/workflows/blackduck-scan.yml +++ /dev/null @@ -1,120 +0,0 @@ -### -## Foundation-security BlackDuck workflow for public repos -# -name: Foundation-Security/Black Duck Scan (PUBLIC) - -on: - push: - tags: - - '**' - workflow_dispatch: - -jobs: - Blackduck-Scan: - runs-on: ubuntu-20.04 - steps: - - name: Check for Secret - run: | - if [ -z "${{ secrets.BLACKDUCK_URL }}" ]; then - echo "Error:blackduck secret not found" - exit 1 - fi - - - name: Checkout source repository - id: checkout-source - uses: actions/checkout@v3 - - - name: Download and Run Detect - id: run-synopsys-detect - env: - detect.blackduck.scan.mode: 'INTELLIGENT' - detect.scan.output.path: '/home/runner/work/_temp/blackduck' - detect.output.path: '/home/runner/work/_temp/blackduck' - detect.project.version.name: ${{ github.ref_type }}/${{ github.ref_name }} - detect.project.name: ${{ github.event.repository.name }} - detect.risk.report.pdf: true - detect.risk.report.pdf.path: /tmp/reports/ - blackduck.url: ${{ secrets.BLACKDUCK_URL }} - blackduck.api.token: ${{ secrets.BLACKDUCK_API_TOKEN }} - shell: bash - run: | - bash <(curl -s -L https://detect.synopsys.com/detect8.sh) - - - name: Run sbom script - id: sbom-report - shell: bash - run: | - chmod +x ./.github/workflows/blackduck/get_bd_sbom.sh - ./.github/workflows/blackduck/get_bd_sbom.sh ${{ secrets.BLACKDUCK_URL }} ${{ secrets.BLACKDUCK_API_TOKEN }} ${{ github.event.repository.name }} ${{ github.ref_type }}/${{ github.ref_name }} - - - name: Set current date as env variable - id: get-date - shell: bash - run: | - echo "NOW=$(date +'%Y-%m-%dT%H-%M-%S')" >> $GITHUB_ENV; - - - name: get report names - id: get-report-names - shell: bash - run: | - echo "pdf-name=`ls -1 /tmp/reports/*.pdf | sed 's#.*/##'`" >> $GITHUB_ENV; - - - name: Normalize report names - id: normalize-report-names - shell: bash - run: | - cd /tmp/reports/ ; - mv ${{ env.pdf-name }} ${{ env.NOW }}-${{ env.pdf-name }}; - mv sbom.zip ${{ env.NOW }}-${{ github.event.repository.name }}-`echo ${{ github.ref_name }}| sed -e 's/\//-/g'`-sbom.zip; - - - - name: Create report artifact - id: create-report-artifact - uses: actions/upload-artifact@v3 - with: - name: risk-report - path: /tmp/reports/*.pdf - - - - name: Create sbom artifact - id: create-sbom-artifact - uses: actions/upload-artifact@v3 - with: - name: sbom-report - path: /tmp/reports/*-sbom.zip - - - Deploy-to-S3: - needs: Blackduck-Scan - runs-on: ubuntu-20.04 - permissions: # These permissions are needed to interact with GitHub's OIDC Token endpoint. - id-token: write - contents: read - steps: - - - name: Download Artifacts - id: download-artifact - uses: actions/download-artifact@v3 - with: - path: /tmp/reports/ - - - name: Configure AWS Credentials - id: configure-aws-credentials - uses: aws-actions/configure-aws-credentials@v1-node16 - with: - role-to-assume: arn:aws:iam::934964804737:role/github-actions-foundation-security - role-session-name: blackduck-public-repo - role-duration-seconds: 1200 #20 minute TTL - aws-region: us-east-1 - - - name: Upload to results to S3 - id: s3-upload - shell: bash - run: | - if [ "$(ls -A /tmp/reports/)" ]; then - aws s3 cp /tmp/reports/ s3://foundation-security/blackduck/${{ github.event.repository.name }}/ --recursive; - fi - continue-on-error: true - env: - AWS_EC2_METADATA_DISABLED: 'true' - diff --git a/.github/workflows/blackduck/get_bd_sbom.sh b/.github/workflows/blackduck/get_bd_sbom.sh deleted file mode 100644 index a53122c13..000000000 --- a/.github/workflows/blackduck/get_bd_sbom.sh +++ /dev/null @@ -1,200 +0,0 @@ -#!/usr/bin/env bash - -set -E - -#### Functions - -function show_banner { - echo "==============================================================" - echo " Create Blackduck SBOM " - echo "==============================================================" - echo -} - -function show_usage { - echo - echo "Usage: ./get-bd-sbom.sh " -} - -function get_bearer { - result=$(curl --silent --location --request POST "${blackduck_url}api/tokens/authenticate" \ - --header "Authorization: token $blackduck_token") - if [[ $(echo "$result" | jq -r .errorCode) != null ]] - then - >&2 echo "ERROR: No bearer token found" - exit 1 - else - echo "$result" | jq -r .bearerToken - fi -} - -function get_project_id { - result=$(curl --silent --location --request GET "${blackduck_url}api/projects?q=name:$project" \ - --header "Authorization: Bearer $bearer_token") - if [[ $(echo "$result" | jq -r .totalCount) -eq 0 ]] - then - >&2 echo "ERROR: No project found with name: $project" - exit 1 - else - echo "$result" | jq -r .items[0]._meta.href - fi -} - -function get_version_id { - result=$(curl --silent --location --request GET "$project_api_url/versions?q=versionName:$version" \ - --header "Authorization: Bearer $bearer_token") - if [[ $(echo "$result" | jq -r .totalCount) -eq 0 ]] - - then - >&2 echo "ERROR: No version found with name: $version" - exit 1 - else - echo "$result" | jq -r .items[0]._meta.href - fi -} - -function create_sbom_report { - dataraw="{\"reportFormat\": \"JSON\", \"reportType\" : \"SBOM\", \"sbomType\" : \"SPDX_22\"}" - result=$(curl --silent --location --request POST "$version_api_url/sbom-reports" \ - --header "Authorization: Bearer $bearer_token" \ - --header 'Accept: application/json' \ - --header 'Content-Type: application/json' \ - --data-raw "$dataraw" ) - if [ "$result" != "" ] - then - >&2 echo "ERROR: error in creating SBOM report" - >&2 echo $result - exit 1 - fi -} - -function get_report_id { - report_status="IN_PROGRESS" - max_retries=50 - retries=0 - - while [ "$report_status" = "IN_PROGRESS" ] - do - ((retries++)) - if [ "$retries" -gt "$max_retries" ]; - then - >&2 echo "ERROR: max retries reached" - exit 1 - fi - echo "| attempt $retries of $max_retries to get SPDX report" - sleep 15 - result=$(curl --silent --location --request GET "$version_api_url/reports" \ - --header "Authorization: Bearer $bearer_token" \ - --header 'Content-Type: application/json') - report_api_url=$(echo "$result" | jq -r '.items[0]._meta.href') - report_status=$(echo "$result" | jq -r '.items[0].status') - echo "| - report_status: $report_status" - done - - if [ "$report_status" != "COMPLETED" ]; - then - >&2 echo " ERROR: report_status is not COMPLETED, it is $report_status." - exit 1 - fi -} - -function download_sbom_report { - curl --silent --location --request GET "$report_api_url/download.zip" \ - -o /tmp/reports/sbom.zip \ - --header "Authorization: Bearer $bearer_token" \ - --header 'Content-Type: application/zip' -} - -function get_report_contents { - curl --silent --location --request GET "$report_api_url/contents" \ - --header "Authorization: Bearer $bearer_token" \ - --header 'Content-Type: application/json' | jq -rc .reportContent[0].fileContent -} - - -#### Main program - -show_banner - -error=false - -if [ -z "$1" ] - then - echo "ERROR: No blackduck-url supplied" - error=true -fi - -if [ -z "$2" ] - then - echo "ERROR: No blackduck-api-token supplied" - error=true -fi - -if [ -z "$3" ] - then - echo "ERROR: No project-name supplied" - error=true -fi - -if [ -z "$4" ] - then - echo "ERROR: No version-name supplied" - error=true -fi - -if [ $error == "true" ] - then - show_usage - exit 1 -fi - -blackduck_url=$1 -blackduck_token=$2 -project=$3 -version=$4 - -sbom_type="SPDX_22" - -echo "+ getting bearer" -bearer_token=$(get_bearer) -echo "| got bearer" -echo - -echo "+ getting project api base url" -project_api_url=$(get_project_id) -echo "| got project api base url: ${project_api_url}" -echo - -echo "+ getting version api base url" -version_api_url=$(get_version_id) -echo "| got version api base url: ${version_api_url}" -echo - -echo "+ creating SBOM report" -if [ "${NO_CREATE}" == true ] -then - echo "| We're not creating a new report for the because of the secret environment variable NO_CREATE" -else - create_sbom_report - echo "| triggered creating SBOM report" -fi -echo - -echo "+ getting SBOM report api base url" -get_report_id -echo "| got SBOM report status: ${report_status}" -echo "| got SBOM report api base url: ${report_api_url}" -echo - -echo "+ getting SBOM report" -download_sbom_report -echo "| got SBOM report" -echo - -echo "+ getting content information" -report_contents=$(get_report_contents) -echo "| got content information" -echo - -echo "sbom-file=sbom.zip" >> $GITHUB_OUTPUT -echo "sbom-contents=${report_contents}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml new file mode 100644 index 000000000..a4a1675c6 --- /dev/null +++ b/.github/workflows/linter.yml @@ -0,0 +1,92 @@ +--- +# Copyright (C) 2024 EnterpriseDB + +name: Linters + +on: + pull_request: + branches: + - master + + push: + branches: + - master + + schedule: + # Lint code base every Monday 12:00 am. The idea here is to catch possible + # issues that were not detected during the normal development workflow. + - cron: '0 0 * * 1' + + workflow_dispatch: + inputs: + source-ref: + description: Source code branch/ref name + default: master + required: true + type: string + +env: + SOURCE_REF: ${{ inputs.source-ref || github.ref }} + GITHUB_TOKEN: ${{ secrets.GH_SLONIK }} + +jobs: + run-super-linter: + name: Run super linter + runs-on: ubuntu-latest + + permissions: + contents: read + packages: read + # To report GitHub Actions status checks + statuses: write + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ env.SOURCE_REF }} + # Full git history is needed to get a proper list of changed files within `super-linter` + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + + - name: Super-linter + uses: super-linter/super-linter/slim@v7 + env: + # To report GitHub Actions status checks + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Linters configuration. + LINTER_RULES_PATH: '.' + # We are not interested in linting these files: + # * Markdown files under `doc` or `sphinx` directories, which belong to the + # old docs, and are going to be replaced soon. + FILTER_REGEX_EXCLUDE: '(doc|sphinx)/.*\.md' + DOCKERFILE_HADOLINT_FILE_NAME: .hadolint.yaml + GITLEAKS_CONFIG_FILE: .gitleaks.toml + MARKDOWN_CONFIG_FILE: .markdownlint.yml + PYTHON_BLACK_CONFIG_FILE: .python-black + PYTHON_FLAKE8_CONFIG_FILE: tox.ini + PYTHON_ISORT_CONFIG_FILE: .isort.cfg + YAML_CONFIG_FILE: .yamllint.yml + YAML_ERROR_ON_WARNING: false + # On runs triggered by PRs we only lint the added/modified files. + VALIDATE_ALL_CODEBASE: ${{ github.event_name != 'pull_request' }} + # Validate file types used in the Barman repo. + # Bash because of bash scripts. + VALIDATE_BASH: true + VALIDATE_BASH_EXEC: true + # Dockerfile because we might add some of them soon. + VALIDATE_DOCKERFILE_HADOLINT: true + # Validate the own GitHub workflows and actions. + VALIDATE_GITHUB_ACTIONS: true + # Search for leaks in the repository. + VALIDATE_GITLEAKS: true + # Validate all documentation files from the repo. + VALIDATE_MARKDOWN: true + # Validate Python code. + VALIDATE_PYTHON_BLACK: true + VALIDATE_PYTHON_FLAKE8: true + VALIDATE_PYTHON_ISORT: true + # Validate YAML files from workflows. + VALIDATE_YAML: true diff --git a/.github/workflows/sonarqube-scan.yml b/.github/workflows/sonarqube-scan.yml deleted file mode 100644 index 8a5acaa31..000000000 --- a/.github/workflows/sonarqube-scan.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: SonarQube-Workflow -on: - pull_request: - branches: [ master ] - push: - branches: [ master ] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: -jobs: - sonarQube: - name: SonarQube-Job - runs-on: ubuntu-latest - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'EnterpriseDB/barman' - steps: - - name: Checkout source repo - uses: actions/checkout@v2 - - name: Checkout GitHub Action Repo - uses: actions/checkout@master - with: - repository: EnterpriseDB/edb-github-actions.git - ref: master - token: ${{ secrets.REPO_ACCESS_TOKEN }} - path: .github/actions/edb-github-actions - - name: SonarQube Scan - uses: ./.github/actions/edb-github-actions/sonarqube - with: - REPO_NAME: '${{github.event.repository.name}}' - SONAR_PROJECT_KEY: '${{secrets.SONARQUBE_PROJECTKEY}}' - SONAR_URL: '${{secrets.SONARQUBE_URL}}' - SONAR_LOGIN: '${{secrets.SONARQUBE_LOGIN}}' - PULL_REQUEST_KEY: '${{github.event.number}}' - PULL_REQUEST_BRANCH: '${{github.head_ref}}' - PULL_REQUEST_BASE_BRANCH: '${{github.base_ref}}' - REPO_DEFAULT_BRANCH: '${{github.event.repository.default_branch}}' - REPO_EXCLUDE_FILES: '**/src/test/**/*,**/docs/**/*,**/build/**' diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 000000000..6f3fd24bf --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,4 @@ +[extend] +# useDefault will extend the base configuration with the default gitleaks config: +# https://github.com/zricethezav/gitleaks/blob/master/config/gitleaks.toml +useDefault = true diff --git a/.hadolint.yaml b/.hadolint.yaml new file mode 100644 index 000000000..f8cbb9da2 --- /dev/null +++ b/.hadolint.yaml @@ -0,0 +1 @@ +failure-threshold: error diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 000000000..a29184f0a --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,3 @@ +[settings] +profile = black +multi_line_output = 3 diff --git a/.markdownlint.yml b/.markdownlint.yml new file mode 100644 index 000000000..37067fb96 --- /dev/null +++ b/.markdownlint.yml @@ -0,0 +1,8 @@ +# MD013/line-length : Line length : https://github.com/DavidAnson/markdownlint/blob/v0.34.0/doc/md013.md +# We don't want the linter to fail just because line-length was exceeded. +MD013: false +# MD024/no-duplicate-heading: https://github.com/DavidAnson/markdownlint/blob/v0.34.0/doc/md024.md +# We don't want the linter to fail when duplicated header names are found. That is not +# relevant for us, and actually we rely on duplicated names when generating the RELNOTES.md +# contents. +MD024: false diff --git a/.python-black b/.python-black new file mode 100644 index 000000000..8bb6ee5f5 --- /dev/null +++ b/.python-black @@ -0,0 +1,2 @@ +[tool.black] +line-length = 88 diff --git a/.yamllint.yml b/.yamllint.yml new file mode 100644 index 000000000..8e12de7d4 --- /dev/null +++ b/.yamllint.yml @@ -0,0 +1,12 @@ +extends: default + +rules: + # comments should visibly make sense + comments: + level: error + comments-indentation: + level: error + # 88 chars should be enough, but don't fail if a line is longer + line-length: + max: 88 + level: warning diff --git a/AUTHORS b/AUTHORS index 4145bc7d5..0f62f395b 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,25 +1,29 @@ Barman maintainers (in alphabetical order): -* Abhijit Menon-Sen -* Didier Michel +* Andre Marchesini +* Barbara Leidens * Giulio Calacoci +* Gustavo Oliveira * Israel Barth -* Jane Threefoot -* Michael Wallace +* Martín Marqués Past contributors (in alphabetical order): +* Abhijit Menon-Sen (architect) * Anna Bellandi (QA/testing) * Britt Cole (documentation reviewer) * Carlo Ascani (developer) +* Didier Michel (developer) * Francesco Canovai (QA/testing) * Gabriele Bartolini (architect) * Gianni Ciolli (QA/testing) * Giulio Calacoci (developer) * Giuseppe Broccolo (developer) +* Jane Threefoot (developer) * Jonathan Battiato (QA/testing) * Leonardo Cecchi (developer) * Marco Nenciarini (project leader) +* Michael Wallace (developer) * Niccolò Fei (QA/testing) * Rubens Souza (QA/testing) * Stefano Bianucci (developer) diff --git a/NEWS b/NEWS index c2c313139..777bfb4e0 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,205 @@ Barman News - History of user-visible changes +Version 3.11.1 - 22 August 2024 + +- Bug fixes: + + - Fix failures in `barman-cloud-backup-delete`. This command was failing when + applying retention policies due to a bug introduced by the previous release. + +Version 3.11.0 - 22 August 2024 + +- Add support for Postgres 17+ incremental backups. This major feature is + composed of several small changes: + + - Add `--incremental` command-line option to `barman backup` command. This is + used to specify the parent backup when taking an incremental backup. The + parent can be either a full backup or another incremental backup. + + - Add `latest-full` shortcut backup ID. Along with `latest`, this can be used + as a shortcut to select the parent backup for an incremental backup. While + `latest` takes the latest backup independently if it is full or incremental, + `latest-full` takes the latest full backup. + + - `barman keep` command can only be applied to full backups when + `backup_method = postgres`. If a full backup has incremental backups that + depend on it, all of the incrementals are also kept by Barman. + + - When deleting a backup all the incremental backups depending on it, if any, + are also removed. + + - Retention policies do not take incremental backups into consideration. As + incremental backups cannot be recovered without having the complete chain of + backups available up to the full backup, only full backups account for + retention policies. + + - `barman recover` needs to combine the full backup with the chain of incremental + backups when recovering. The new CLI option `--local-staging-path`, and the + corresponding `local_staging_path` configuration option, are used to specify + the path in the Barman host where the backups will be combined when recovering + an incremental backup. + +- Changes to `barman show-backup` output: + + - Add the “Estimated cluster size” field. It's useful to have an estimation + of the data directory size of a cluster when restoring a backup. It’s + particularly useful when recovering compressed backups or incremental + backups, situations where the size of the backup doesn’t reflect the size of the + data directory in Postgres. In JSON format, this is stored as + `cluster_size`. + + - Add the “WAL summarizer” field. This field shows if `summarize_wal` was + enabled in Postgres at the time the backup was taken. In JSON format, this + is stored as `server_information.summarize_wal`. This field is omitted for + Postgres 16 and older. + + - Add “Data checksums” field. This shows if `data_checkums` was enabled in + Postgres at the time the backup was taken. In JSON format, this is stored as + `server_information.data_checksums`. + + - Add the “Backup method” field. This shows the backup method used for this + backup. In JSON format, this is stored as + `base_backup_information.backup_method`. + + - Rename the field “Disk Usage” as “Backup Size”. The latter provides a more + comprehensive name which represents the size of the backup in the Barman + host. The JSON field under `base_backup_information` was also renamed from + `disk_usage` to `backup_size`. + + - Add the “WAL size” field. This shows the size of the WALs required by the + backup. In JSON format, this is stored as + `base_backup_information.wal_size`. + + - Refactor the field “Incremental size”. It is now named “Resources saving” + and it now shows an estimation of resources saved when taking incremental + backups with `rsync` or `pg_basebackup`. It compares the backup size with + the estimated cluster size to estimate the amount of disk and network + resources that were saved by taking an incremental backup. In JSON format, + the field was renamed from `incremental_size` to `resource_savings` under `base_backup_information`. + + - Add the `system_id` field to the JSON document. This field contains the + system identifier of Postgres. It was present in console format, but was + missing in JSON format. + + - Add fields related with Postgres incremental backups: + + - “Backup type”: indicates if the Postgres backup is full or incremental. In + JSON format, this is stored as `backup_type` under `base_backup_information`. + + - “Root backup”: the ID of the full backup that is the root of a chain of + one or more incremental backups. In JSON format, this is stored as + `catalog_information.root_backup_id`. + + - “Parent backup”: the ID of the full or incremental backup from which this + incremental backup was taken. In JSON format, this is stored as + `catalog_information.parent_backup_id`. + + - “Children Backup(s)”: the IDs of the incremental backups that were taken + with this backup as the parent. In JSON format, this is stored as + `catalog_information.children_backup_ids`. + + - “Backup chain size”: the number of backups in the chain from this + incremental backup up to the root backup. In JSON format, this is + stored as `catalog_information.chain_size`. + +- Changes to `barman list-backup` output: + + - It now includes the backup type in the JSON output, which can be either + `rsync` for backups taken with rsync, `full` or `incremental` for backups + taken with `pg_basebackup`, or `snapshot` for cloud snapshots. When printing + to the console the backup type is represented by the corresponding labels + `R`, `F`, `I` or `S`. + + - Remove tablespaces information from the output. That was bloating the + output. Tablespaces information can still be found in the output of + `barman show-backup`. + +- Always set a timestamp with a time zone when configuring + `recovery_target_time` through `barman recover`. Previously, if no time zone + was explicitly set through `--target-time`, Barman would configure + `recovery_target_time` without a time zone in Postgres. Without a time zone, + Postgres would assume whatever is configured through `timezone` GUC in + Postgres. From now on Barman will issue a warning and configure + `recovery_target_time` with the time zone of the Barman host if no time zone + is set by the user through `--target-time` option. + +- When recovering a backup with the “no get wal” approach and `--target-lsn` is set, + copy only the WAL files required to reach the configured target. Previously + Barman would copy all the WAL files from its archive to Postgres. + +- When recovering a backup with the “no get wal” approach and `--target-immediate` + is set, copy only the WAL files required to reach the consistent point. + Previously Barman would copy all the WAL files from its archive to Postgres. + +- `barman-wal-restore` now moves WALs from the spool directory to `pg_wal` + instead of copying them. This can improve performance if the spool directory + and the `pg_wal` directory are in the same partition. + +- `barman check-backup` now shows the reason why a backup was marked as `FAILED` + in the output and logs. Previously for a user to know why the backup was + marked as `FAILED`, they would need to run `barman show-backup` command. + +- Add configuration option `aws_await_snapshots_timeout` and the corresponding + `--aws-await-snapshots-timeout` command-line option on `barman-cloud-backup`. + This specifies the timeout in seconds to wait for snapshot backups to reach + the completed state. + +- Add a keep-alive mechanism to rsync-based backups. Previously the Postgres + session created by Barman to run `pg_backup_start()` and `pg_backup_stop()` would + stay idle for as long as the base backup copy would take. That could lead to a + firewall or router dropping the connection because it was idle for a long + time. The keep-alive mechanism sends heartbeat queries to Postgres + through that connection, thus reducing the likelihood of a connection + getting dropped. The interval between heartbeats can be controlled through the new + configuration option `keepalive_interval` and the corresponding CLI + option `--keepalive-interval` of the `barman backup` command. + +- Bug fixes: + + - When recovering a backup with the “no get wal” approach and `--target-time` + set, copy all WAL files. Previously Barman would attempt to “guess” the WAL + files required by Postgres to reach the configured target time. However, + the mechanism was not robust enough as it was based on the stats of the WAL + file in the Barman host (more specifically the creation time). For example: + if there were archiving or streaming lag between Postgres and Barman, that + could be enough for recovery to fail because Barman would miss to copy all + the required WAL files due to the weak check based on file stats. + + - Pin `python-snappy` to `0.6.1` when running Barman through Python 3.6 or + older. Newer versions of `python-snappy` require `cramjam` version `2.7.0` or + newer, and these are only available for Python 3.7 or newer. + + - `barman receive-wal` now exits with code `1` instead of `0` in the following + cases: + + - Being unable to run with `--reset` flag because `pg_receivewal` is + running. + + - Being unable to start `pg_receivewal` process because it is already + running. + + - Fix and improve information about Python in `barman diagnose` output: + + - The command now makes sure to use the same Python interpreter under which + Barman is installed when outputting the Python version through + `python_ver` JSON key. Previously, if an environment had multiple Python + installations and/or virtual environments, the output could eventually be + misleading, as it could be fetched from a different Python interpreter. + + - Added a `python_executable` key to the JSON output. That contains the path + to the exact Python interpreter being used by Barman. + +Version 3.10.1 - 12 June 2024 + +- Bug fixes: + - Make `argcomplete` optional to avoid installation issues on some + platforms. + - Load `barman.auto.conf` only when the file exists. + - Emit a warning when the `cfg_changes.queue` file is malformed. + - Correct in documentation the postgresql version where + `pg_checkpoint` is available. + - Add `--no-partial` option to `barman-cloud-wal-restore`. + Version 3.10.0 - 24 January 2024 - Limit the average bandwidth used by `barman-cloud-backup` when backing diff --git a/README.rst b/README.rst index 5960216a1..bf9e59cce 100644 --- a/README.rst +++ b/README.rst @@ -45,7 +45,7 @@ Web resources Licence ------- -© Copyright 2011-2023 EnterpriseDB UK Limited +© Copyright 2011-2024 EnterpriseDB UK Limited Barman is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free diff --git a/barman/backup.py b/barman/backup.py index fd164c493..bbbf4a604 100644 --- a/barman/backup.py +++ b/barman/backup.py @@ -44,6 +44,7 @@ from barman.config import BackupOptions from barman.exceptions import ( AbortedRetryHookScript, + BackupException, CompressionIncompatibility, LockFileBusy, SshCommandException, @@ -360,8 +361,8 @@ def get_last_backup_id(self, status_filter=DEFAULT_STATUS_FILTER): Get the id of the latest/last backup in the catalog (if exists) :param status_filter: The status of the backup to return, - default to DEFAULT_STATUS_FILTER. - :return string|None: ID of the backup + default to :attr:`DEFAULT_STATUS_FILTER`. + :return str|None: ID of the backup """ available_backups = self.get_available_backups(status_filter) if len(available_backups) == 0: @@ -370,6 +371,29 @@ def get_last_backup_id(self, status_filter=DEFAULT_STATUS_FILTER): ids = sorted(available_backups.keys()) return ids[-1] + def get_last_full_backup_id(self, status_filter=DEFAULT_STATUS_FILTER): + """ + Get the id of the latest/last FULL backup in the catalog (if exists) + + :param status_filter: The status of the backup to return, + default to :attr:`DEFAULT_STATUS_FILTER`. + :return str|None: ID of the backup + """ + available_full_backups = list( + filter( + lambda backup: backup.is_full_and_eligible_for_incremental(), + self.get_available_backups(status_filter).values(), + ) + ) + + if len(available_full_backups) == 0: + return None + + backup_infos = sorted( + available_full_backups, key=lambda backup_info: backup_info.backup_id + ) + return backup_infos[-1].backup_id + def get_first_backup_id(self, status_filter=DEFAULT_STATUS_FILTER): """ Get the id of the oldest/first backup in the catalog (if exists) @@ -435,33 +459,6 @@ def delete_backup(self, backup, skip_wal_cleanup_if_standalone=True): backups. :return bool: True if deleted, False if could not delete the backup """ - if self.should_keep_backup(backup.backup_id): - output.warning( - "Skipping delete of backup %s for server %s " - "as it has a current keep request. If you really " - "want to delete this backup please remove the keep " - "and try again.", - backup.backup_id, - self.config.name, - ) - return False - available_backups = self.get_available_backups(status_filter=(BackupInfo.DONE,)) - minimum_redundancy = self.server.config.minimum_redundancy - # Honour minimum required redundancy - if backup.status == BackupInfo.DONE and minimum_redundancy >= len( - available_backups - ): - output.warning( - "Skipping delete of backup %s for server %s " - "due to minimum redundancy requirements " - "(minimum redundancy = %s, " - "current redundancy = %s)", - backup.backup_id, - self.config.name, - minimum_redundancy, - len(available_backups), - ) - return False # Keep track of when the delete operation started. delete_start_time = datetime.datetime.now() @@ -529,6 +526,7 @@ def delete_backup(self, backup, skip_wal_cleanup_if_standalone=True): remove_until, timelines_to_protect, wal_ranges_to_protect ): output.info("\t%s", name) + # As last action, remove the backup directory, # ending the delete operation try: @@ -553,6 +551,30 @@ def delete_backup(self, backup, skip_wal_cleanup_if_standalone=True): human_readable_timedelta(delete_end_time - delete_start_time), ) + # remove its reference from its parent if it is an incremental backup + parent_backup = backup.get_parent_backup_info() + if parent_backup: + parent_backup.children_backup_ids.remove(backup.backup_id) + if not parent_backup.children_backup_ids: + parent_backup.children_backup_ids = None + parent_backup.save() + + # rsync backups can have deduplication at filesystem level by using + # "reuse_backup = link". The deduplication size is calculated at the + # time the backup is taken. If we remove a backup, it may be the case + # that the next backup in the catalog is a rsync backup which was taken + # with the "link" option. With that possibility in mind, we re-calculate the + # deduplicated size of the next rsync backup because the removal of the + # previous backup can impact on that number. + # Note: we have no straight forward way of identifying if the next rsync + # backup in the catalog was taken with "link" or not because + # "reuse_backup" value is not stored in the "backup.info" file. In any + # case, the "re-calculation" can still be performed even if "link" was not + # used, and the only drawback is that we will waste some (small) amount + # of CPU/disk usage. + if next_backup and next_backup.backup_type == "rsync": + self._set_backup_sizes(next_backup) + # Remove the sync lockfile if exists sync_lock = ServerBackupSyncLock( self.config.barman_lock_directory, self.config.name, backup.backup_id @@ -584,13 +606,107 @@ def delete_backup(self, backup, skip_wal_cleanup_if_standalone=True): return True - def backup(self, wait=False, wait_timeout=None, name=None): + def _set_backup_sizes(self, backup_info, fsync=False): + """ + Set the actual size on disk of a backup. + + Optionally fsync all files in the backup. + + :param LocalBackupInfo backup_info: the backup to update + :param bool fsync: whether to fsync files to disk + """ + backup_size = 0 + deduplicated_size = 0 + backup_dest = backup_info.get_basebackup_directory() + for dir_path, _, file_names in os.walk(backup_dest): + if fsync: + # If fsync, execute fsync() on the containing directory + fsync_dir(dir_path) + for filename in file_names: + file_path = os.path.join(dir_path, filename) + # If fsync, execute fsync() on all the contained files + file_stat = fsync_file(file_path) if fsync else os.stat(file_path) + backup_size += file_stat.st_size + # Excludes hard links from real backup size and only counts + # unique files for deduplicated size + if file_stat.st_nlink == 1: + deduplicated_size += file_stat.st_size + # Save size into BackupInfo object + backup_info.set_attribute("size", backup_size) + backup_info.set_attribute("deduplicated_size", deduplicated_size) + backup_info.save() + + def validate_backup_args(self, **kwargs): + """ + Validate backup arguments and Postgres configurations. Arguments + might be syntactically correct but still be invalid if necessary + Postgres configurations are not met. + + :kwparam str parent_backup_id: id of the parent backup when taking a + Postgres incremental backup + :raises BackupException: if a command argument is considered invalid + """ + if "parent_backup_id" in kwargs: + self._validate_incremental_backup_configs(**kwargs) + + def _validate_incremental_backup_configs(self, **kwargs): + """ + Check required configurations for a Postgres incremental backup + + :raises BackupException: if a required configuration is missing + """ + if self.server.postgres.server_version < 170000: + raise BackupException( + "Postgres version 17 or greater is required for incremental backups " + "using the Postgres backup method" + ) + + if self.config.backup_method != "postgres": + raise BackupException( + "Backup using the `--incremental` flag is available only for " + "'backup_method = postgres'. Check Barman's documentation for " + "more help on this topic." + ) + + summarize_wal = self.server.postgres.get_setting("summarize_wal") + if summarize_wal != "on": + raise BackupException( + "'summarize_wal' option has to be enabled in the Postgres server " + "to perform an incremental backup using the Postgres backup method" + ) + + if self.config.backup_compression is not None: + raise BackupException( + "Incremental backups cannot be taken with " + "'backup_compression' set in the configuration options." + ) + + parent_backup_id = kwargs.get("parent_backup_id") + parent_backup_info = self.get_backup(parent_backup_id) + + if parent_backup_info: + if parent_backup_info.summarize_wal != "on": + raise BackupException( + "The specified backup is not eligible as a parent for an " + "incremental backup because WAL summaries were not enabled " + "when that backup was taken." + ) + if parent_backup_info.compression is not None: + raise BackupException( + "The specified backup cannot be a parent for an " + "incremental backup. Reason: " + "Compressed backups are not eligible as parents of incremental backups." + ) + + def backup(self, wait=False, wait_timeout=None, name=None, **kwargs): """ Performs a backup for the server :param bool wait: wait for all the required WAL files to be archived :param int|None wait_timeout: :param str|None name: the friendly name to be saved with this backup + :kwparam str parent_backup_id: id of the parent backup when taking a + Postgres incremental backup :return BackupInfo: the generated BackupInfo """ _logger.debug("initialising backup information") @@ -605,6 +721,11 @@ def backup(self, wait=False, wait_timeout=None, name=None): ) backup_info.set_attribute("systemid", self.server.systemid) + backup_info.set_attribute( + "parent_backup_id", + kwargs.get("parent_backup_id"), + ) + backup_info.save() self.backup_cache_add(backup_info) output.info( @@ -694,6 +815,22 @@ def backup(self, wait=False, wait_timeout=None, name=None): finally: if backup_info: + # IF is an incremental backup, we save here child backup info id + # inside the parent list of children. no matter if the backup + # is successful or not. This is needed to be able to retrieve + # also failed incremental backups for removal or other operations + # like show-backup. + parent_backup_info = backup_info.get_parent_backup_info() + + if parent_backup_info: + if parent_backup_info.children_backup_ids: + parent_backup_info.children_backup_ids.append( # type: ignore + backup_info.backup_id + ) + else: + parent_backup_info.children_backup_ids = [backup_info.backup_id] + parent_backup_info.save() + backup_info.save() # Make sure we are not holding any PostgreSQL connection @@ -1379,23 +1516,10 @@ def backup_fsync_and_set_sizes(self, backup_info): # Calculate the base backup size self.executor.current_action = "calculating backup size" _logger.debug(self.executor.current_action) - backup_size = 0 - deduplicated_size = 0 - backup_dest = backup_info.get_basebackup_directory() - for dir_path, _, file_names in os.walk(backup_dest): - # execute fsync() on the containing directory - fsync_dir(dir_path) - # execute fsync() on all the contained files - for filename in file_names: - file_path = os.path.join(dir_path, filename) - file_stat = fsync_file(file_path) - backup_size += file_stat.st_size - # Excludes hard links from real backup size - if file_stat.st_nlink == 1: - deduplicated_size += file_stat.st_size - # Save size into BackupInfo object - backup_info.set_attribute("size", backup_size) - backup_info.set_attribute("deduplicated_size", deduplicated_size) + # Set backup sizes with fsync. We need to fsync files here to make sure + # the backup files are persisted to disk, so we don't lose the backup in + # the event of a system crash. + self._set_backup_sizes(backup_info, fsync=True) if backup_info.size > 0: deduplication_ratio = 1 - ( float(backup_info.deduplicated_size) / backup_info.size @@ -1487,6 +1611,10 @@ def check_backup(self, backup_info): ) backup_info.status = BackupInfo.FAILED backup_info.save() + output.error( + "This backup has been marked as FAILED due to the " + "following reason: %s" % backup_info.error + ) return if end_wal <= last_archived_wal: diff --git a/barman/backup_executor.py b/barman/backup_executor.py index 445b38d3d..96318552c 100644 --- a/barman/backup_executor.py +++ b/barman/backup_executor.py @@ -51,6 +51,7 @@ DataTransferFailure, FsOperationFailed, PostgresConnectionError, + PostgresConnectionLost, PostgresIsInRecovery, SnapshotBackupException, SshCommandException, @@ -58,6 +59,7 @@ ) from barman.fs import UnixLocalCommand, UnixRemoteCommand, unix_command_factory from barman.infofile import BackupInfo +from barman.postgres import PostgresKeepAlive from barman.postgres_plumbing import EXCLUDE_LIST, PGDATA_EXCLUDE_LIST from barman.remote_status import RemoteStatusMixin from barman.utils import ( @@ -574,6 +576,7 @@ def backup_copy(self, backup_info): :param barman.infofile.LocalBackupInfo backup_info: backup information """ + # Make sure the destination directory exists, ensure the # right permissions to the destination dir backup_dest = backup_info.get_data_directory() @@ -608,6 +611,13 @@ def backup_copy(self, backup_info): # for the whole duration of the copy self.server.close() + # Find the backup_manifest file path of the parent backup in case + # it is an incremental backup + parent_backup_info = backup_info.get_parent_backup_info() + parent_backup_manifest_path = None + if parent_backup_info: + parent_backup_manifest_path = parent_backup_info.get_backup_manifest_path() + pg_basebackup = PgBaseBackup( connection=self.server.streaming, destination=backup_dest, @@ -624,6 +634,7 @@ def backup_copy(self, backup_info): compression=self.backup_compression, err_handler=self._err_handler, out_handler=PgBaseBackup.make_logging_handler(logging.INFO), + parent_backup_manifest_path=parent_backup_manifest_path, ) # Do the actual copy @@ -857,6 +868,7 @@ def backup(self, backup_info): self._update_action_from_strategy() raise + connection_error = False try: # save any metadata changed by start_backup() call # This must be inside the try-except, because it could fail @@ -885,20 +897,34 @@ def backup(self, backup_info): # the begin_wal value is surely known. Doing it twice is safe # because this function is useful only during the first backup. self._purge_unused_wal_files(backup_info) - except BaseException: + + except PostgresConnectionLost: + # This exception is most likely to be raised by the PostgresKeepAlive, + # meaning that we lost the connection (and session) during the backup. + connection_error = True + raise + except BaseException as ex: # we do not need to do anything here besides re-raising the # exception. It will be handled in the external try block. output.error("The backup has failed %s", self.current_action) + # As we have found that in certain corner cases the exception + # passing through this block is not logged or even hidden by + # other exceptions happening in the finally block, we are adding a + # debug log line to make sure that the exception is visible. + _logger.debug("Backup failed: %s" % ex, exc_info=True) raise else: self.current_action = "issuing stop of the backup" finally: - output.info("Asking PostgreSQL server to finalize the backup.") - try: - self.strategy.stop_backup(backup_info) - except BaseException: - self._update_action_from_strategy() - raise + # If a connection error has been raised, it means we lost our session in the + # Postgres server. In such cases, it's useless to try a backup stop command. + if not connection_error: + output.info("Asking PostgreSQL server to finalize the backup.") + try: + self.strategy.stop_backup(backup_info) + except BaseException: + self._update_action_from_strategy() + raise def _local_check(self, check_strategy): """ @@ -1182,6 +1208,26 @@ def validate_configuration(self): "backup_compression option is not supported by rsync backup_method" ) + def backup(self, *args, **kwargs): + """ + Perform an Rsync backup. + + .. note:: + This method currently only calls the parent backup method but inside a keepalive + context to ensure the connection does not become idle long enough to get dropped + by a firewall, for instance. This is important to ensure that ``pg_backup_start()`` + and ``pg_backup_stop()`` are called within the same session. + """ + try: + with PostgresKeepAlive( + self.server.postgres, self.config.keepalive_interval, True + ): + super(RsyncBackupExecutor, self).backup(*args, **kwargs) + except PostgresConnectionLost: + raise BackupException( + "Connection to the Postgres server was lost during the backup." + ) + def backup_copy(self, backup_info): """ Perform the actual copy of the backup using Rsync. @@ -1771,6 +1817,20 @@ def _pg_get_metadata(self, backup_info): msg = "\t%s, %s, %s" % (item.oid, item.name, item.location) _logger.info(msg) + # Set data_checksums state + data_checksums = self.postgres.get_setting("data_checksums") + backup_info.set_attribute("data_checksums", data_checksums) + + # Get summarize_wal information for incremental backups + # Postgres major version should be >= 17 + backup_info.set_attribute("summarize_wal", None) + if self.postgres.server_version >= 170000: + summarize_wal = self.postgres.get_setting("summarize_wal") + backup_info.set_attribute("summarize_wal", summarize_wal) + + # Set total size of the PostgreSQL server + backup_info.set_attribute("cluster_size", self.postgres.current_size) + @staticmethod def _backup_info_from_start_location(backup_info, start_info): """ diff --git a/barman/cli.py b/barman/cli.py index 5af3db00c..83d6f2e9f 100644 --- a/barman/cli.py +++ b/barman/cli.py @@ -36,7 +36,10 @@ if sys.version_info.major < 3: from argparse import Action, _SubParsersAction, _ActionsContainer -import argcomplete +try: + import argcomplete +except ImportError: + argcomplete = None from collections import OrderedDict from contextlib import closing @@ -49,7 +52,7 @@ from barman.config import ( ConfigChangesProcessor, RecoveryOptions, - parse_recovery_staging_path, + parse_staging_path, ) from barman.exceptions import ( BadXlogSegmentName, @@ -403,6 +406,13 @@ def backup_completer(prefix, parsed_args, **kwargs): action="store_false", default=SUPPRESS, ), + argument( + "--incremental", + completer=backup_completer, + dest="backup_id", + help="performs an incremental backup. An ID of a previous backup must " + "be provided ('latest' and 'latest-full' are also available options)", + ), argument( "--reuse-backup", nargs="?", @@ -474,6 +484,13 @@ def backup_completer(prefix, parsed_args, **kwargs): default=None, type=check_non_negative, ), + argument( + "--keepalive-interval", + help="An interval, in seconds, at which a heartbeat query will be sent " + "to the server to keep the libpq connection alive during an Rsync backup.", + dest="keepalive_interval", + type=check_non_negative, + ), argument( "--name", help="a name which can be used to reference this backup in barman " @@ -512,12 +529,20 @@ def backup(args): if not manage_server_command(server, name): continue + incremental_kwargs = {} + + if args.backup_id is not None: + parent_backup_info = parse_backup_id(server, args) + if parent_backup_info: + incremental_kwargs["parent_backup_id"] = parent_backup_info.backup_id if args.reuse_backup is not None: server.config.reuse_backup = args.reuse_backup if args.retry_sleep is not None: server.config.basebackup_retry_sleep = args.retry_sleep if args.retry_times is not None: server.config.basebackup_retry_times = args.retry_times + if args.keepalive_interval is not None: + server.config.keepalive_interval = args.keepalive_interval if hasattr(args, "immediate_checkpoint"): # As well as overriding the immediate_checkpoint value in the config # we must also update the immediate_checkpoint attribute on the @@ -544,6 +569,7 @@ def backup(args): wait=args.wait, wait_timeout=args.wait_timeout, backup_name=args.backup_name, + **incremental_kwargs, ) output.close_and_exit() @@ -840,6 +866,16 @@ def rebuild_xlogdb(args): "backup." ), ), + argument( + "--local-staging-path", + help=( + "A path to a location on the local host where incremental backups " + "will be combined during the recovery. This location must have " + "enough available space to temporarily hold the new synthetic " + "backup. This option is *required* when recovering from an " + "incremental backup." + ), + ), argument( "--recovery-conf-filename", dest="recovery_conf_filename", @@ -898,9 +934,7 @@ def recover(args): # Set the recovery staging path from the cli if it is set if args.recovery_staging_path is not None: try: - recovery_staging_path = parse_recovery_staging_path( - args.recovery_staging_path - ) + recovery_staging_path = parse_staging_path(args.recovery_staging_path) except ValueError as exc: output.error("Cannot parse recovery staging path: %s", str(exc)) output.close_and_exit() @@ -921,6 +955,32 @@ def recover(args): ) output.close_and_exit() + # If the backup to be recovered is incremental then there are additional + # checks to be carried out + if backup_id.is_incremental: + # Set the local staging path from the cli if it is set + if args.local_staging_path is not None: + try: + local_staging_path = parse_staging_path(args.local_staging_path) + except ValueError as exc: + output.error("Cannot parse local staging path: %s", str(exc)) + output.close_and_exit() + server.config.local_staging_path = local_staging_path + # If the backup is incremental but there is no local_staging_path + # then this is an error - the user *must* tell barman where recovery + # data can be staged. + if server.config.local_staging_path is None: + output.error( + "Cannot recover from backup '%s' of server '%s': " + "backup will be combined with pg_combinebackup in the " + "barman host but no local staging path is provided. " + "Either set local_staging_path in the Barman config " + "or use the --local-staging-path argument.", + args.backup_id, + server.config.name, + ) + output.close_and_exit() + # decode the tablespace relocation rules tablespaces = {} if args.tablespace: @@ -1083,7 +1143,7 @@ def recover(args): target_action=getattr(args, "target_action", None), standby_mode=getattr(args, "standby_mode", None), recovery_conf_filename=args.recovery_conf_filename, - **snapshot_kwargs + **snapshot_kwargs, ) except RecoveryException as exc: output.error(force_str(exc)) @@ -1803,6 +1863,13 @@ def keep(args): ) % (backup_info.backup_id, backup_info.status) output.error(msg) output.close_and_exit() + if backup_info.is_incremental: + msg = ( + "Unable to execute the keep command on backup %s: is an incremental backup.\n" + "Only full backups are eligible for the use of the keep command." + ) % (backup_info.backup_id) + output.error(msg) + output.close_and_exit() backup_manager.keep_backup(backup_info.backup_id, args.target) @@ -1991,9 +2058,10 @@ def global_config(args): # Load additional configuration files config.load_configuration_files_directory() - config.load_config_file( - "%s/.barman.auto.conf" % config.get("barman", "barman_home") - ) + # Handle the autoconf file, load it only if exists + autoconf_path = "%s/.barman.auto.conf" % config.get("barman", "barman_home") + if os.path.exists(autoconf_path): + config.load_config_file(autoconf_path) # We must validate the configuration here in order to have # both output and logging configured config.validate_global_config() @@ -2381,7 +2449,8 @@ def main(): """ # noinspection PyBroadException try: - argcomplete.autocomplete(p) + if argcomplete: + argcomplete.autocomplete(p) args = p.parse_args() global_config(args) if args.command is None: diff --git a/barman/clients/cloud_backup.py b/barman/clients/cloud_backup.py index 0e8dfa7a4..b7a5c0f16 100755 --- a/barman/clients/cloud_backup.py +++ b/barman/clients/cloud_backup.py @@ -419,6 +419,13 @@ def parse_arguments(args=None): help="The name of the AWS region containing the EC2 VM and storage volumes " "defined by the --snapshot-instance and --snapshot-disk arguments.", ) + s3_arguments.add_argument( + "--aws-await-snapshots-timeout", + default=3600, + help="The length of time in seconds to wait for snapshots to be created in AWS before " + "timing out (default: 3600 seconds)", + type=check_positive, + ) azure_arguments.add_argument( "--encryption-scope", help="The name of an encryption scope defined in the Azure Blob Storage " diff --git a/barman/clients/cloud_restore.py b/barman/clients/cloud_restore.py index 38033426f..f2a3975bf 100644 --- a/barman/clients/cloud_restore.py +++ b/barman/clients/cloud_restore.py @@ -300,9 +300,11 @@ def download_backup(self, backup_info, destination_dir, tablespaces): "Extracting %s to %s (%s)", file_info.path, target_dir, - "decompressing " + file_info.compression - if file_info.compression - else "no compression", + ( + "decompressing " + file_info.compression + if file_info.compression + else "no compression" + ), ) self.cloud_interface.extract_tar(file_info.path, target_dir) diff --git a/barman/clients/cloud_walrestore.py b/barman/clients/cloud_walrestore.py index 30304bbc3..d118ad073 100644 --- a/barman/clients/cloud_walrestore.py +++ b/barman/clients/cloud_walrestore.py @@ -32,7 +32,7 @@ from barman.cloud_providers import get_cloud_interface from barman.exceptions import BarmanException from barman.utils import force_str -from barman.xlog import hash_dir, is_any_xlog_file, is_backup_file +from barman.xlog import hash_dir, is_any_xlog_file, is_backup_file, is_partial_file def main(args=None): @@ -68,7 +68,7 @@ def main(args=None): logging.error("Bucket %s does not exist", cloud_interface.bucket_name) raise OperationErrorExit() - downloader.download_wal(config.wal_name, config.wal_dest) + downloader.download_wal(config.wal_name, config.wal_dest, config.no_partial) except Exception as exc: logging.error("Barman cloud WAL restore exception: %s", force_str(exc)) @@ -90,6 +90,12 @@ def parse_arguments(args=None): "Currently AWS S3, Azure Blob Storage and Google Cloud Storage are supported.", ) + parser.add_argument( + "--no-partial", + help="Do not download partial WAL files", + action="store_true", + default=False, + ) parser.add_argument( "wal_name", help="The value of the '%%f' keyword (according to 'restore_command').", @@ -118,12 +124,13 @@ def __init__(self, cloud_interface, server_name): self.cloud_interface = cloud_interface self.server_name = server_name - def download_wal(self, wal_name, wal_dest): + def download_wal(self, wal_name, wal_dest, no_partial): """ Download a WAL file from cloud storage :param str wal_name: Name of the WAL file :param str wal_dest: Full path of the destination WAL file + :param bool no_partial: Do not download partial WAL files """ # Correctly format the source path on s3 @@ -163,6 +170,10 @@ def download_wal(self, wal_name, wal_dest): elif is_backup_file(basename): logging.info("Skipping backup file: %s", item) continue + # Exclude partial files if required + elif no_partial and is_partial_file(basename): + logging.info("Skipping partial file: %s", item) + continue # Found candidate remote_name = item diff --git a/barman/clients/walrestore.py b/barman/clients/walrestore.py index afc93c9c0..d7541698b 100755 --- a/barman/clients/walrestore.py +++ b/barman/clients/walrestore.py @@ -74,7 +74,7 @@ def main(args=None): return # never reached # If the file is present in SPOOL_DIR use it and terminate - try_deliver_from_spool(config, dest_file) + try_deliver_from_spool(config, dest_file.name) # If required load the list of files to download in parallel additional_files = peek_additional_files(config) @@ -241,22 +241,19 @@ def try_deliver_from_spool(config, dest_file): return otherwise. :param argparse.Namespace config: the configuration from command line - :param dest_file: The destination file object + :param dest_file: The path to the destination file """ - spool_file = os.path.join(config.spool_dir, config.wal_name) + spool_file = str(os.path.join(config.spool_dir, config.wal_name)) # id the file is not present, give up if not os.path.exists(spool_file): return try: - shutil.copyfileobj(open(spool_file, "rb"), dest_file) - os.unlink(spool_file) + shutil.move(spool_file, dest_file) sys.exit(0) except IOError as e: - exit_with_error( - "Failure copying %s to %s: %s" % (spool_file, dest_file.name, e) - ) + exit_with_error("Failure moving %s to %s: %s" % (spool_file, dest_file, e)) def exit_with_error(message, status=2, sleep=0): diff --git a/barman/cloud_providers/__init__.py b/barman/cloud_providers/__init__.py index f40cdcace..c65317bfe 100644 --- a/barman/cloud_providers/__init__.py +++ b/barman/cloud_providers/__init__.py @@ -197,7 +197,12 @@ def get_snapshot_interface(config): elif config.cloud_provider == "aws-s3": from barman.cloud_providers.aws_s3 import AwsCloudSnapshotInterface - return AwsCloudSnapshotInterface(config.aws_profile, config.aws_region) + args = [ + config.aws_profile, + config.aws_region, + config.aws_await_snapshots_timeout, + ] + return AwsCloudSnapshotInterface(*args) else: raise CloudProviderUnsupported( "No snapshot provider for cloud provider: %s" % config.cloud_provider @@ -245,7 +250,9 @@ def get_snapshot_interface_from_server_config(server_config): from barman.cloud_providers.aws_s3 import AwsCloudSnapshotInterface return AwsCloudSnapshotInterface( - server_config.aws_profile, server_config.aws_region + server_config.aws_profile, + server_config.aws_region, + server_config.aws_await_snapshots_timeout, ) else: raise CloudProviderUnsupported( diff --git a/barman/cloud_providers/aws_s3.py b/barman/cloud_providers/aws_s3.py index 2e0960879..7cf4cb330 100644 --- a/barman/cloud_providers/aws_s3.py +++ b/barman/cloud_providers/aws_s3.py @@ -17,6 +17,7 @@ # along with Barman. If not, see import logging +import math import shutil from io import RawIOBase @@ -462,18 +463,31 @@ class AwsCloudSnapshotInterface(CloudSnapshotInterface): https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-creating-snapshot.html """ - def __init__(self, profile_name=None, region=None): + def __init__(self, profile_name=None, region=None, await_snapshots_timeout=3600): """ Creates the client necessary for creating and managing snapshots. :param str profile_name: AWS auth profile identifier. :param str region: The AWS region in which snapshot resources are located. + :param int await_snapshots_timeout: The maximum time in seconds to wait for + snapshots to complete. """ self.session = boto3.Session(profile_name=profile_name) # If a specific region was provided then this overrides any region which may be # defined in the profile self.region = region or self.session.region_name self.ec2_client = self.session.client("ec2", region_name=self.region) + self.await_snapshots_timeout = await_snapshots_timeout + + def _get_waiter_config(self): + delay = 15 + # Use ceil so that we always wait for at least the specified timeout + max_attempts = math.ceil(self.await_snapshots_timeout / delay) + return { + "Delay": delay, + # Ensure we always try waiting at least once + "MaxAttempts": max(max_attempts, 1), + } def _get_instance_metadata(self, instance_identifier): """ @@ -764,7 +778,10 @@ def take_snapshot_backup(self, backup_info, instance_identifier, volumes): snapshot_ids = [snapshot.identifier for snapshot in snapshots] logging.info("Waiting for completion of snapshots: %s", ", ".join(snapshot_ids)) waiter = self.ec2_client.get_waiter("snapshot_completed") - waiter.wait(Filters=[{"Name": "snapshot-id", "Values": snapshot_ids}]) + waiter.wait( + Filters=[{"Name": "snapshot-id", "Values": snapshot_ids}], + WaiterConfig=self._get_waiter_config(), + ) backup_info.snapshots_info = AwsSnapshotsInfo( snapshots=snapshots, diff --git a/barman/command_wrappers.py b/barman/command_wrappers.py index 4e32af554..141c06eeb 100644 --- a/barman/command_wrappers.py +++ b/barman/command_wrappers.py @@ -918,6 +918,7 @@ def __init__( immediate=False, check=True, compression=None, + parent_backup_manifest_path=None, args=None, **kwargs ): @@ -937,6 +938,9 @@ def __init__( allowed values of the Command obj :param barman.compression.PgBaseBackupCompression compression: the pg_basebackup compression options used for this backup + :param str parent_backup_manifest_path: + the path to a backup_manifest file from a previous backup which can + be used to perform an incremental backup :param List[str] args: additional arguments """ PostgreSQLClient.__init__( @@ -972,6 +976,10 @@ def __init__( if bwlimit is not None and bwlimit > 0: self.args.append("--max-rate=%s" % bwlimit) + # If it has a manifest file path it means it is an incremental backup + if parent_backup_manifest_path: + self.args.append("--incremental=%s" % parent_backup_manifest_path) + # Immediate checkpoint if immediate: self.args.append("--checkpoint=fast") @@ -1142,6 +1150,64 @@ def __init__( self.args += args +class PgCombineBackup(PostgreSQLClient): + """ + Wrapper class for the ``pg_combinebackup`` system command + """ + + COMMAND_ALTERNATIVES = ["pg_combinebackup"] + + def __init__( + self, + destination, + command, + tbs_mapping=None, + connection=None, + version=None, + app_name=None, + check=True, + args=None, + **kwargs + ): + """ + Constructor + + :param str destination: destination directory path + :param str command: the command to use + :param None|Dict[str, str] tbs_mapping: used for tablespace + :param PostgreSQL connection: an object representing + a database connection + :param Version version: the command version + :param str app_name: the application name to use for the connection + :param bool check: check if the return value is in the list of + allowed values of the :class:`Command` obj + :param None|List[str] args: additional arguments + """ + PostgreSQLClient.__init__( + self, + connection=connection, + command=command, + version=version, + app_name=app_name, + check=check, + **kwargs + ) + + # Set the backup destination + self.args = ["--output=%s" % destination] + + # The tablespace mapping option is repeated once for each tablespace + if tbs_mapping: + for tbs_source, tbs_destination in tbs_mapping.items(): + self.args.append( + "--tablespace-mapping=%s=%s" % (tbs_source, tbs_destination) + ) + + # Manage additional args + if args: + self.args += args + + class BarmanSubProcess(object): """ Wrapper class for barman sub instances diff --git a/barman/config.py b/barman/config.py index e0a7152a4..b569e0241 100644 --- a/barman/config.py +++ b/barman/config.py @@ -79,7 +79,6 @@ class CsvOption(set): - """ Base class for CSV options. @@ -360,7 +359,7 @@ def parse_backup_method(value): ) -def parse_recovery_staging_path(value): +def parse_staging_path(value): if value is None or os.path.isabs(value): return value raise ValueError("Invalid value : '%s' (must be an absolute path)" % value) @@ -489,6 +488,7 @@ class ServerConfig(BaseConfig): "archiver", "archiver_batch_size", "autogenerate_manifest", + "aws_await_snapshots_timeout", "aws_profile", "aws_region", "azure_credential", @@ -521,9 +521,11 @@ class ServerConfig(BaseConfig): "gcp_zone", "immediate_checkpoint", "incoming_wals_directory", + "keepalive_interval", "last_backup_maximum_age", "last_backup_minimum_size", "last_wal_maximum_age", + "local_staging_path", "max_incoming_wals_queue", "minimum_redundancy", "network_compression", @@ -584,6 +586,7 @@ class ServerConfig(BaseConfig): "archiver", "archiver_batch_size", "autogenerate_manifest", + "aws_await_snapshots_timeout", "aws_profile", "aws_region", "azure_credential", @@ -609,9 +612,11 @@ class ServerConfig(BaseConfig): "forward_config_path", "gcp_project", "immediate_checkpoint", + "keepalive_internval", "last_backup_maximum_age", "last_backup_minimum_size", "last_wal_maximum_age", + "local_staging_path", "max_incoming_wals_queue", "minimum_redundancy", "network_compression", @@ -661,6 +666,7 @@ class ServerConfig(BaseConfig): "archiver": "off", "archiver_batch_size": "0", "autogenerate_manifest": "false", + "aws_await_snapshots_timeout": "3600", "backup_directory": "%(barman_home)s/%(name)s", "backup_method": "rsync", "backup_options": "", @@ -674,6 +680,7 @@ class ServerConfig(BaseConfig): "forward_config_path": "false", "immediate_checkpoint": "false", "incoming_wals_directory": "%(backup_directory)s/incoming", + "keepalive_interval": "60", "minimum_redundancy": "0", "network_compression": "false", "parallel_jobs": "1", @@ -702,6 +709,7 @@ class ServerConfig(BaseConfig): "archiver": parse_boolean, "archiver_batch_size": int, "autogenerate_manifest": parse_boolean, + "aws_await_snapshots_timeout": int, "backup_compression": parse_backup_compression, "backup_compression_format": parse_backup_compression_format, "backup_compression_level": int, @@ -714,10 +722,12 @@ class ServerConfig(BaseConfig): "check_timeout": int, "disabled": parse_boolean, "forward_config_path": parse_boolean, + "keepalive_interval": int, "immediate_checkpoint": parse_boolean, "last_backup_maximum_age": parse_time_interval, "last_backup_minimum_size": parse_si_suffix, "last_wal_maximum_age": parse_time_interval, + "local_staging_path": parse_staging_path, "max_incoming_wals_queue": int, "network_compression": parse_boolean, "parallel_jobs": int, @@ -725,7 +735,7 @@ class ServerConfig(BaseConfig): "parallel_jobs_start_batch_size": int, "primary_checkpoint_timeout": int, "recovery_options": RecoveryOptions, - "recovery_staging_path": parse_recovery_staging_path, + "recovery_staging_path": parse_staging_path, "create_slot": parse_create_slot, "reuse_backup": parse_reuse_backup, "snapshot_disks": parse_snapshot_disks, @@ -1337,20 +1347,24 @@ def load_configuration_files_directory(self): def load_config_file(self, cfile): filename = os.path.basename(cfile) - if os.path.isfile(cfile): - # Load a file - _logger.debug("Including configuration file: %s", filename) - self._config.read_config(cfile) - if self._is_global_config_changed(): - msg = ( - "the configuration file %s contains a not empty [barman] section" - % filename - ) - _logger.fatal(msg) - raise SystemExit("FATAL: %s" % msg) + if os.path.exists(cfile): + if os.path.isfile(cfile): + # Load a file + _logger.debug("Including configuration file: %s", filename) + self._config.read_config(cfile) + if self._is_global_config_changed(): + msg = ( + "the configuration file %s contains a not empty [barman] section" + % filename + ) + _logger.fatal(msg) + raise SystemExit("FATAL: %s" % msg) + else: + # Add an warning message that a file has been discarded + _logger.warn("Discarding configuration file: %s (not a file)", filename) else: - # Add an info that a file has been discarded - _logger.warn("Discarding configuration file: %s (not a file)", filename) + # Add an warning message that a file has been discarded + _logger.warn("Discarding configuration file: %s (not found)", filename) def _is_model(self, name): """ @@ -1857,6 +1871,11 @@ def read_file(path) -> List[ConfigChangeSet]: return json.load(queue_file, object_hook=ConfigChangeSet.from_dict) except FileNotFoundError: return [] + except json.JSONDecodeError: + output.warning( + "Malformed or empty configuration change queue: %s" % queue_file.name + ) + return [] def __enter__(self): """ diff --git a/barman/copy_controller.py b/barman/copy_controller.py index fce030444..8ad245395 100644 --- a/barman/copy_controller.py +++ b/barman/copy_controller.py @@ -506,7 +506,7 @@ def _rsync_set_pre_31_mode(self): """ _logger.info( "Detected rsync version less than 3.1. " - "top using '--ignore-missing-args' argument." + "Stopping use of '--ignore-missing-args' argument." ) self.rsync_has_ignore_missing_args = False self.rsync_cache.clear() diff --git a/barman/exceptions.py b/barman/exceptions.py index e90bbc1f9..cc31f0bf7 100644 --- a/barman/exceptions.py +++ b/barman/exceptions.py @@ -257,6 +257,12 @@ def __str__(self): return "" +class PostgresConnectionLost(PostgresException): + """ + The Postgres connection was lost during an execution + """ + + class PostgresAppNameError(PostgresConnectionError): """ Error setting application name with PostgreSQL server diff --git a/barman/fs.py b/barman/fs.py index a561926b5..502103388 100644 --- a/barman/fs.py +++ b/barman/fs.py @@ -16,6 +16,7 @@ # You should have received a copy of the GNU General Public License # along with Barman. If not, see . +import sys import logging import re import shutil @@ -265,8 +266,12 @@ def get_system_info(self): self.cmd("uname", args=["-a"]) result["kernel_ver"] = self.internal_cmd.out.rstrip() - self.cmd("python", args=["--version", "2>&1"]) - result["python_ver"] = self.internal_cmd.out.rstrip() + result["python_ver"] = "Python %s.%s.%s" % ( + sys.version_info.major, + sys.version_info.minor, + sys.version_info.micro, + ) + result["python_executable"] = sys.executable self.cmd("rsync", args=["--version", "2>&1"]) try: result["rsync_ver"] = self.internal_cmd.out.splitlines(True)[0].rstrip() diff --git a/barman/hooks.py b/barman/hooks.py index a77173ff6..eaca20198 100644 --- a/barman/hooks.py +++ b/barman/hooks.py @@ -210,7 +210,6 @@ def run(self): class RetryHookScriptRunner(HookScriptRunner): - """ A 'retry' hook script is a special kind of hook script that Barman tries to run indefinitely until it either returns a SUCCESS or diff --git a/barman/infofile.py b/barman/infofile.py index 03f293c34..15267390d 100644 --- a/barman/infofile.py +++ b/barman/infofile.py @@ -108,6 +108,32 @@ def load_datetime_tz(time_str): return timestamp +def dump_backup_ids(ids): + """ + Dump a list of backup IDs to disk as a string. + + :param list[str]|None ids: list of backup IDs, if any + :return str|None: the dumped string. + """ + if ids: + return ",".join(ids) + else: + return None + + +def load_backup_ids(string): + """ + Load a list of backup IDs from disk as a :class:`list`. + + :param string: the string to be loaded as a list. + :return list[str]|None: the list of backup IDs, if any. + """ + if string: + return string.split(",") + else: + return None + + class Field(object): def __init__(self, name, dump=None, load=None, default=None, doc=None): """ @@ -496,6 +522,16 @@ class BackupInfo(FieldListFile): snapshots_info = Field( "snapshots_info", load=load_snapshots_info, dump=output_snapshots_info ) + data_checksums = Field("data_checksums") + summarize_wal = Field("summarize_wal") + parent_backup_id = Field("parent_backup_id") + children_backup_ids = Field( + "children_backup_ids", + dump=dump_backup_ids, + load=load_backup_ids, + ) + + cluster_size = Field("cluster_size", load=int) __slots__ = "backup_id", "backup_version" @@ -557,6 +593,94 @@ def set_attribute(self, key, value): """ setattr(self, key, value) + @property + def is_incremental(self): + """ + Only checks if the backup_info is an incremental backup + + .. note:: + This property only makes sense in the context of local backups stored in the + Barman server. However, this property is used for retention policies + processing, code which is shared among local and cloud backups. As this + property always returns ``False`` for cloud backups, it can safely be reused + in their code paths as well, though. + + :return bool: ``True`` if this backup has a parent, ``False`` otherwise. + """ + return self.parent_backup_id is not None + + @property + def has_children(self): + """ + Only checks if the backup_info has children + + .. note:: + This property only makes sense in the context of local backups stored in the + Barman server. However, this property is used for retention policies + processing, code which is shared among local and cloud backups. As this + property always returns ``False`` for cloud backups, it can safely be reused + in their code paths as well, though. + + :return bool: ``True`` if this backup has at least one child, ``False`` otherwise. + """ + return self.children_backup_ids is not None + + @property + def backup_type(self): + """ + Returns a string with the backup type label. + + .. note:: + Even though this property is available in this base class, it is not + expected to be used in the context of cloud backups. + + The backup type can be one of the following: + - ``snapshot``: If the backup mode starts with ``snapshot``. + - ``rsync``: If the backup mode starts with ``rsync``. + - ``incremental``: If the mode is ``postgres`` and the backup is incremental. + - ``full``: If the mode is ``postgres`` and the backup is not incremental. + + :return str: The backup type label. + """ + if self.mode.startswith("snapshot"): + return "snapshot" + elif self.mode.startswith("rsync"): + return "rsync" + return "incremental" if self.is_incremental else "full" + + @property + def deduplication_ratio(self): + """ + Returns a value between and including ``0`` and ``1`` related to the estimate + deduplication ratio of the backup. + + .. note:: + For ``rsync`` backups, the :attr:`size` of the backup, which is the sum of + all file sizes in basebackup directory, is used to calculate the + ratio. For ``postgres`` backups, the :attr:`cluster_size` is used, which contains + the estimated size of the Postgres cluster at backup time. + + We perform this calculation to make an estimation of how much network and disk + I/O has been saved when taking an incremental backup through ``rsync`` or through + ``pg_basebackup``. + + We abuse of the term "deduplication" here. It makes more sense to ``rsync`` than to + ``postgres`` method. However, the idea is the same in both cases: get an estimation + of resources saving. + + .. note:: + Even though this property is available in this base class, it is not + expected to be used in the context of cloud backups. + + :return float: The backup deduplication ratio. + """ + size = self.cluster_size + if self.backup_type == "rsync": + size = self.size + if size and self.deduplicated_size: + return 1 - (self.deduplicated_size / size) + return 0 + def to_dict(self): """ Return the backup_info content as a simple dictionary @@ -790,3 +914,190 @@ def save(self, filename=None, file_object=None): if not os.path.exists(dir_name): os.makedirs(dir_name) super(LocalBackupInfo, self).save(filename=filename, file_object=file_object) + + def get_backup_manifest_path(self): + """ + Get the full path to the backup manifest file + + :return str: the full path to the backup manifest file. + """ + return os.path.join(self.get_data_directory(), "backup_manifest") + + def get_parent_backup_info(self): + """ + If the backup is incremental, build the :class:`LocalBackupInfo` object + for the parent backup and return it. + If the backup is not incremental OR the status of the parent + backup is ``EMPTY``, return ``None``. + + :return LocalBackupInfo|None: the parent backup info object, + or None if it does not exist or is empty. + """ + if self.is_incremental: + backup_info = LocalBackupInfo( + self.server, + backup_id=self.parent_backup_id, + ) + + if backup_info.status != BackupInfo.EMPTY: + return backup_info + + return None + + def get_child_backup_info(self, child_backup_id): + """ + Allow to retrieve a specific child of the current incremental backup, + if there are any. + + If the child backup exists, the LocalBackupInfo object for it is returned. + If does not exist or its status is `EMPTY`, return None + + :param str child_backup_id: the ID of the child backup to retrieve + + :return LocalBackupInfo|None: the child backup info object, + or None if it does not exist or is empty. + """ + if self.children_backup_ids: + if child_backup_id in self.children_backup_ids: + backup_info = LocalBackupInfo( + self.server, + backup_id=child_backup_id, + ) + + if backup_info.status != BackupInfo.EMPTY: + return backup_info + + return None + + def walk_backups_tree(self, return_self=True): + """ + Walk through all the children backups of the current backup. + + .. note:: + The objects are returned with a bottom-up approach, including all + children backups plus the caller backup. + + :param bool return_self: Whether to return the current backup. + Default to ``True``. + + :yields: a generator of :class:`LocalBackupInfo` objects for each + backup, walking from the leaves to self. + """ + + if self.children_backup_ids: + for child_backup_id in self.children_backup_ids: + backup_info = LocalBackupInfo( + self.server, + backup_id=child_backup_id, + ) + yield from backup_info.walk_backups_tree() + if not return_self: + return + yield self + + def walk_to_root(self, return_self=True): + """ + Walk through all the parent backups of the current backup. + + .. note:: + The objects are returned with a bottom-up approach, including all + parents backups plus the caller backup if *return_self* is ``True``. + + :param bool return_self: Whether to return the current backup. + Default to ``True``. + + :yield: a generator of :class:`LocalBackupInfo` objects for each parent backup. + """ + + if return_self: + yield self + backup_info = self.get_parent_backup_info() + while backup_info: + yield backup_info + backup_info = backup_info.get_parent_backup_info() + + def is_checksum_consistent(self): + """ + Check if all backups in the chain are consistent with their checksums + configurations. + + The backup chain is considered inconsistent if the current backup was taken with + ``data_checksums`` enabled and any of its ascendants were taken with it disabled. + It is considered consistent otherwise. + + .. note:: + While this method was created to check inconsistencies in chains of one + or more (Postgres 17+ core) incremental backups, it can be safely used + with any Postgres version and with any Barman backup method. That is + true because it always returns ``True`` when called for a Postgres full + backup or for a rsync backup. + + :return bool: ``True`` if it is consistent, ``False`` otherwise. + """ + if self.data_checksums != "on" or not self.is_incremental: + return True + for backup in self.walk_to_root(return_self=False): + if backup.data_checksums == "off": + return False + return True + + def is_full_and_eligible_for_incremental(self): + """ + Check if this is a full backup taken with `postgres` method and which is + eligible to be a parent for an incremental backup. + + .. note:: + Only consider backups which are eligible for Postgres core + incremental backups: + + * backup_method = ``postgres`` + * summarize_wal = ``on`` + * is_incremental = ``False`` + + :return bool: True if it's a full backup or False if not. + """ + if ( + self.mode == "postgres" + and self.summarize_wal == "on" + and not self.is_incremental + ): + return True + return False + + +class SyntheticBackupInfo(LocalBackupInfo): + def __init__( + self, server, base_directory, backup_id=None, info_file=None, **kwargs + ): + """ + Stores meta information about a single synthetic backup. + + .. note:: + A synthetic backup is a base backup which was artificially created + through ``pg_combinebackup``. A synthetic backup is not part of + the Barman backup catalog, and only exists so we are able to + recover a backup created by ``pg_combinebackup`` utility, as + almost all functions and methods require a backup info object. + + The only difference from this class to its parent :class:`LocalBackupInfo` + is that it accepts a custom base directory for the backup as synthetic + backups are expected to live on directories other than the default + ``/base`` path. + + :param barman.server.Server server: the server that owns the + synthetic backup + :param str base_directory: the root directory where this synthetic + backup resides, essentially an override to the + ``server.config.basebackups_directory`` configuration. + :param str|None backup_id: the backup id of this backup + :param None|str|TextIO info_file: path or file descriptor of an existing + synthetic ``backup.info`` file + """ + self.base_directory = base_directory + super(SyntheticBackupInfo, self).__init__( + server, info_file, backup_id, **kwargs + ) + + def get_basebackup_directory(self): + """Get the backup directory based on its base directory""" + return os.path.join(self.base_directory, self.backup_id) diff --git a/barman/output.py b/barman/output.py index 85769815e..119411aea 100644 --- a/barman/output.py +++ b/barman/output.py @@ -676,23 +676,20 @@ def result_list_backup(self, backup_info, backup_size, wal_size, retention_statu self.info(backup_info.backup_id) return - out_list = ["%s %s " % (backup_info.server_name, backup_info.backup_id)] + out_list = ["%s %s" % (backup_info.server_name, backup_info.backup_id)] + if backup_info.backup_name is not None: - out_list.append("'%s' - " % backup_info.backup_name) - else: - out_list.append("- ") + out_list.append(" '%s'" % backup_info.backup_name) + + # Set backup type label + out_list.append(" - %s - " % backup_info.backup_type[0].upper()) + if backup_info.status in BackupInfo.STATUS_COPY_DONE: end_time = backup_info.end_time.ctime() out_list.append( "%s - Size: %s - WAL Size: %s" % (end_time, pretty_size(backup_size), pretty_size(wal_size)) ) - if backup_info.tablespaces: - tablespaces = [ - ("%s:%s" % (tablespace.name, tablespace.location)) - for tablespace in backup_info.tablespaces - ] - out_list.append(" (tablespaces: %s)" % ", ".join(tablespaces)) if backup_info.status == BackupInfo.WAITING_FOR_WALS: out_list.append(" - %s" % BackupInfo.WAITING_FOR_WALS) if retention_status and retention_status != BackupInfo.NONE: @@ -712,15 +709,51 @@ def render_show_backup_general(backup_info, output_fun, row): :param str row: format string which allows for `key: value` rows to be formatted """ - if "backup_name" in backup_info and backup_info["backup_name"] is not None: - output_fun(row.format("Backup Name", backup_info["backup_name"])) + backup_name = backup_info.get("backup_name") + if backup_name: + output_fun(row.format("Backup Name", backup_name)) + output_fun(row.format("Server Name", backup_info["server_name"])) - if backup_info["systemid"]: - output_fun(row.format("System Id", backup_info["systemid"])) + + system_id = backup_info.get("systemid") + if system_id: + output_fun(row.format("System Id", system_id)) + output_fun(row.format("Status", backup_info["status"])) if backup_info["status"] in BackupInfo.STATUS_COPY_DONE: output_fun(row.format("PostgreSQL Version", backup_info["version"])) output_fun(row.format("PGDATA directory", backup_info["pgdata"])) + cluster_size = backup_info.get("cluster_size") + if cluster_size: + output_fun( + row.format("Estimated Cluster Size", pretty_size(cluster_size)) + ) + output_fun("") + + @staticmethod + def render_show_backup_server(backup_info, output_fun, header_row, nested_row): + """ + Render server metadata in plain text form. + + :param dict backup_info: a dictionary containing the backup metadata + :param function output_fun: function which accepts a string and sends it to + an output writer + :param str header_row: format string which allows for single value header + rows to be formatted + :param str nested_row: format string which allows for `key: value` rows to be + formatted + """ + + data_checksums = backup_info.get("data_checksums") + summarize_wal = backup_info.get("summarize_wal") + if data_checksums or summarize_wal: + output_fun(header_row.format("Server information")) + if data_checksums: + output_fun( + nested_row.format("Checksums", backup_info["data_checksums"]) + ) + if summarize_wal: + output_fun(nested_row.format("WAL summarizer", summarize_wal)) output_fun("") @staticmethod @@ -794,66 +827,115 @@ def render_show_backup_base(backup_info, output_fun, header_row, nested_row): formatted """ output_fun(header_row.format("Base backup information")) - if backup_info["size"] is not None: - disk_usage_output = "{}".format(pretty_size(backup_info["size"])) - if "wal_size" in backup_info and backup_info["wal_size"] is not None: - disk_usage_output += " ({} with WALs)".format( - pretty_size(backup_info["size"] + backup_info["wal_size"]), + backup_method = backup_info.get("mode") + if backup_method: + output_fun(nested_row.format("Backup Method", backup_method)) + + # The "show-backup" for the cloud takes the input of a backup_info, + # not the result of a get_backup_ext_info() call, Instead, it + # takes a backup_info.to_dict(). So in those cases, the following + # fields will not exist: `backup_type`, `deduplication_ratio`, + # "root_backup_id", "chain_size", "est_dedup_size", "copy_time", + # "analysis_time" and "estimated_throughput". + # Because of this, this ugly piece of code is a temporary workaround + # so we do not break the show-backup for the "cloud-backup-show". + + backup_type = backup_info.get("backup_type") + backup_size = backup_info.get("deduplicated_size") + # Show only for postgres backups + if backup_type and backup_method == "postgres": + output_fun(nested_row.format("Backup Type", backup_type)) + + wal_size = backup_info.get("wal_size") + if backup_size: + backup_size_output = "{}".format(pretty_size(backup_size)) + if wal_size: + backup_size_output += " ({} with WALs)".format( + pretty_size(backup_size + wal_size), ) - output_fun(nested_row.format("Disk usage", disk_usage_output)) - if backup_info["deduplicated_size"] is not None and backup_info["size"] > 0: - deduplication_ratio = 1 - ( - float(backup_info["deduplicated_size"]) / backup_info["size"] - ) - dedupe_output = "{} (-{})".format( - pretty_size(backup_info["deduplicated_size"]), + output_fun(nested_row.format("Backup Size", backup_size_output)) + if wal_size: + output_fun(nested_row.format("WAL Size", pretty_size(wal_size))) + + # Show only for incremental and rsync backups + est_dedup_size = backup_info.get("est_dedup_size") + deduplication_ratio = backup_info.get("deduplication_ratio") + cluster_size = backup_info.get("cluster_size") + size = backup_info.get("size") + if est_dedup_size is None: + est_dedup_size = 0 + deduplication_ratio = 0 + sz = cluster_size + if backup_method != "postgres": + sz = size + if sz and backup_size: + deduplication_ratio = 1 - (backup_size / sz) + # The following operation needs to use cluster_size o size + # so we do not break backward compatibility when old backups + # taken with barman < 3.11 are present in the backup catalog. + # Old backups do not have the cluster_size field. + if cluster_size or size: + est_dedup_size = (cluster_size or size) * deduplication_ratio + + if backup_type and backup_type in {"rsync", "incremental"}: + dedupe_output = "{} ({})".format( + pretty_size(est_dedup_size), "{percent:.2%}".format(percent=deduplication_ratio), ) - output_fun(nested_row.format("Incremental size", dedupe_output)) + output_fun(nested_row.format("Resources saved", dedupe_output)) + output_fun(nested_row.format("Timeline", backup_info["timeline"])) output_fun(nested_row.format("Begin WAL", backup_info["begin_wal"])) output_fun(nested_row.format("End WAL", backup_info["end_wal"])) + # This is WAL stuff... - if "wal_num" in backup_info: - output_fun(nested_row.format("WAL number", backup_info["wal_num"])) - - if "wal_compression_ratio" in backup_info: - # Output WAL compression ratio for basebackup WAL files - if backup_info["wal_compression_ratio"] > 0: - wal_compression_output = "{percent:.2%}".format( - percent=backup_info["wal_compression_ratio"] - ) + wal_num = backup_info.get("wal_num") + if wal_num: + output_fun(nested_row.format("WAL number", wal_num)) + + wal_compression_ratio = backup_info.get("wal_compression_ratio", 0) + # Output WAL compression ratio for basebackup WAL files + if wal_compression_ratio > 0: + wal_compression_output = "{percent:.2%}".format( + percent=backup_info["wal_compression_ratio"] + ) - output_fun( - nested_row.format("WAL compression ratio", wal_compression_output) - ) + output_fun( + nested_row.format("WAL compression ratio", wal_compression_output) + ) # Back to regular stuff output_fun(nested_row.format("Begin time", backup_info["begin_time"])) output_fun(nested_row.format("End time", backup_info["end_time"])) - # If copy statistics are available print a summary - copy_stats = backup_info.get("copy_stats") - if copy_stats: - copy_time = copy_stats.get("copy_time") - if copy_time: - value = human_readable_timedelta(datetime.timedelta(seconds=copy_time)) - # Show analysis time if it is more than a second - analysis_time = copy_stats.get("analysis_time") - if analysis_time is not None and analysis_time >= 1: - value += " + {} startup".format( - human_readable_timedelta( - datetime.timedelta(seconds=analysis_time) - ) - ) - output_fun(nested_row.format("Copy time", value)) - size = backup_info["deduplicated_size"] or backup_info["size"] - if size is not None: - value = "{}/s".format(pretty_size(size / copy_time)) - number_of_workers = copy_stats.get("number_of_workers", 1) - if number_of_workers > 1: - value += " (%s jobs)" % number_of_workers - output_fun(nested_row.format("Estimated throughput", value)) + # If copy statistics are available, show summary + copy_time = backup_info.get("copy_time", 0) + analysis_time = backup_info.get("analysis_time", 0) + est_throughput = backup_info.get("estimated_throughput") + number_of_workers = backup_info.get("number_of_workers", 1) + + copy_stats = backup_info.get("copy_stats", {}) + if copy_stats and not copy_time: + copy_time = copy_stats.get("copy_time", 0) + analysis_time = copy_stats.get("analysis_time", 0) + number_of_workers = copy_stats.get("number_of_workers", 1) + if copy_time: + copy_time_output = human_readable_timedelta( + datetime.timedelta(seconds=copy_time) + ) + if analysis_time >= 1: + copy_time_output += " + {} startup".format( + human_readable_timedelta(datetime.timedelta(seconds=analysis_time)) + ) + output_fun(nested_row.format("Copy time", copy_time_output)) + if est_throughput: + est_througput_output = "{}/s".format(pretty_size(est_throughput)) + + if number_of_workers > 1: + est_througput_output += " (%s jobs)" % number_of_workers + output_fun( + nested_row.format("Estimated throughput", est_througput_output) + ) output_fun(nested_row.format("Begin Offset", backup_info["begin_offset"])) output_fun(nested_row.format("End Offset", backup_info["end_offset"])) @@ -968,6 +1050,28 @@ def render_show_backup_catalog_info( "multiple timelines interacting with this backup" ) + backup_type = backup_info.get("backup_type") + # Show only for incremental backups + if backup_type == "incremental": + output_fun(nested_row.format("Root Backup", backup_info["root_backup_id"])) + output_fun( + nested_row.format("Parent Backup", backup_info["parent_backup_id"]) + ) + output_fun( + nested_row.format("Backup chain size", backup_info["chain_size"]) + ) + backup_method = backup_info.get("mode") + + # Show only for postgres backups + if backup_method == "postgres": + if backup_info["children_backup_ids"] is not None: + output_fun( + nested_row.format( + "Children Backup(s)", + backup_info["children_backup_ids"], + ) + ) + @staticmethod def render_show_backup(backup_info, output_fun): """ @@ -983,6 +1087,9 @@ def render_show_backup(backup_info, output_fun): output_fun("Backup {}:".format(backup_info["backup_id"])) ConsoleOutputWriter.render_show_backup_general(backup_info, output_fun, row) + ConsoleOutputWriter.render_show_backup_server( + backup_info, output_fun, header_row, nested_row + ) if backup_info["status"] in BackupInfo.STATUS_COPY_DONE: ConsoleOutputWriter.render_show_backup_snapshots( backup_info, output_fun, header_row, nested_row @@ -1476,13 +1583,14 @@ def result_list_backup(self, backup_info, backup_size, wal_size, retention_statu self.json_output[server_name].append(backup_info.backup_id) return - output = dict( - backup_id=backup_info.backup_id, - ) + output = dict(backup_id=backup_info.backup_id) if backup_info.backup_name is not None: output.update({"backup_name": backup_info.backup_name}) + # Set backup type label + output.update({"backup_type": backup_info.backup_type}) + if backup_info.status in BackupInfo.STATUS_COPY_DONE: output.update( dict( @@ -1496,12 +1604,6 @@ def result_list_backup(self, backup_info, backup_size, wal_size, retention_statu retention_status=retention_status or BackupInfo.NONE, ) ) - output["tablespaces"] = [] - if backup_info.tablespaces: - for tablespace in backup_info.tablespaces: - output["tablespaces"].append( - dict(name=tablespace.name, location=tablespace.location) - ) else: output.update(dict(status=backup_info.status)) @@ -1510,24 +1612,50 @@ def result_list_backup(self, backup_info, backup_size, wal_size, retention_statu def result_show_backup(self, backup_ext_info): """ Output all available information about a backup in show-backup command + in json format. - The argument has to be the result - of a Server.get_backup_ext_info() call + The argument has to be the result of a + :meth:`barman.server.Server.get_backup_ext_info` call :param dict backup_ext_info: a dictionary containing the info to display """ data = dict(backup_ext_info) + server_name = data["server_name"] + # General information output = self.json_output[server_name] = dict( backup_id=data["backup_id"], status=data["status"] ) + backup_name = data.get("backup_name") + if backup_name: + output.update({"backup_name": backup_name}) + system_id = data.get("systemid") + if system_id: + output.update({"system_id": system_id}) + backup_type = data.get("backup_type") + if backup_type: + output.update({"backup_type": backup_type}) + + # Server information + output["server_information"] = dict( + data_checksums=data["data_checksums"], + summarize_wal=data["summarize_wal"], + ) - if "backup_name" in data and data["backup_name"] is not None: - output.update({"backup_name": data["backup_name"]}) - + cluster_size = data.get("cluster_size") if data["status"] in BackupInfo.STATUS_COPY_DONE: + # This check is needed to keep backward compatibility between + # barman versions <= 3.10.x, where cluster_size is not present. + if cluster_size: + output.update( + dict( + cluster_size=pretty_size(data["cluster_size"]), + cluster_size_bytes=data["cluster_size"], + ) + ) + # General information output.update( dict( postgresql_version=data["version"], @@ -1535,91 +1663,136 @@ def result_show_backup(self, backup_ext_info): tablespaces=[], ) ) - if "snapshots_info" in data and data["snapshots_info"]: - output["snapshots_info"] = data["snapshots_info"] - if data["tablespaces"]: - for item in data["tablespaces"]: - output["tablespaces"].append( - dict(name=item.name, location=item.location, oid=item.oid) - ) + # Base Backup information output["base_backup_information"] = dict( - disk_usage=pretty_size(data["size"]), - disk_usage_bytes=data["size"], - disk_usage_with_wals=pretty_size(data["size"] + data["wal_size"]), - disk_usage_with_wals_bytes=data["size"] + data["wal_size"], + backup_method=data["mode"], + backup_size=pretty_size(data["deduplicated_size"]), + backup_size_bytes=data["deduplicated_size"], + backup_size_with_wals=pretty_size( + data["deduplicated_size"] + data["wal_size"] + ), + backup_size_with_wals_bytes=data["deduplicated_size"] + + data["wal_size"], + wal_size=pretty_size(data["wal_size"]), + wal_size_bytes=data["wal_size"], + timeline=data["timeline"], + begin_wal=data["begin_wal"], + end_wal=data["end_wal"], + wal_num=data["wal_num"], + begin_time_timestamp=str(int(timestamp(data["begin_time"]))), + begin_time=data["begin_time"].isoformat(sep=" "), + end_time_timestamp=str(int(timestamp(data["end_time"]))), + end_time=data["end_time"].isoformat(sep=" "), + begin_offset=data["begin_offset"], + end_offset=data["end_offset"], + begin_lsn=data["begin_xlog"], + end_lsn=data["end_xlog"], ) - if data["deduplicated_size"] is not None and data["size"] > 0: - deduplication_ratio = 1 - ( - float(data["deduplicated_size"]) / data["size"] - ) + + if backup_type and backup_type in {"rsync", "incremental"}: output["base_backup_information"].update( dict( - incremental_size=pretty_size(data["deduplicated_size"]), - incremental_size_bytes=data["deduplicated_size"], - incremental_size_ratio="-{percent:.2%}".format( - percent=deduplication_ratio + resources_saved=pretty_size(data["est_dedup_size"]), + resources_saved_bytes=int(data["est_dedup_size"]), + resources_saved_percentage="{percent:.2%}".format( + percent=data["deduplication_ratio"] ), ) ) - output["base_backup_information"].update( - dict( - timeline=data["timeline"], - begin_wal=data["begin_wal"], - end_wal=data["end_wal"], - ) - ) - if data["wal_compression_ratio"] > 0: + + wal_comp_ratio = data.get("wal_compression_ratio", 0) + if wal_comp_ratio > 0: output["base_backup_information"].update( dict( wal_compression_ratio="{percent:.2%}".format( - percent=data["wal_compression_ratio"] + percent=wal_comp_ratio ) ) ) - output["base_backup_information"].update( - dict( - begin_time_timestamp=str(int(timestamp(data["begin_time"]))), - begin_time=data["begin_time"].isoformat(sep=" "), - end_time_timestamp=str(int(timestamp(data["end_time"]))), - end_time=data["end_time"].isoformat(sep=" "), + + cp_time = data.get("copy_time", 0) + ans_time = data.get("analysis_time", 0) + est_throughput = data.get("estimated_throughput") + num_workers = data.get("number_of_workers", 1) + + copy_stats = data.get("copy_stats") or {} + if copy_stats and not cp_time: + cp_time = copy_stats.get("copy_time", 0) + ans_time = copy_stats.get("analysis_time", 0) + num_workers = copy_stats.get("number_of_workers", 1) + if cp_time: + output["base_backup_information"].update( + dict( + copy_time=human_readable_timedelta( + datetime.timedelta(seconds=cp_time) + ), + copy_time_seconds=cp_time, + ) ) - ) - copy_stats = data.get("copy_stats") - if copy_stats: - copy_time = copy_stats.get("copy_time") - analysis_time = copy_stats.get("analysis_time", 0) - if copy_time: + if ans_time >= 1: output["base_backup_information"].update( dict( - copy_time=human_readable_timedelta( - datetime.timedelta(seconds=copy_time) - ), - copy_time_seconds=copy_time, analysis_time=human_readable_timedelta( - datetime.timedelta(seconds=analysis_time) + datetime.timedelta(seconds=ans_time) ), - analysis_time_seconds=analysis_time, + analysis_time_seconds=ans_time, ) ) - size = data["deduplicated_size"] or data["size"] + + if est_throughput is None: + est_throughput = data["deduplicated_size"] / cp_time + est_througput_output = "{}/s".format(pretty_size(est_throughput)) + output["base_backup_information"].update( + dict( + throughput=est_througput_output, + throughput_bytes=int(est_throughput), + ) + ) + if num_workers: output["base_backup_information"].update( dict( - throughput="%s/s" % pretty_size(size / copy_time), - throughput_bytes=size / copy_time, - number_of_workers=copy_stats.get("number_of_workers", 1), + number_of_workers=num_workers, ) ) - output["base_backup_information"].update( - dict( - begin_offset=data["begin_offset"], - end_offset=data["end_offset"], - begin_lsn=data["begin_xlog"], - end_lsn=data["end_xlog"], + # Tablespace information + if data["tablespaces"]: + for item in data["tablespaces"]: + output["tablespaces"].append( + dict(name=item.name, location=item.location, oid=item.oid) + ) + + # Backups catalog information + previous_backup_id = data.setdefault("previous_backup_id", "not available") + next_backup_id = data.setdefault("next_backup_id", "not available") + output["catalog_information"] = { + "retention_policy": data["retention_policy_status"] or "not enforced", + "previous_backup": previous_backup_id + or "- (this is the oldest base backup)", + "next_backup": next_backup_id or "- (this is the latest base backup)", + } + + if backup_type == "incremental": + output["catalog_information"].update( + dict( + root_backup_id=data["root_backup_id"], + parent_backup_id=data["parent_backup_id"], + chain_size=data["chain_size"], + ) ) - ) + if data["mode"] == "postgres": + children_bkp_ids = None + if data["children_backup_ids"]: + children_bkp_ids = data["children_backup_ids"].split(",") + output["catalog_information"].update( + dict( + children_backup_ids=children_bkp_ids, + ) + ) + + # WAL information wal_output = output["wal_information"] = dict( no_of_files=data["wal_until_next_num"], disk_usage=pretty_size(data["wal_until_next_size"]), @@ -1642,24 +1815,17 @@ def result_show_backup(self, backup_ext_info): percent=data["wal_until_next_compression_ratio"] ) if data["children_timelines"]: - wal_output[ - "_WARNING" - ] = "WAL information is inaccurate \ + wal_output["_WARNING"] = ( + "WAL information is inaccurate \ due to multiple timelines interacting with \ this backup" + ) for history in data["children_timelines"]: wal_output["timelines"].append(str(history.tli)) - previous_backup_id = data.setdefault("previous_backup_id", "not available") - next_backup_id = data.setdefault("next_backup_id", "not available") - - output["catalog_information"] = { - "retention_policy": data["retention_policy_status"] or "not enforced", - "previous_backup": previous_backup_id - or "- (this is the oldest base backup)", - "next_backup": next_backup_id or "- (this is the latest base backup)", - } - + # Snapshots information + if "snapshots_info" in data and data["snapshots_info"]: + output["snapshots_info"] = data["snapshots_info"] else: if data["error"]: output["error"] = data["error"] diff --git a/barman/postgres.py b/barman/postgres.py index 8acb84f02..21df0e3d4 100644 --- a/barman/postgres.py +++ b/barman/postgres.py @@ -25,6 +25,10 @@ import logging from abc import ABCMeta from multiprocessing import Process, Queue +import os +import signal +import threading +import time try: from queue import Empty @@ -40,6 +44,7 @@ ConninfoException, PostgresAppNameError, PostgresConnectionError, + PostgresConnectionLost, PostgresDuplicateReplicationSlot, PostgresException, PostgresInvalidReplicationSlot, @@ -457,6 +462,8 @@ class PostgreSQLConnection(PostgreSQL): WALSTREAMER = 2 ANY_STREAMING_CLIENT = (STANDBY, WALSTREAMER) + HEARTBEAT_QUERY = "SELECT 1" + def __init__( self, conninfo, @@ -498,6 +505,11 @@ def connect(self): raise PostgresAppNameError(force_str(e).strip()) return self._conn + @property + def has_connection(self): + """Checks if the Postgres connection has already been set""" + return True if self._conn is not None else False + @property def server_txt_version(self): """ @@ -598,11 +610,11 @@ def has_backup_privileges(self): OR ( ( - pg_has_role(CURRENT_USER, 'pg_monitor', 'MEMBER') + pg_has_role(CURRENT_USER, 'pg_monitor', 'USAGE') OR ( - pg_has_role(CURRENT_USER, 'pg_read_all_settings', 'MEMBER') - AND pg_has_role(CURRENT_USER, 'pg_read_all_stats', 'MEMBER') + pg_has_role(CURRENT_USER, 'pg_read_all_settings', 'USAGE') + AND pg_has_role(CURRENT_USER, 'pg_read_all_stats', 'USAGE') ) ) AND @@ -652,7 +664,7 @@ def has_checkpoint_privileges(self): return True else: role_check_query = ( - "select pg_has_role(CURRENT_USER ,'pg_checkpoint', 'MEMBER');" + "select pg_has_role(CURRENT_USER ,'pg_checkpoint', 'USAGE');" ) try: cur = self._cursor() @@ -682,11 +694,11 @@ def has_monitoring_privileges(self): monitoring_check_query = """ SELECT ( - pg_has_role(CURRENT_USER, 'pg_monitor', 'MEMBER') + pg_has_role(CURRENT_USER, 'pg_monitor', 'USAGE') OR ( - pg_has_role(CURRENT_USER, 'pg_read_all_settings', 'MEMBER') - AND pg_has_role(CURRENT_USER, 'pg_read_all_stats', 'MEMBER') + pg_has_role(CURRENT_USER, 'pg_read_all_settings', 'USAGE') + AND pg_has_role(CURRENT_USER, 'pg_read_all_stats', 'USAGE') ) ) """ @@ -1602,6 +1614,26 @@ def get_synchronous_standby_names(self): # the format of the synchronous_standby_names content return [x.strip().strip('"') for x in names_list.split(",")] + def send_heartbeat_query(self): + """ + Sends a heartbeat query to the server with the already opened connection. + + :returns tuple[bool, Exception|None]: A tuple where the first value is a boolean + indicating if the query executed successfully or not and the second is the + exception raised by ``psycopg2`` in case it did not succeed. + """ + try: + with self._conn.cursor() as cursor: + cursor.execute(self.HEARTBEAT_QUERY) + _logger.debug("Sent heartbeat query to maintain the current connection") + return True, None + except psycopg2.Error as ex: + _logger.debug( + "Failed to execute heartbeat query on the current connection: %s" + % force_str(ex) + ) + return False, ex + @property def name_map(self): """ @@ -1813,3 +1845,108 @@ def stop_exclusive_backup(self): return self._stop_backup( super(StandbyPostgreSQLConnection, self).stop_exclusive_backup ) + + +class PostgresKeepAlive: + """ + Context manager to maintain a Postgres connection alive. + + A child thread is spawned to execute heartbeat queries in the background + at a specified interval during its living context. + + It does not open or close any connections on its own. Instead, it waits for the + specified connection to be opened on the main thread before start sending any query. + + :cvar THREAD_NAME: The name identifying the keep-alive thread. + """ + + THREAD_NAME = "barman_keepalive_thread" + + def __init__(self, postgres, interval, raise_exception=False): + """ + Constructor. + + :param barman.postgres.PostgreSQLConnection postgres: The + Postgres connection to keep alive. + :param int interval: An interval in seconds at which a + heartbeat query will be sent to keep the connection alive. + A value <= ``0`` won't start the keepalive. + :param bool raise_exception: A boolean indicating if an exception + should be raised in case the connection is lost. If ``True``, a + ``PostgresConnectionLost`` exception will be raised as soon as + it's noticed a connection failure. If ``False``, it will keep executing + normally until the context exits. + """ + self.postgres = postgres + self.interval = interval + self.raise_exception = raise_exception + self._stop_thread = threading.Event() + self._thread = threading.Thread( + target=self._run_keep_alive, + name=self.THREAD_NAME, + ) + + def _prepare_signal_handler(self): + """ + Set up a signal handler to raise an exception on the main thread when + the keep-alive thread wishes to interrupt it. This method listens for a + ``SIGUSR1`` signal and, when received, raises a ``PostgresConnectionLost`` + exception. + + .. note:: + This code is, and only works if, executed while on the main thread. We + are not able to set a signal listener on a child thread, therefore this + method must be executed before the keep-alive thread starts. + """ + + def raise_exception(signum, frame): + raise PostgresConnectionLost("Connection to Postgres server was lost.") + + signal.signal(signal.SIGUSR1, raise_exception) + + def _raise_exception_on_main(self): + """ + Trigger an exception on the main thread at whatever frame is being executed + at the moment. This is done by sending a ``SIGUSR1`` signal to the process, + which will be caught by the signal handler set previously in this class. + + .. note:: + This is an alternative way of interrupting the main thread's work, since + there is no direct way of killing or raising exceptions on the main thread + from a child thread in Python. A handler for this signal has been set + beforehand by the ``_prepare_signal_handler`` method in this class. + """ + os.kill(os.getpid(), signal.SIGUSR1) + + def _run_keep_alive(self): + """Runs the keepalive until a stop-thread event is set""" + while not self._stop_thread.is_set(): + if not self.postgres.has_connection: + # Wait for the connection to be opened on the main thread + time.sleep(1) + continue + + success, ex = self.postgres.send_heartbeat_query() + + if not success and self.raise_exception: + # If one of the below exeptions was raised by psycopg2, it most likely + # means that the connection (and consequently, the session) was lost. In + # such cases, we can stop the keep-alive exection and raise the exception + if isinstance(ex, (psycopg2.InterfaceError, psycopg2.OperationalError)): + self._stop_thread.set() + self._raise_exception_on_main() + + self._stop_thread.wait(self.interval) + + def __enter__(self): + """Enters context. Starts the thread""" + if self.interval > 0: + if self.raise_exception: + self._prepare_signal_handler() + self._thread.start() + + def __exit__(self, exc_type, exc_val, exc_tb): + """Exits context. Makes sure the thread is terminated""" + if self.interval > 0: + self._stop_thread.set() + self._thread.join() diff --git a/barman/recovery_executor.py b/barman/recovery_executor.py index be2dccca5..4df0a7ad6 100644 --- a/barman/recovery_executor.py +++ b/barman/recovery_executor.py @@ -24,13 +24,14 @@ import collections import datetime + +from functools import partial import logging import os import re import shutil import socket import tempfile -import time from io import BytesIO import dateutil.parser @@ -38,7 +39,7 @@ from barman import output, xlog from barman.cloud_providers import get_snapshot_interface_from_backup_info -from barman.command_wrappers import RsyncPgData +from barman.command_wrappers import PgCombineBackup, RsyncPgData from barman.config import RecoveryOptions from barman.copy_controller import RsyncCopyController from barman.exceptions import ( @@ -59,8 +60,8 @@ NoneCompression, ) import barman.fs as fs -from barman.infofile import BackupInfo, LocalBackupInfo -from barman.utils import force_str, mkpath +from barman.infofile import BackupInfo, LocalBackupInfo, SyntheticBackupInfo +from barman.utils import force_str, mkpath, total_seconds # generic logger for this module _logger = logging.getLogger(__name__) @@ -254,7 +255,12 @@ def recover( # Retrieve a list of required log files required_xlog_files = tuple( self.server.get_required_xlog_files( - backup_info, target_tli, recovery_info["target_epoch"] + backup_info, + target_tli, + None, + None, + target_lsn, + target_immediate, ) ) @@ -448,7 +454,6 @@ def _set_pitr_targets( is reached :param str|None target_action: recovery target action for PITR """ - target_epoch = None target_datetime = None # Calculate the integer value of TLI if a keyword is provided @@ -492,6 +497,12 @@ def _set_pitr_targets( tzinfo=dateutil.tz.tzlocal() ) + output.warning( + "No time zone has been specified through '--target-time' " + "command-line option. Barman assumed the same time zone from " + "the Barman host.", + ) + # Check if the target time is reachable from the # selected backup if backup_info.end_time > target_datetime: @@ -501,8 +512,6 @@ def _set_pitr_targets( % (target_datetime, backup_info.end_time) ) - ms = target_datetime.microsecond / 1000000.0 - target_epoch = time.mktime(target_datetime.timetuple()) + ms targets["time"] = str(target_datetime) if target_xid: targets["xid"] = str(target_xid) @@ -572,7 +581,6 @@ def _set_pitr_targets( "Can't enable recovery target action when PITR is not required" ) - recovery_info["target_epoch"] = target_epoch recovery_info["target_datetime"] = target_datetime def _retrieve_safe_horizon(self, recovery_info, backup_info, dest): @@ -1048,7 +1056,19 @@ def _generate_recovery_conf( # Writes recovery target if target_time: - recovery_conf_lines.append("recovery_target_time = '%s'" % target_time) + # 'target_time' is the value as it came from '--target-time' command-line + # option, which may be without a time zone. When writing the actual Postgres + # configuration we should use a value with an explicit time zone set, so we + # avoid hitting pitfalls. We use the 'target_datetime' which was prevously + # added to 'recovery_info'. It already handles the cases where the user + # specifies no time zone, and uses the Barman host time zone as a fallback. + # In short: if 'target_time' is present it means the user asked for a + # specific point in time, but we need a sanitized value to use in the + # Postgres configuration, so we use 'target_datetime'. + # See '_set_pitr_targets'. + recovery_conf_lines.append( + "recovery_target_time = '%s'" % recovery_info["target_datetime"], + ) if target_xid: recovery_conf_lines.append("recovery_target_xid = '%s'" % target_xid) if target_lsn: @@ -1400,6 +1420,7 @@ def _backup_copy( output.info( "Staging compressed backup files on the recovery host in: %s", staging_dir ) + recovery_info["cmd"].create_dir_if_not_exists(staging_dir, mode="700") recovery_info["cmd"].validate_file_mode(staging_dir, mode="700") recovery_info["staging_dir"] = staging_dir @@ -1790,6 +1811,321 @@ def _backup_copy(self, backup_info, dest, remote_command=None, **kwargs): raise DataTransferFailure.from_command_error("rsync", e, msg) +class IncrementalRecoveryExecutor(RemoteConfigRecoveryExecutor): + """ + Recovery executor for recovery of Postgres incremental backups. + + This class implements the combine backup process as well as the + recovery of the newly combined backup by reusing some of the logic + from the :class:`RecoveryExecutor` class. + """ + + def __init__(self, backup_manager): + """ + Constructor + + :param barman.backup.BackupManager backup_manager: the :class:`BackupManager` + owner of the executor + """ + super(IncrementalRecoveryExecutor, self).__init__(backup_manager) + self.combine_start_time = None + self.combine_end_time = None + + def recover(self, backup_info, dest, remote_command=None, **kwargs): + """ + Performs the recovery of an incremental backup. + + It first combines all backups in the backup chain, full to incremental, + then proceeds with the recovery of the generated synthetic backup. + + This method should be called in a :func:`contextlib.closing` context. + + :param barman.infofile.BackupInfo backup_info: the incremental + backup to recover + :param str dest: the destination directory + :param str|None remote_command: The remote command to recover + the base backup, in case of remote backup. + :return dict: ``recovery_info`` dictionary, holding the values related + with the recovery process. + """ + # First combine the backups, generating a new synthetic backup in the staging area + combine_directory = self.config.local_staging_path + synthetic_backup_info = self._combine_backups(backup_info, combine_directory) + + # Add the backup directory created in the staging area to be deleted after recovery + synthetic_backup_dir = synthetic_backup_info.get_basebackup_directory() + self.temp_dirs.append(fs.LocalLibPathDeletionCommand(synthetic_backup_dir)) + + # Perform the standard recovery process passing the synthetic backup + recovery_info = super(IncrementalRecoveryExecutor, self).recover( + synthetic_backup_info, dest, remote_command=remote_command, **kwargs + ) + + # If the checksum configuration is not consistent among all backups in the chain, we + # raise a warning at the end so the user can optionally take action about it + if not backup_info.is_checksum_consistent(): + output.warning( + "You recovered from an incremental backup where checksums were enabled on " + "that backup, but not all backups in the chain. It is advised to disable, and " + "optionally re-enable, checksums on the destination directory to avoid failures." + ) + + return recovery_info + + def _combine_backups(self, backup_info, dest): + """ + Combines the backup chain into a single synthetic backup using the + ``pg_combinebackup`` utility. + + :param barman.infofile.LocalBackupInfo backup_info: the incremental + backup to be recovered + :param str dest: the directory where the synthetic backup is going + to be mounted on + :return barman.infofile.SyntheticBackupInfo: the backup info file of the + combined backup + """ + self.combine_start_time = datetime.datetime.now() + + # Build the synthetic backup info from the incremental backup as it has + # the most recent data relevant to the recovery. Also, the combine process + # should be transparent to the end user so e.g. the .barman-recover.info file + # that is created on destination and also the backup_id that is appended to the + # manifest file in further steps of the recovery should be the same as the incremental + synthetic_backup_info = SyntheticBackupInfo( + self.server, + base_directory=dest, + backup_id=backup_info.backup_id, + ) + synthetic_backup_info.load(filename=backup_info.filename) + + dest_dirs = [synthetic_backup_info.get_data_directory()] + + # Maps the tablespaces from the old backup directory to the new synthetic + # backup directory. This mapping is passed to the pg_combinebackup as input + tbs_map = {} + if backup_info.tablespaces: + for tablespace in backup_info.tablespaces: + source = backup_info.get_data_directory(tablespace_oid=tablespace.oid) + destination = synthetic_backup_info.get_data_directory( + tablespace_oid=tablespace.oid + ) + tbs_map[source] = destination + dest_dirs.append(destination) + + # Prepare the destination directories for pgdata and tablespaces + for _dir in dest_dirs: + self._prepare_destination(_dir) + + # Retrieve pg_combinebackup version information + remote_status = self._fetch_remote_status() + + # Get the backup chain data paths to be passed to the pg_combinebackup + backups_chain = self._get_backup_chain_paths(backup_info) + + self._start_message(synthetic_backup_info) + + pg_combinebackup = PgCombineBackup( + destination=synthetic_backup_info.get_data_directory(), + command=remote_status["pg_combinebackup_path"], + version=remote_status["pg_combinebackup_version"], + app_name=None, + tbs_mapping=tbs_map, + retry_times=self.config.basebackup_retry_times, + retry_sleep=self.config.basebackup_retry_sleep, + retry_handler=partial(self._retry_handler, dest_dirs), + out_handler=PgCombineBackup.make_logging_handler(logging.INFO), + args=backups_chain, + ) + + # Do the actual combine + try: + pg_combinebackup() + except CommandFailedException as e: + msg = "Combine action failure on directory '%s'" % dest + raise DataTransferFailure.from_command_error("pg_combinebackup", e, msg) + + self._end_message(synthetic_backup_info) + + self.combine_end_time = datetime.datetime.now() + combine_time = total_seconds(self.combine_end_time - self.combine_start_time) + synthetic_backup_info.copy_stats = { + "combine_time": combine_time, + } + + return synthetic_backup_info + + def _backup_copy( + self, + backup_info, + dest, + tablespaces=None, + remote_command=None, + **kwargs, + ): + """ + Perform the actual copy/move of the synthetic backup to destination + + :param barman.infofile.SyntheticBackupInfo backup_info: the synthetic + backup info file + :param str dest: the destination directory + :param dict[str,str]|None tablespaces: a tablespace + name -> location map (for relocation) + :param str|None remote_command: default ``None``. The remote command to + recover the backup, in case of remote backup + """ + # If it is a remote recovery we just follow the standard rsync copy process + if remote_command: + super(IncrementalRecoveryExecutor, self)._backup_copy( + backup_info, dest, tablespaces, remote_command, **kwargs + ) + return + # If it is a local recovery we move the content from staging to destination + # Starts with tablespaces + if backup_info.tablespaces: + for tablespace in backup_info.tablespaces: + # By default a tablespace goes in the same location where + # it was on the source server when the backup was taken + destination = tablespace.location + # If a relocation has been requested for this tablespace + # use the user provided target directory + if tablespaces and tablespace.name in tablespaces: + destination = tablespaces[tablespace.name] + # Move the content of the tablespace directory to destination directory + self._prepare_destination(destination) + tbs_source = backup_info.get_data_directory( + tablespace_oid=tablespace.oid + ) + self._move_to_destination(source=tbs_source, destination=destination) + + # Then procede to move the content of the data directory + # We don't move the pg_tblspc as the _prepare_tablespaces method called earlier + # in the process already created this directory and required symlinks in the destination + data_source = backup_info.get_data_directory() + self._move_to_destination( + source=data_source, destination=dest, exclude_path_names={"pg_tblspc"} + ) + + def _move_to_destination(self, source, destination, exclude_path_names=set()): + """ + Move all files and directories contained within *source* to *destination*. + + :param str source: the source directory path from which underlying + files and directories will be moved + :param str destination: the destination directory path where to move the + files and directories contained within *source* + :param set[str] exclude_path_names: name of directories or files to be + excluded from the moving action. + """ + for file_or_dir in os.listdir(source): + if file_or_dir not in exclude_path_names: + file_or_dir_path = os.path.join(source, file_or_dir) + try: + shutil.move(file_or_dir_path, destination) + except shutil.Error: + output.error( + "Destination directory '%s' must be empty." % destination + ) + output.close_and_exit() + + def _get_backup_chain_paths(self, backup_info): + """ + Get the path of each backup in the chain, from the full backup to + the specified incremental backup. + + :param barman.infofile.LocalBackupInfo backup_info: The incremental backup + :return Iterator[barman.infofile.LocalBackupInfo]: iterator of paths of + the backups in the chain, going from the full to the incremental backup + pointed by *backup_info* + """ + return reversed( + [backup.get_data_directory() for backup in backup_info.walk_to_root()] + ) + + def _prepare_destination(self, dest_dir): + """ + Prepare the destination directory or file before moving it. + + This method is responsible for removing a directory if it already + exists, then (re)creating it and ensuring the correct permissions + on the directory. + + :param str dest_dir: destination directory + """ + # Remove a dir if exists. Ignore eventual errors + shutil.rmtree(dest_dir, ignore_errors=True) + # create the dir + mkpath(dest_dir) + # Ensure the right permissions to the destination directory + # chmod 0700 octal + os.chmod(dest_dir, 448) + + def _retry_handler(self, dest_dirs, attempt): + """ + Handler invoked during a combine backup in case of retry. + + The method simply warn the user of the failure and + remove the already existing directories of the backup. + + :param list[str] dest_dirs: destination directories + :param int attempt: attempt number (starting from 0) + """ + output.warning( + "Failure combining backups using pg_combinebackup (attempt %s)", attempt + ) + output.warning( + "The files created so far will be removed and " + "the combine process will restart in %s seconds", + "30", + ) + # Remove all the destination directories and reinit the backup + for _dir in dest_dirs: + self._prepare_destination(_dir) + + def _fetch_remote_status(self): + """ + Gather info from the remote server. + + This method does not raise any exception in case of errors, + but set the missing values to ``None`` in the resulting dictionary. + + :return dict[str, str|bool]: the pg_combinebackup client information + of the remote server. + """ + remote_status = dict.fromkeys( + ( + "pg_combinebackup_installed", + "pg_combinebackup_path", + "pg_combinebackup_version", + ), + None, + ) + + # Test pg_combinebackup existence + version_info = PgCombineBackup.get_version_info(self.server.path) + + if version_info["full_path"]: + remote_status["pg_combinebackup_installed"] = True + remote_status["pg_combinebackup_path"] = version_info["full_path"] + remote_status["pg_combinebackup_version"] = version_info["full_version"] + else: + remote_status["pg_combinebackup_installed"] = False + + return remote_status + + def _start_message(self, backup_info): + output.info( + "Start combining backup via pg_combinebackup for backup %s on %s", + backup_info.backup_id, + backup_info.base_directory, + ) + + def _end_message(self, backup_info): + output.info( + "End combining backup via pg_combinebackup for backup %s", + backup_info.backup_id, + ) + + def recovery_executor_factory(backup_manager, command, backup_info): """ Method in charge of building adequate RecoveryExecutor depending on the context @@ -1797,6 +2133,8 @@ def recovery_executor_factory(backup_manager, command, backup_info): :param: command barman.fs.UnixLocalCommand :return: RecoveryExecutor instance """ + if backup_info.is_incremental: + return IncrementalRecoveryExecutor(backup_manager) if backup_info.snapshots_info is not None: return SnapshotRecoveryExecutor(backup_manager) compression = backup_info.compression diff --git a/barman/retention_policies.py b/barman/retention_policies.py index 5be1b9c68..ec8b820f7 100644 --- a/barman/retention_policies.py +++ b/barman/retention_policies.py @@ -126,6 +126,33 @@ def to_json(self): """ return "%s %s %s" % (self.mode, self.value, self.unit) + def _propagate_retention_status_to_children(self, backup_info, report, ret_status): + """ + Propagate retention status to all backups in the tree. + + .. note:: + This has a side-effect. It modifies or add data to *report* dict. + + :param barman.infofile.BackupInfo backup_info: The object we want to + propagate the RETENTION STATUS from. + :param dict[str, str] report: The report data structure to be modified. + Each key is the ID of a backup, and its value is the retention status + of that backup. + :param str ret_status: The status of the backup according to retention + policies + """ + # As KEEP status doesn't make sense for incremental backups, we simply + # set them as VALID if their root full backup has KEEP annotation + if ret_status in (BackupInfo.KEEP_STANDALONE, BackupInfo.KEEP_FULL): + ret_status = BackupInfo.VALID + backup_tree = backup_info.walk_backups_tree(return_self=False) + for backup in backup_tree: + report[backup.backup_id] = ret_status + _logger.debug( + "Propagating %s retention status of backup %s to %s." + % (ret_status, backup_info.backup_id, backup.backup_id) + ) + class RedundancyRetentionPolicy(RetentionPolicy): """ @@ -170,6 +197,13 @@ def _backup_report(self, source): # NOTE: reverse key orders (simulate reverse chronology) i = 0 for bid in sorted(backups.keys(), reverse=True): + if backups[bid].is_incremental: + _logger.debug( + "Ignoring incremental backup %s. The retention status will" + " be propagated from %s." + % (backups[bid], backups[bid].parent_backup_id) + ) + continue if backups[bid].status == BackupInfo.DONE: keep_target = self.server.get_keep_target(bid) if keep_target == KeepManager.TARGET_STANDALONE: @@ -185,6 +219,14 @@ def _backup_report(self, source): i = i + 1 else: report[bid] = BackupInfo.NONE + + if backups[bid].has_children: + status = report[bid] + self._propagate_retention_status_to_children( + backup_info=backups[bid], + report=report, + ret_status=status, + ) return report def _wal_report(self): @@ -271,6 +313,13 @@ def _backup_report(self, source): valid = 0 # NOTE: reverse key orders (simulate reverse chronology) for bid in sorted(backups.keys(), reverse=True): + if backups[bid].is_incremental: + _logger.debug( + "Ignoring incremental backup %s. The retention status will" + " be propagated from %s." + % (backups[bid], backups[bid].parent_backup_id) + ) + continue # We are interested in DONE backups only if backups[bid].status == BackupInfo.DONE: keep_target = self.server.get_keep_target(bid) @@ -350,6 +399,14 @@ def _backup_report(self, source): found = True else: report[bid] = BackupInfo.NONE + + if backups[bid].has_children: + status = report[bid] + self._propagate_retention_status_to_children( + backup_info=backups[bid], + report=report, + ret_status=status, + ) return report def _wal_report(self): diff --git a/barman/server.py b/barman/server.py index 21c958bd5..b04a587eb 100644 --- a/barman/server.py +++ b/barman/server.py @@ -44,6 +44,7 @@ from barman.copy_controller import RsyncCopyController from barman.exceptions import ( ArchiverFailure, + BackupException, BadXlogSegmentName, CommandFailedException, ConninfoException, @@ -1512,50 +1513,96 @@ def show(self): output.result("show_server", self.config.name, result) def delete_backup(self, backup): - """Deletes a backup + """ + Deletes a backup. + + Performs some checks to confirm that the backup can indeed be deleted + and if so it is deleted along with all backups that depend on it, if any. - :param backup: the backup to delete + :param barman.infofile.LocalBackupInfo backup: the backup to delete + :return bool: True if deleted, False if could not delete the backup """ - try: - # Lock acquisition: if you can acquire a ServerBackupLock - # it means that no other processes like a backup or another delete - # are running on that server for that backup id, - # so there is no need to check the backup status. - # Simply proceed with the normal delete process. - server_backup_lock = ServerBackupLock( - self.config.barman_lock_directory, self.config.name + if self.backup_manager.should_keep_backup(backup.backup_id): + output.warning( + "Skipping delete of backup %s for server %s " + "as it has a current keep request. If you really " + "want to delete this backup please remove the keep " + "and try again.", + backup.backup_id, + self.config.name, ) - server_backup_lock.acquire( - server_backup_lock.raise_if_fail, server_backup_lock.wait + return False + + # Honour minimum required redundancy + available_backups = self.get_available_backups(status_filter=(BackupInfo.DONE,)) + minimum_redundancy = self.config.minimum_redundancy + if backup.status == BackupInfo.DONE and minimum_redundancy >= len( + available_backups + ): + output.warning( + "Skipping delete of backup %s for server %s " + "due to minimum redundancy requirements " + "(minimum redundancy = %s, " + "current redundancy = %s)", + backup.backup_id, + self.config.name, + minimum_redundancy, + len(available_backups), + ) + return False + + if backup.children_backup_ids: + output.warning( + "Backup %s has incremental backups which depend on it. " + "Deleting all backups in the tree", + backup.backup_id, ) - server_backup_lock.release() + + try: + # Lock acquisition: if you can acquire a ServerBackupLock it means + # that no other processes like a backup or another delete is running + with ServerBackupLock(self.config.barman_lock_directory, self.config.name): + # Delete the backup along with all its descendants in the + # backup tree i.e. all its subsequent incremental backups. + # If it has no descendants or it is an rsync backup then + # only the current backup is deleted. + deleted = False + backups_to_delete = backup.walk_backups_tree() + for del_backup in backups_to_delete: + deleted = self.perform_delete_backup(del_backup) + if not deleted and del_backup.backup_id != backup.backup_id: + output.error( + "Failed to delete one of its incremental backups. Make sure " + "all its dependent backups are deletable and try again." + ) + break + + return deleted except LockFileBusy: - # Otherwise if the lockfile is busy, a backup process is actually - # running on that server. To be sure that it's safe - # to delete the backup, we must check its status and its position - # in the catalogue. - # If it is the first and it is STARTED or EMPTY, we are trying to - # remove a running backup. This operation must be forbidden. - # Otherwise, normally delete the backup. - first_backup_id = self.get_first_backup_id(BackupInfo.STATUS_ALL) - if backup.backup_id == first_backup_id and backup.status in ( - BackupInfo.STARTED, - BackupInfo.EMPTY, - ): - output.error( - "Another action is in progress for the backup %s" - " of server %s. Impossible to delete the backup." - % (backup.backup_id, self.config.name) - ) - return + # Otherwise if the lockfile is busy, a backup process is actually running + output.error( + "Another process in running on server %s. " + "Impossible to delete the backup." % self.config.name + ) + return False except LockFilePermissionDenied as e: # We cannot access the lockfile. # Exit without removing the backup. output.error("Permission denied, unable to access '%s'" % e) - return + return False + + def perform_delete_backup(self, backup): + """ + Performs the deletion of a backup. + Deletes a single backup, ensuring that no other process can access + the backup simultaneously during its deletion. + + :param barman.infofile.LocalBackupInfo backup: the backup to delete + :return bool: True if deleted, False if could not delete the backup + """ try: # Take care of the backup lock. # Only one process can modify a backup at a time @@ -1581,15 +1628,15 @@ def delete_backup(self, backup): "Another process is holding the lock for " "backup %s of server %s." % (backup.backup_id, self.config.name) ) - return + return False except LockFilePermissionDenied as e: # We cannot access the lockfile. # warn the user and terminate output.error("Permission denied, unable to access '%s'" % e) - return + return False - def backup(self, wait=False, wait_timeout=None, backup_name=None): + def backup(self, wait=False, wait_timeout=None, backup_name=None, **kwargs): """ Performs a backup for the server :param bool wait: wait for all the required WAL files to be archived @@ -1598,12 +1645,16 @@ def backup(self, wait=False, wait_timeout=None, backup_name=None): before timing out :param str|None backup_name: a friendly name by which this backup can be referenced in the future + :kwparam str parent_backup_id: id of the parent backup when taking a + Postgres incremental backup """ # The 'backup' command is not available on a passive node. # We assume that if we get here the node is not passive assert not self.passive_node try: + # validate arguments, raise BackupException if any error is found + self.backup_manager.validate_backup_args(**kwargs) # Default strategy for check in backup is CheckStrategy # This strategy does not print any output - it only logs checks strategy = CheckStrategy() @@ -1616,6 +1667,9 @@ def backup(self, wait=False, wait_timeout=None, backup_name=None): return # check required backup directories exist self._make_directories() + except BackupException as e: + output.error("failed to start backup: %s", force_str(e)) + return except OSError as e: output.error("failed to create %s directory: %s", e.filename, e.strerror) return @@ -1635,6 +1689,7 @@ def backup(self, wait=False, wait_timeout=None, backup_name=None): wait=wait, wait_timeout=wait_timeout, name=backup_name, + **kwargs, ) # Archive incoming WALs and update WAL catalogue @@ -1650,6 +1705,16 @@ def backup(self, wait=False, wait_timeout=None, backup_name=None): if not previous_backup: self.backup_manager.remove_wal_before_backup(backup_info) + # check if the backup chain (in case it is a Postgres incremental) is consistent + # with their checksums configurations + if not backup_info.is_checksum_consistent(): + output.warning( + "This is an incremental backup taken with `data_checksums = on` whereas " + "some previous backups in the chain were taken with `data_checksums = off`. " + "This can lead to potential recovery issues. Consider taking a new full backup " + "to avoid having inconsistent backup chains." + ) + if backup_info.status == BackupInfo.WAITING_FOR_WALS: output.warning( "IMPORTANT: this backup is classified as " @@ -1681,11 +1746,23 @@ def get_last_backup_id(self, status_filter=BackupManager.DEFAULT_STATUS_FILTER): """ Get the id of the latest/last backup in the catalog (if exists) + :param status_filter: The status of the backup to return, + default to :attr:`BackupManager.DEFAULT_STATUS_FILTER`. + :return str|None: ID of the backup + """ + return self.backup_manager.get_last_backup_id(status_filter) + + def get_last_full_backup_id( + self, status_filter=BackupManager.DEFAULT_STATUS_FILTER + ): + """ + Get the id of the latest/last FULL backup in the catalog (if exists) + :param status_filter: The status of the backup to return, default to DEFAULT_STATUS_FILTER. :return string|None: ID of the backup """ - return self.backup_manager.get_last_backup_id(status_filter) + return self.backup_manager.get_last_full_backup_id(status_filter) def get_first_backup_id(self, status_filter=BackupManager.DEFAULT_STATUS_FILTER): """ @@ -1772,13 +1849,36 @@ def get_next_backup(self, backup_id): return self.backup_manager.get_next_backup(backup_id) def get_required_xlog_files( - self, backup, target_tli=None, target_time=None, target_xid=None + self, + backup, + target_tli=None, + target_time=None, + target_xid=None, + target_lsn=None, + target_immediate=False, ): """ - Get the xlog files required for a recovery - params: BackupInfo backup: a backup object - params: target_tli : target timeline - param: target_time: target time + Get the xlog files required for a recovery. + + .. note:: + *target_time* and *target_xid* are ignored by this method. As it can be very + expensive to parse WAL dumps to identify which WAL files are required to + honor the specific targets, we simply copy all WAL files up to the + calculated target timeline, so we make sure recovery will be able to finish + successfully (assuming the archived WALs honor the specified targets). + + On the other hand, *target_tli*, *target_lsn* and *target_immediate* are + easier to handle, so we only copy the WALs required to reach the requested + targets. + + :param BackupInfo backup: a backup object + :param target_tli : target timeline, either a timeline ID or one of the keywords + supported by Postgres + :param target_time: target time, in epoch + :param target_xid: target transaction ID + :param target_lsn: target LSN + :param target_immediate: target that ends recovery as soon as + consistency is reached. Defaults to ``False``. """ begin = backup.begin_wal end = backup.end_wal @@ -1799,6 +1899,16 @@ def get_required_xlog_files( if not target_tli: target_tli, _, _ = xlog.decode_segment_name(end) calculated_target_tli = target_tli + + # If a target LSN was specified, get the name of the last WAL file that is + # required for the recovery process + if target_lsn: + target_wal = xlog.location_to_xlogfile_name_offset( + target_lsn, + calculated_target_tli, + backup.xlog_segment_size, + )["file_name"] + with self.xlogdb() as fxlogdb: for line in fxlogdb: wal_info = WalFileInfo.from_xlogdb_line(line) @@ -1812,11 +1922,13 @@ def get_required_xlog_files( tli, _, _ = xlog.decode_segment_name(wal_info.name) if tli > calculated_target_tli: continue - yield wal_info if wal_info.name > end: - end = wal_info.name - if target_time and wal_info.time > target_time: + if target_immediate: break + if target_lsn and wal_info.name > target_wal: + break + end = wal_info.name + yield wal_info # return all the remaining history files for line in fxlogdb: wal_info = WalFileInfo.from_xlogdb_line(line) @@ -2828,12 +2940,12 @@ def receive_wal(self, reset=False): except LockFileBusy: # If another process is running for this server, if reset: - output.info( + output.error( "Unable to reset the status of receive-wal " "for server %s. Process is still running" % self.config.name ) else: - output.info( + output.error( "Another receive-wal process is already running " "for server %s." % self.config.name ) @@ -2925,6 +3037,9 @@ def get_backup_ext_info(self, backup_info): * the Server.get_wal_info() return value * the context in the catalog (if available) * the retention policy status + * the copy statistics + * the incremental backups information + * extra backup.info properties :param backup_info: the target backup :rtype dict: all information about a backup @@ -2938,27 +3053,24 @@ def get_backup_ext_info(self, backup_info): next_backup = self.backup_manager.get_next_backup( backup_ext_info["backup_id"] ) + backup_ext_info["previous_backup_id"] = None + backup_ext_info["next_backup_id"] = None if previous_backup: backup_ext_info["previous_backup_id"] = previous_backup.backup_id - else: - backup_ext_info["previous_backup_id"] = None if next_backup: backup_ext_info["next_backup_id"] = next_backup.backup_id - else: - backup_ext_info["next_backup_id"] = None except UnknownBackupIdException: # no next_backup_id and previous_backup_id items # means "Not available" pass backup_ext_info.update(self.get_wal_info(backup_info)) + + backup_ext_info["retention_policy_status"] = None if self.enforce_retention_policies: policy = self.config.retention_policy backup_ext_info["retention_policy_status"] = policy.backup_status( backup_info.backup_id ) - else: - backup_ext_info["retention_policy_status"] = None - # Check any child timeline exists children_timelines = self.get_children_timelines( backup_ext_info["timeline"], forked_after=backup_info.end_xlog @@ -2966,6 +3078,43 @@ def get_backup_ext_info(self, backup_info): backup_ext_info["children_timelines"] = children_timelines + # If copy statistics are available + copy_stats = backup_ext_info.get("copy_stats") + if copy_stats: + analysis_time = copy_stats.get("analysis_time", 0) + if analysis_time >= 1: + backup_ext_info["analysis_time"] = analysis_time + copy_time = copy_stats.get("copy_time", 0) + if copy_time > 0: + backup_ext_info["copy_time"] = copy_time + dedup_size = backup_ext_info.get("deduplicated_size", 0) + if dedup_size > 0: + estimated_throughput = dedup_size / copy_time + backup_ext_info["estimated_throughput"] = estimated_throughput + number_of_workers = copy_stats.get("number_of_workers", 1) + if number_of_workers > 1: + backup_ext_info["number_of_workers"] = number_of_workers + + backup_chain = [backup for backup in backup_info.walk_to_root()] + chain_size = len(backup_chain) + # last is root + root_backup_info = backup_chain[-1] + # "Incremental" backups + backup_ext_info["root_backup_id"] = root_backup_info.backup_id + backup_ext_info["chain_size"] = chain_size + # Properties added to the result dictionary + backup_ext_info["backup_type"] = backup_info.backup_type + backup_ext_info["deduplication_ratio"] = backup_info.deduplication_ratio + # A new field "cluster_size" was added to backup.info to be + # able to calculate the resource saved by "incremental" backups + # introduced in Postgres 17. + # To keep backward compatibility between versions, barman relies + # on two possible values to calculate "est_dedup_size", + # "size" being used for older versions when "cluster_size" + # is non existent (None). + backup_ext_info["est_dedup_size"] = ( + backup_ext_info["cluster_size"] or backup_ext_info["size"] + ) * backup_ext_info["deduplication_ratio"] return backup_ext_info def show_backup(self, backup_info): diff --git a/barman/utils.py b/barman/utils.py index 139fec95b..08752e67d 100644 --- a/barman/utils.py +++ b/barman/utils.py @@ -864,6 +864,8 @@ def get_backup_id_using_shortcut(server, shortcut, BackupInfo): backup_id = server.get_first_backup_id() elif shortcut in ("last-failed"): backup_id = server.get_last_backup_id([BackupInfo.FAILED]) + elif shortcut in ("latest-full", "last-full"): + backup_id = server.get_last_full_backup_id() elif is_backup_id(shortcut): backup_id = shortcut diff --git a/barman/version.py b/barman/version.py index 5ac2cab3c..a0430c656 100644 --- a/barman/version.py +++ b/barman/version.py @@ -20,4 +20,4 @@ This module contains the current Barman version. """ -__version__ = "3.10.0" +__version__ = "3.11.1" diff --git a/doc/barman-cloud-backup-delete.1 b/doc/barman-cloud-backup-delete.1 index 08088d8f6..8e6a75a27 100644 --- a/doc/barman-cloud-backup-delete.1 +++ b/doc/barman-cloud-backup-delete.1 @@ -1,6 +1,6 @@ .\" Automatically generated by Pandoc 2.2.1 .\" -.TH "BARMAN\-CLOUD\-BACKUP\-DELETE" "1" "January 24, 2024" "Barman User manuals" "Version 3.10.0" +.TH "BARMAN\-CLOUD\-BACKUP\-DELETE" "1" "August 22, 2024" "Barman User manuals" "Version 3.11.1" .hy .SH NAME .PP diff --git a/doc/barman-cloud-backup-delete.1.md b/doc/barman-cloud-backup-delete.1.md index 16d4e3c7f..a413973e9 100644 --- a/doc/barman-cloud-backup-delete.1.md +++ b/doc/barman-cloud-backup-delete.1.md @@ -1,6 +1,6 @@ -% BARMAN-CLOUD-BACKUP-DELETE(1) Barman User manuals | Version 3.10.0 +% BARMAN-CLOUD-BACKUP-DELETE(1) Barman User manuals | Version 3.11.1 % EnterpriseDB -% January 24, 2024 +% August 22, 2024 # NAME diff --git a/doc/barman-cloud-backup-keep.1 b/doc/barman-cloud-backup-keep.1 index 395d81dbd..cf2b6513e 100644 --- a/doc/barman-cloud-backup-keep.1 +++ b/doc/barman-cloud-backup-keep.1 @@ -1,6 +1,6 @@ .\" Automatically generated by Pandoc 2.2.1 .\" -.TH "BARMAN\-CLOUD\-BACKUP\-DELETE" "1" "January 24, 2024" "Barman User manuals" "Version 3.10.0" +.TH "BARMAN\-CLOUD\-BACKUP\-DELETE" "1" "August 22, 2024" "Barman User manuals" "Version 3.11.1" .hy .SH NAME .PP diff --git a/doc/barman-cloud-backup-keep.1.md b/doc/barman-cloud-backup-keep.1.md index 92c020422..797d2a83d 100644 --- a/doc/barman-cloud-backup-keep.1.md +++ b/doc/barman-cloud-backup-keep.1.md @@ -1,6 +1,6 @@ -% BARMAN-CLOUD-BACKUP-DELETE(1) Barman User manuals | Version 3.10.0 +% BARMAN-CLOUD-BACKUP-DELETE(1) Barman User manuals | Version 3.11.1 % EnterpriseDB -% January 24, 2024 +% August 22, 2024 # NAME diff --git a/doc/barman-cloud-backup-list.1 b/doc/barman-cloud-backup-list.1 index 1919aab59..bd1394efc 100644 --- a/doc/barman-cloud-backup-list.1 +++ b/doc/barman-cloud-backup-list.1 @@ -1,6 +1,6 @@ .\" Automatically generated by Pandoc 2.2.1 .\" -.TH "BARMAN\-CLOUD\-BACKUP\-LIST" "1" "January 24, 2024" "Barman User manuals" "Version 3.10.0" +.TH "BARMAN\-CLOUD\-BACKUP\-LIST" "1" "August 22, 2024" "Barman User manuals" "Version 3.11.1" .hy .SH NAME .PP diff --git a/doc/barman-cloud-backup-list.1.md b/doc/barman-cloud-backup-list.1.md index 41bd5a399..04ee124ce 100644 --- a/doc/barman-cloud-backup-list.1.md +++ b/doc/barman-cloud-backup-list.1.md @@ -1,6 +1,6 @@ -% BARMAN-CLOUD-BACKUP-LIST(1) Barman User manuals | Version 3.10.0 +% BARMAN-CLOUD-BACKUP-LIST(1) Barman User manuals | Version 3.11.1 % EnterpriseDB -% January 24, 2024 +% August 22, 2024 # NAME diff --git a/doc/barman-cloud-backup-show.1 b/doc/barman-cloud-backup-show.1 index 5075341e2..fe1ba031d 100644 --- a/doc/barman-cloud-backup-show.1 +++ b/doc/barman-cloud-backup-show.1 @@ -1,6 +1,6 @@ .\" Automatically generated by Pandoc 2.2.1 .\" -.TH "BARMAN\-CLOUD\-BACKUP\-SHOW" "1" "January 24, 2024" "Barman User manuals" "Version 3.10.0" +.TH "BARMAN\-CLOUD\-BACKUP\-SHOW" "1" "August 22, 2024" "Barman User manuals" "Version 3.11.1" .hy .SH NAME .PP diff --git a/doc/barman-cloud-backup-show.1.md b/doc/barman-cloud-backup-show.1.md index 220a0cab7..f5b164b23 100644 --- a/doc/barman-cloud-backup-show.1.md +++ b/doc/barman-cloud-backup-show.1.md @@ -1,6 +1,6 @@ -% BARMAN-CLOUD-BACKUP-SHOW(1) Barman User manuals | Version 3.10.0 +% BARMAN-CLOUD-BACKUP-SHOW(1) Barman User manuals | Version 3.11.1 % EnterpriseDB -% January 24, 2024 +% August 22, 2024 # NAME diff --git a/doc/barman-cloud-backup.1 b/doc/barman-cloud-backup.1 index 0d593bbbb..692f7d842 100644 --- a/doc/barman-cloud-backup.1 +++ b/doc/barman-cloud-backup.1 @@ -1,6 +1,6 @@ .\" Automatically generated by Pandoc 2.2.1 .\" -.TH "BARMAN\-CLOUD\-BACKUP" "1" "January 24, 2024" "Barman User manuals" "Version 3.10.0" +.TH "BARMAN\-CLOUD\-BACKUP" "1" "August 22, 2024" "Barman User manuals" "Version 3.11.1" .hy .SH NAME .PP diff --git a/doc/barman-cloud-backup.1.md b/doc/barman-cloud-backup.1.md index 48d201ca6..bd5b7701e 100644 --- a/doc/barman-cloud-backup.1.md +++ b/doc/barman-cloud-backup.1.md @@ -1,6 +1,6 @@ -% BARMAN-CLOUD-BACKUP(1) Barman User manuals | Version 3.10.0 +% BARMAN-CLOUD-BACKUP(1) Barman User manuals | Version 3.11.1 % EnterpriseDB -% January 24, 2024 +% August 22, 2024 # NAME diff --git a/doc/barman-cloud-check-wal-archive.1 b/doc/barman-cloud-check-wal-archive.1 index 88b38c878..94fdade2f 100644 --- a/doc/barman-cloud-check-wal-archive.1 +++ b/doc/barman-cloud-check-wal-archive.1 @@ -1,7 +1,7 @@ .\" Automatically generated by Pandoc 2.2.1 .\" -.TH "BARMAN\-CLOUD\-CHECK\-WAL\-ARCHIVE" "1" "January 24, 2024" "Barman User manuals" "Version -3.10.0" +.TH "BARMAN\-CLOUD\-CHECK\-WAL\-ARCHIVE" "1" "August 22, 2024" "Barman User manuals" "Version +3.11.1" .hy .SH NAME .PP diff --git a/doc/barman-cloud-check-wal-archive.1.md b/doc/barman-cloud-check-wal-archive.1.md index 4db3db38a..b8accb3fd 100644 --- a/doc/barman-cloud-check-wal-archive.1.md +++ b/doc/barman-cloud-check-wal-archive.1.md @@ -1,6 +1,6 @@ -% BARMAN-CLOUD-CHECK-WAL-ARCHIVE(1) Barman User manuals | Version 3.10.0 +% BARMAN-CLOUD-CHECK-WAL-ARCHIVE(1) Barman User manuals | Version 3.11.1 % EnterpriseDB -% January 24, 2024 +% August 22, 2024 # NAME diff --git a/doc/barman-cloud-restore.1 b/doc/barman-cloud-restore.1 index 765611252..0444b16b7 100644 --- a/doc/barman-cloud-restore.1 +++ b/doc/barman-cloud-restore.1 @@ -1,6 +1,6 @@ .\" Automatically generated by Pandoc 2.2.1 .\" -.TH "BARMAN\-CLOUD\-RESTORE" "1" "January 24, 2024" "Barman User manuals" "Version 3.10.0" +.TH "BARMAN\-CLOUD\-RESTORE" "1" "August 22, 2024" "Barman User manuals" "Version 3.11.1" .hy .SH NAME .PP diff --git a/doc/barman-cloud-restore.1.md b/doc/barman-cloud-restore.1.md index 46fadccd3..0fc9d990a 100644 --- a/doc/barman-cloud-restore.1.md +++ b/doc/barman-cloud-restore.1.md @@ -1,6 +1,6 @@ -% BARMAN-CLOUD-RESTORE(1) Barman User manuals | Version 3.10.0 +% BARMAN-CLOUD-RESTORE(1) Barman User manuals | Version 3.11.1 % EnterpriseDB -% January 24, 2024 +% August 22, 2024 # NAME diff --git a/doc/barman-cloud-wal-archive.1 b/doc/barman-cloud-wal-archive.1 index 3cda8ca1e..4ab81c393 100644 --- a/doc/barman-cloud-wal-archive.1 +++ b/doc/barman-cloud-wal-archive.1 @@ -1,6 +1,6 @@ .\" Automatically generated by Pandoc 2.2.1 .\" -.TH "BARMAN\-CLOUD\-WAL\-ARCHIVE" "1" "January 24, 2024" "Barman User manuals" "Version 3.10.0" +.TH "BARMAN\-CLOUD\-WAL\-ARCHIVE" "1" "August 22, 2024" "Barman User manuals" "Version 3.11.1" .hy .SH NAME .PP diff --git a/doc/barman-cloud-wal-archive.1.md b/doc/barman-cloud-wal-archive.1.md index b54e5aa34..04c5a166f 100644 --- a/doc/barman-cloud-wal-archive.1.md +++ b/doc/barman-cloud-wal-archive.1.md @@ -1,6 +1,6 @@ -% BARMAN-CLOUD-WAL-ARCHIVE(1) Barman User manuals | Version 3.10.0 +% BARMAN-CLOUD-WAL-ARCHIVE(1) Barman User manuals | Version 3.11.1 % EnterpriseDB -% January 24, 2024 +% August 22, 2024 # NAME diff --git a/doc/barman-cloud-wal-restore.1 b/doc/barman-cloud-wal-restore.1 index 0054faaa5..f1cc8a5b3 100644 --- a/doc/barman-cloud-wal-restore.1 +++ b/doc/barman-cloud-wal-restore.1 @@ -1,6 +1,6 @@ .\" Automatically generated by Pandoc 2.2.1 .\" -.TH "BARMAN\-CLOUD\-WAL\-RESTORE" "1" "January 24, 2024" "Barman User manuals" "Version 3.10.0" +.TH "BARMAN\-CLOUD\-WAL\-RESTORE" "1" "August 22, 2024" "Barman User manuals" "Version 3.11.1" .hy .SH NAME .PP @@ -30,6 +30,7 @@ usage:\ barman\-cloud\-wal\-restore\ [\-V]\ [\-\-help]\ [\-v\ |\ \-q]\ [\-t] \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ [\-\-profile\ AWS_PROFILE] \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ [\-\-read\-timeout\ READ_TIMEOUT] \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ [\-\-azure\-credential\ {azure\-cli,managed\-identity}] +\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ [\-\-no\-partial] \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ source_url\ server_name\ wal_name\ wal_dest This\ script\ can\ be\ used\ as\ a\ `restore_command`\ to\ download\ WAL\ files @@ -53,6 +54,7 @@ optional\ arguments: \ \ \-t,\ \-\-test\ \ \ \ \ \ \ \ \ \ \ \ Test\ cloud\ connectivity\ and\ exit \ \ \-\-cloud\-provider\ {aws\-s3,azure\-blob\-storage,google\-cloud\-storage} \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ The\ cloud\ provider\ to\ use\ as\ a\ storage\ backend +\ \ \-\-no\-partial\ \ \ \ \ \ \ \ \ \ Do\ not\ download\ partial\ WAL\ files Extra\ options\ for\ the\ aws\-s3\ cloud\ provider: \ \ \-\-endpoint\-url\ ENDPOINT_URL diff --git a/doc/barman-cloud-wal-restore.1.md b/doc/barman-cloud-wal-restore.1.md index 7675e3b26..a0fcc496c 100644 --- a/doc/barman-cloud-wal-restore.1.md +++ b/doc/barman-cloud-wal-restore.1.md @@ -1,6 +1,6 @@ -% BARMAN-CLOUD-WAL-RESTORE(1) Barman User manuals | Version 3.10.0 +% BARMAN-CLOUD-WAL-RESTORE(1) Barman User manuals | Version 3.11.1 % EnterpriseDB -% January 24, 2024 +% August 22, 2024 # NAME @@ -30,6 +30,7 @@ usage: barman-cloud-wal-restore [-V] [--help] [-v | -q] [-t] [--profile AWS_PROFILE] [--read-timeout READ_TIMEOUT] [--azure-credential {azure-cli,managed-identity}] + [--no-partial] source_url server_name wal_name wal_dest This script can be used as a `restore_command` to download WAL files @@ -53,6 +54,7 @@ optional arguments: -t, --test Test cloud connectivity and exit --cloud-provider {aws-s3,azure-blob-storage,google-cloud-storage} The cloud provider to use as a storage backend + --no-partial Do not download partial WAL files Extra options for the aws-s3 cloud provider: --endpoint-url ENDPOINT_URL diff --git a/doc/barman-wal-archive.1 b/doc/barman-wal-archive.1 index 04f3ab60a..77b1f9320 100644 --- a/doc/barman-wal-archive.1 +++ b/doc/barman-wal-archive.1 @@ -1,6 +1,6 @@ .\" Automatically generated by Pandoc 2.2.1 .\" -.TH "BARMAN\-WAL\-ARCHIVE" "1" "January 24, 2024" "Barman User manuals" "Version 3.10.0" +.TH "BARMAN\-WAL\-ARCHIVE" "1" "August 22, 2024" "Barman User manuals" "Version 3.11.1" .hy .SH NAME .PP diff --git a/doc/barman-wal-archive.1.md b/doc/barman-wal-archive.1.md index a336a5be5..cdecdd73e 100644 --- a/doc/barman-wal-archive.1.md +++ b/doc/barman-wal-archive.1.md @@ -1,6 +1,6 @@ -% BARMAN-WAL-ARCHIVE(1) Barman User manuals | Version 3.10.0 +% BARMAN-WAL-ARCHIVE(1) Barman User manuals | Version 3.11.1 % EnterpriseDB -% January 24, 2024 +% August 22, 2024 # NAME diff --git a/doc/barman-wal-restore.1 b/doc/barman-wal-restore.1 index a1a9669a2..628469af8 100644 --- a/doc/barman-wal-restore.1 +++ b/doc/barman-wal-restore.1 @@ -1,6 +1,6 @@ .\" Automatically generated by Pandoc 2.2.1 .\" -.TH "BARMAN\-WAL\-RESTORE" "1" "January 24, 2024" "Barman User manuals" "Version 3.10.0" +.TH "BARMAN\-WAL\-RESTORE" "1" "August 22, 2024" "Barman User manuals" "Version 3.11.1" .hy .SH NAME .PP diff --git a/doc/barman-wal-restore.1.md b/doc/barman-wal-restore.1.md index 1b6866b83..eaee5e2dc 100644 --- a/doc/barman-wal-restore.1.md +++ b/doc/barman-wal-restore.1.md @@ -1,6 +1,6 @@ -% BARMAN-WAL-RESTORE(1) Barman User manuals | Version 3.10.0 +% BARMAN-WAL-RESTORE(1) Barman User manuals | Version 3.11.1 % EnterpriseDB -% January 24, 2024 +% August 22, 2024 # NAME diff --git a/doc/barman.1 b/doc/barman.1 index a59842d77..f6f1e09cb 100644 --- a/doc/barman.1 +++ b/doc/barman.1 @@ -1,6 +1,6 @@ .\" Automatically generated by Pandoc 2.2.1 .\" -.TH "BARMAN" "1" "January 24, 2024" "Barman User manuals" "Version 3.10.0" +.TH "BARMAN" "1" "August 22, 2024" "Barman User manuals" "Version 3.11.1" .hy .SH NAME .PP @@ -98,6 +98,14 @@ present in the configuration file. .RS .RE .TP +.B \-\-incremental [BACKUP_ID] +performs a block\-level incremental backup. +A \f[C]BACKUP_ID\f[] or backup ID shortcut of a previous backup must be +provided, which references a previous backup in the catalog to be used +as the parent backup from which the incremental is taken. +.RS +.RE +.TP .B \-\-reuse\-backup [INCREMENTAL_TYPE] Overrides \f[C]reuse_backup\f[] option behaviour. Possible values for \f[C]INCREMENTAL_TYPE\f[] are: @@ -179,6 +187,14 @@ archived before timing out .RS .RE .TP +.B \-\-keepalive\-interval +an interval, in seconds, at which a hearbeat query will be sent to the +server to keep the libpq connection alive during an Rsync backup. +Default is 60. +A value of 0 disables it. +.RS +.RE +.TP .B \-\-manifest forces the creation of a backup manifest file at the end of a backup. Overrides value of the parameter \f[C]autogenerate_manifest\f[], from @@ -377,49 +393,48 @@ either directly or by retention policy. .TP .B list\-backups \f[I]SERVER_NAME\f[] Show available backups for \f[C]SERVER_NAME\f[]. -This command is useful to retrieve a backup ID. +This command is useful to retrieve a backup ID and the backup type. For example: .RS .RE .IP .nf \f[C] -servername\ 20111104T102647\ \-\ Fri\ Nov\ \ 4\ 10:26:48\ 2011\ \-\ Size:\ 17.0\ MiB\ \-\ WAL\ Size:\ 100\ B +servername\ 20111104T102647\ \-\ F\ \-\ Fri\ Nov\ \ 4\ 10:26:48\ 2011\ \-\ Size:\ 17.0\ MiB\ \-\ WAL\ Size:\ 100\ B \f[] .fi +.PP +In this case, \f[I]20111104T102647\f[] is the backup ID, and \f[C]F\f[] +is the backup type label for a full backup taken with +\f[C]pg_basebackup\f[]. +The backup type label displayed by this command uses one of the +following values: \- \f[C]F\f[]: for full backups taken with +\f[C]pg_basebackup\f[] \- \f[C]I\f[]: for incremental backups taken with +\f[C]pg_basebackup\f[] \- \f[C]R\f[]: for backups taken with +\f[C]rsync\f[] \- \f[C]S\f[]: for cloud snapshot backups list\-files +\f[I][OPTIONS]\f[] \f[I]SERVER_NAME\f[] \f[I]BACKUP_ID\f[] : List all +the files in a particular backup, identified by the server name and the +backup ID. +See the Backup ID shortcuts section below for available shortcuts. .IP .nf \f[C] -In\ this\ case,\ *20111104T102647*\ is\ the\ backup\ ID. +\-\-target\ *TARGET_TYPE* +:\ \ \ \ Possible\ values\ for\ TARGET_TYPE\ are: + +\ \ \ \ \ \-\ *data*:\ lists\ just\ the\ data\ files; +\ \ \ \ \ \-\ *standalone*:\ lists\ the\ base\ backup\ files,\ including\ required +\ \ \ \ \ \ \ WAL\ files; +\ \ \ \ \ \-\ *wal*:\ lists\ all\ the\ WAL\ files\ between\ the\ start\ of\ the\ base +\ \ \ \ \ \ \ backup\ and\ the\ end\ of\ the\ log\ /\ the\ start\ of\ the\ following\ base +\ \ \ \ \ \ \ backup\ (depending\ on\ whether\ the\ specified\ base\ backup\ is\ the\ most +\ \ \ \ \ \ \ recent\ one\ available); +\ \ \ \ \ \-\ *full*:\ same\ as\ data\ +\ wal. + +\ \ \ \ The\ default\ value\ is\ `standalone`. \f[] .fi .TP -.B list\-files \f[I][OPTIONS]\f[] \f[I]SERVER_NAME\f[] \f[I]BACKUP_ID\f[] -List all the files in a particular backup, identified by the server name -and the backup ID. -See the Backup ID shortcuts section below for available shortcuts. -.RS -.TP -.B \-\-target \f[I]TARGET_TYPE\f[] -Possible values for TARGET_TYPE are: -.RS -.IP \[bu] 2 -\f[I]data\f[]: lists just the data files; -.IP \[bu] 2 -\f[I]standalone\f[]: lists the base backup files, including required WAL -files; -.IP \[bu] 2 -\f[I]wal\f[]: lists all the WAL files between the start of the base -backup and the end of the log / the start of the following base backup -(depending on whether the specified base backup is the most recent one -available); -.IP \[bu] 2 -\f[I]full\f[]: same as data + wal. -.PP -The default value is \f[C]standalone\f[]. -.RE -.RE -.TP .B list\-servers Show all the configured servers, and their descriptions. .RS @@ -653,6 +668,16 @@ and has no effect otherwise. .RS .RE .TP +.B \-\-local\-staging\-path \f[I]STAGING_PATH\f[] +A path to a location on the barman host where the chain of backups will +be combined before being copied to the destination directory. +Contents created inside the staging path are removed at the end of the +recovery process. +This option is \f[I]required\f[] when recovering from incremental +backups (backup_method=postgres) and has no effect otherwise. +.RS +.RE +.TP .B \-\-recovery\-conf\-filename \f[I]RECOVERY_CONF_FILENAME\f[] The name of the file where Barman should write the PostgreSQL recovery options when recovering backups for PostgreSQL versions 12 and later. @@ -739,21 +764,30 @@ server Show detailed information about a particular backup, identified by the server name and the backup ID. See the Backup ID shortcuts section below for available shortcuts. -For example: +The following example is from a block\-level incremental backup (which +requires Postgres version >= 17): .RS .RE .IP .nf \f[C] -Backup\ 20150828T130001: +Backup\ 20240814T017504: \ \ Server\ Name\ \ \ \ \ \ \ \ \ \ \ \ :\ quagmire \ \ Status\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ :\ DONE \ \ PostgreSQL\ Version\ \ \ \ \ :\ 90402 \ \ PGDATA\ directory\ \ \ \ \ \ \ :\ /srv/postgresql/9.4/main/data +\ \ Estimated\ Cluster\ Size\ :\ 22.4\ MiB + +\ \ Server\ information: +\ \ \ \ Checksums\ \ \ \ \ \ \ \ \ \ \ \ :\ on +\ \ \ \ WAL\ summarizer\ \ \ \ \ \ \ :\ on \ \ Base\ backup\ information: -\ \ \ \ Disk\ usage\ \ \ \ \ \ \ \ \ \ \ :\ 12.4\ TiB\ (12.4\ TiB\ with\ WALs) -\ \ \ \ Incremental\ size\ \ \ \ \ :\ 4.9\ TiB\ (\-60.02%) +\ \ \ \ Backup\ Method\ \ \ \ \ \ \ \ :\ postgres +\ \ \ \ Backup\ Type\ \ \ \ \ \ \ \ \ \ :\ incremental +\ \ \ \ Backup\ Size\ \ \ \ \ \ \ \ \ \ :\ 22.3\ MiB\ (54.3\ MiB\ with\ WALs) +\ \ \ \ WAL\ Size\ \ \ \ \ \ \ \ \ \ \ \ \ :\ 32.0\ MiB +\ \ \ \ Resources\ saved\ \ \ \ \ :\ 19.5\ MiB\ (86.80%) \ \ \ \ Timeline\ \ \ \ \ \ \ \ \ \ \ \ \ :\ 1 \ \ \ \ Begin\ WAL\ \ \ \ \ \ \ \ \ \ \ \ :\ 0000000100000CFD000000AD \ \ \ \ End\ WAL\ \ \ \ \ \ \ \ \ \ \ \ \ \ :\ 0000000100000D0D00000008 @@ -761,6 +795,8 @@ Backup\ 20150828T130001: \ \ \ \ WAL\ compression\ ratio:\ 79.51% \ \ \ \ Begin\ time\ \ \ \ \ \ \ \ \ \ \ :\ 2015\-08\-28\ 13:00:01.633925+00:00 \ \ \ \ End\ time\ \ \ \ \ \ \ \ \ \ \ \ \ :\ 2015\-08\-29\ 10:27:06.522846+00:00 +\ \ \ \ Copy\ time\ \ \ \ \ \ \ \ \ \ \ \ :\ 1\ second +\ \ \ \ Estimated\ throughput\ :\ 2.0\ MiB/s \ \ \ \ Begin\ Offset\ \ \ \ \ \ \ \ \ :\ 1575048 \ \ \ \ End\ Offset\ \ \ \ \ \ \ \ \ \ \ :\ 13853016 \ \ \ \ Begin\ XLOG\ \ \ \ \ \ \ \ \ \ \ :\ CFD/AD180888 @@ -777,8 +813,54 @@ Backup\ 20150828T130001: \ \ \ \ Retention\ Policy\ \ \ \ \ :\ not\ enforced \ \ \ \ Previous\ Backup\ \ \ \ \ \ :\ 20150821T130001 \ \ \ \ Next\ Backup\ \ \ \ \ \ \ \ \ \ :\ \-\ (this\ is\ the\ latest\ base\ backup) +\ \ \ \ Root\ Backup\ \ \ \ \ \ \ \ \ \ :\ 20240814T015504 +\ \ \ \ Parent\ Backup\ \ \ \ \ \ \ \ :\ 20240814T016504 +\ \ \ \ Backup\ chain\ size\ \ \ \ :\ 3 +\ \ \ \ Children\ Backup(s)\ \ \ :\ 20240814T018515 \f[] .fi +.RS +.PP +\f[B]NOTE:\f[] Depending on the version of your Postgres Server and/or +the type of the backup, the output of \f[C]barman\ show\-backup\f[] +command may be different. +For example, fields like "Root Backup", "Parent Backup", "Backup chain +size", and "Children Backup(s)" only make sense when showing information +about a block\-level incremental backup taken with +\f[C]backup_method\ =\ postgres\f[] and using Postgres 17 or newer, thus +those fields are omitted for other kind of backups or older versions of +Postgres. +.PP +Also note that \f[C]show\-backup\f[] relies on the backup metadata so if +a backup was created with Barman version 3.10 or earlier, the backup +will not contain the fields added in version 3.11 (which are those added +after the introduction of "incremental" backups in PostgreSQL 17). +.PP +These are the possible values for the field "Backup Type": +.IP \[bu] 2 +\f[C]rsync\f[]: for a backup taken with \f[C]rsync\f[]; +.IP \[bu] 2 +\f[C]full\f[]: for a full backup taken with \f[C]pg_basebackup\f[]; +.IP \[bu] 2 +\f[C]incremental\f[]: for an incremental backup taken with +\f[C]pg_basebackup\f[]; +.IP \[bu] 2 +\f[C]snapshot\f[]: for a snapshot\-based backup taken in the cloud. +.PP +Below you can find a list of fields that may be shown or omitted +depending on the type of the backup: +.IP \[bu] 2 +\f[C]Resources\ saved\f[]: available for "rsync" and "incremental" +backups; +.IP \[bu] 2 +\f[C]Root\ Backup\f[], \f[C]Parent\ Backup\f[], +\f[C]Backup\ chain\ size\f[]: available for "incremental" backups only; +.IP \[bu] 2 +\f[C]Children\ Backup(s)\f[]: available for "full" and "incremental" +backups; +.IP \[bu] 2 +\f[C]Snapshot\ information\f[]: available for "snapshot" backups only. +.RE .TP .B show\-servers \f[I]SERVER_NAME\f[] Show information about \f[C]SERVER_NAME\f[], including: @@ -932,7 +1014,17 @@ same ast \f[I]first\f[]. Latest failed backup, in chronological order. .RS .RE -.SH EXIT STATUS +.TP +.B last\-full +Latest full\-backup eligible for a block\-level incremental backup using +the \f[C]\-\-incremental\f[] option. +.RS +.RE +.TP +.B latest\-full +same as \f[I]last\-full\f[] # EXIT STATUS +.RS +.RE .TP .B 0 Success diff --git a/doc/barman.1.d/00-header.md b/doc/barman.1.d/00-header.md index faf682a09..e98833620 100644 --- a/doc/barman.1.d/00-header.md +++ b/doc/barman.1.d/00-header.md @@ -1,3 +1,3 @@ -% BARMAN(1) Barman User manuals | Version 3.10.0 +% BARMAN(1) Barman User manuals | Version 3.11.1 % EnterpriseDB -% January 24, 2024 +% August 22, 2024 diff --git a/doc/barman.1.d/50-backup.md b/doc/barman.1.d/50-backup.md index 816b5ae8a..6a2c929ae 100644 --- a/doc/barman.1.d/50-backup.md +++ b/doc/barman.1.d/50-backup.md @@ -19,6 +19,11 @@ backup *SERVER_NAME* Overrides value of the parameter `immediate_checkpoint`, if present in the configuration file. + --incremental [BACKUP_ID] + : performs a block-level incremental backup. A `BACKUP_ID` or [backup ID shortcut](#shortcuts) + of a previous backup must be provided, which references a previous backup + in the catalog to be used as the parent backup from which the incremental is taken. + --reuse-backup [INCREMENTAL_TYPE] : Overrides `reuse_backup` option behaviour. Possible values for `INCREMENTAL_TYPE` are: @@ -75,6 +80,11 @@ backup *SERVER_NAME* : the time, in seconds, spent waiting for the required WAL files to be archived before timing out + --keepalive-interval + : an interval, in seconds, at which a hearbeat query will be sent to the + server to keep the libpq connection alive during an Rsync backup. Default + is 60. A value of 0 disables it. + --manifest : forces the creation of a backup manifest file at the end of a backup. Overrides value of the parameter `autogenerate_manifest`, diff --git a/doc/barman.1.d/50-list-backups.md b/doc/barman.1.d/50-list-backups.md index 799a7730c..5d436bc62 100644 --- a/doc/barman.1.d/50-list-backups.md +++ b/doc/barman.1.d/50-list-backups.md @@ -1,9 +1,13 @@ list-backups *SERVER_NAME* : Show available backups for `SERVER_NAME`. This command is useful to - retrieve a backup ID. For example: + retrieve a backup ID and the backup type. For example: ``` -servername 20111104T102647 - Fri Nov 4 10:26:48 2011 - Size: 17.0 MiB - WAL Size: 100 B +servername 20111104T102647 - F - Fri Nov 4 10:26:48 2011 - Size: 17.0 MiB - WAL Size: 100 B ``` - In this case, *20111104T102647* is the backup ID. +In this case, *20111104T102647* is the backup ID, and `F` is the backup type label for a full backup taken with `pg_basebackup`. The backup type label displayed by this command uses one of the following values: + - `F`: for full backups taken with `pg_basebackup` + - `I`: for incremental backups taken with `pg_basebackup` + - `R`: for backups taken with `rsync` + - `S`: for cloud snapshot backups \ No newline at end of file diff --git a/doc/barman.1.d/50-recover.md b/doc/barman.1.d/50-recover.md index 965d55eea..9ef3ae907 100644 --- a/doc/barman.1.d/50-recover.md +++ b/doc/barman.1.d/50-recover.md @@ -106,6 +106,14 @@ recover *\[OPTIONS\]* *SERVER_NAME* *BACKUP_ID* *DESTINATION_DIRECTORY* This option is *required* when recovering from compressed backups and has no effect otherwise. + --local-staging-path *STAGING_PATH* + : A path to a location on the barman host where the chain of backups will + be combined before being copied to the destination directory. Contents + created inside the staging path are removed at the + end of the recovery process. This option is *required* when recovering + from incremental backups (backup_method=postgres) and has no effect + otherwise. + --recovery-conf-filename *RECOVERY_CONF_FILENAME* : The name of the file where Barman should write the PostgreSQL recovery options when recovering backups for PostgreSQL versions 12 and later. diff --git a/doc/barman.1.d/50-show-backup.md b/doc/barman.1.d/50-show-backup.md index 9e8a491ec..57b81d8c3 100644 --- a/doc/barman.1.d/50-show-backup.md +++ b/doc/barman.1.d/50-show-backup.md @@ -1,18 +1,27 @@ show-backup *SERVER_NAME* *BACKUP_ID* : Show detailed information about a particular backup, identified by the server name and the backup ID. See the [Backup ID shortcuts](#shortcuts) - section below for available shortcuts. For example: + section below for available shortcuts. The following example is from + a block-level incremental backup (which requires Postgres version >= 17): ``` -Backup 20150828T130001: +Backup 20240814T017504: Server Name : quagmire Status : DONE PostgreSQL Version : 90402 PGDATA directory : /srv/postgresql/9.4/main/data + Estimated Cluster Size : 22.4 MiB + + Server information: + Checksums : on + WAL summarizer : on Base backup information: - Disk usage : 12.4 TiB (12.4 TiB with WALs) - Incremental size : 4.9 TiB (-60.02%) + Backup Method : postgres + Backup Type : incremental + Backup Size : 22.3 MiB (54.3 MiB with WALs) + WAL Size : 32.0 MiB + Resources saved : 19.5 MiB (86.80%) Timeline : 1 Begin WAL : 0000000100000CFD000000AD End WAL : 0000000100000D0D00000008 @@ -20,6 +29,8 @@ Backup 20150828T130001: WAL compression ratio: 79.51% Begin time : 2015-08-28 13:00:01.633925+00:00 End time : 2015-08-29 10:27:06.522846+00:00 + Copy time : 1 second + Estimated throughput : 2.0 MiB/s Begin Offset : 1575048 End Offset : 13853016 Begin XLOG : CFD/AD180888 @@ -36,4 +47,39 @@ Backup 20150828T130001: Retention Policy : not enforced Previous Backup : 20150821T130001 Next Backup : - (this is the latest base backup) + Root Backup : 20240814T015504 + Parent Backup : 20240814T016504 + Backup chain size : 3 + Children Backup(s) : 20240814T018515 ``` + +> **NOTE:** +> Depending on the version of your Postgres Server and/or the type +> of the backup, the output of `barman show-backup` command may +> be different. For example, fields like "Root Backup", "Parent Backup", +> "Backup chain size", and "Children Backup(s)" only make sense when +> showing information about a block-level incremental backup taken +> with `backup_method = postgres` and using Postgres 17 or newer, +> thus those fields are omitted for other kind of backups or older versions +> of Postgres. +> +> Also note that `show-backup` relies on the backup metadata so if a backup +> was created with Barman version 3.10 or earlier, the backup will not +> contain the fields added in version 3.11 (which are those added after +> the introduction of "incremental" backups in PostgreSQL 17). +> +> These are the possible values for the field "Backup Type": +> +> * `rsync`: for a backup taken with `rsync`; +> * `full`: for a full backup taken with `pg_basebackup`; +> * `incremental`: for an incremental backup taken with `pg_basebackup`; +> * `snapshot`: for a snapshot-based backup taken in the cloud. +> +> Below you can find a list of fields that may be shown or omitted depending +> on the type of the backup: +> +> * `Resources saved`: available for "rsync" and "incremental" backups; +> * `Root Backup`, `Parent Backup`, `Backup chain size`: available for +> "incremental" backups only; +> * `Children Backup(s)`: available for "full" and "incremental" backups; +> * `Snapshot information`: available for "snapshot" backups only. diff --git a/doc/barman.1.d/70-backup-id-shortcuts.md b/doc/barman.1.d/70-backup-id-shortcuts.md index 5cddd2784..227a2295d 100644 --- a/doc/barman.1.d/70-backup-id-shortcuts.md +++ b/doc/barman.1.d/70-backup-id-shortcuts.md @@ -17,3 +17,10 @@ oldest last-failed : Latest failed backup, in chronological order. + +last-full +: Latest full-backup eligible for a block-level incremental backup using + the `--incremental` option. + +latest-full +: same as *last-full* \ No newline at end of file diff --git a/doc/barman.5 b/doc/barman.5 index 8088008e7..8bfdc7610 100644 --- a/doc/barman.5 +++ b/doc/barman.5 @@ -1,6 +1,6 @@ .\" Automatically generated by Pandoc 2.2.1 .\" -.TH "BARMAN" "5" "January 24, 2024" "Barman User manuals" "Version 3.10.0" +.TH "BARMAN" "5" "August 22, 2024" "Barman User manuals" "Version 3.11.1" .hy .SH NAME .PP @@ -121,6 +121,15 @@ The option is ignored if the backup method is not rsync. Scope: Global/Server/Model. .RE .TP +.B aws_await_snapshots_timeout +The length of time in seconds to wait for snapshots to be created in AWS +before timing out. +Positive integer, default 3600. +.RS +.PP +Scope: Global/Server/Model. +.RE +.TP .B aws_profile The name of the AWS profile to use when authenticating with AWS (e.g. INI section in AWS credentials file). @@ -509,6 +518,16 @@ Requires \f[C]archiver\f[] to be enabled. Scope: Server. .RE .TP +.B keepalive_interval +An interval, in seconds, at which a hearbeat query will be sent to the +server to keep the libpq connection alive during an Rsync backup. +Default is 60. +A value of 0 disables it. +.RS +.PP +Scope: Global/Server/Model. +.RE +.TP .B last_backup_maximum_age This option identifies a time frame that must contain the latest backup. If the latest backup is older than the time frame, barman check command @@ -549,6 +568,18 @@ Syntax is the same as last_backup_maximum_age (above). Scope: Global/Server/Model. .RE .TP +.B local_staging_path +A path to a location on the local host where incremental backups will be +combined during the recovery. +This location must have enough available space to temporarily hold the +new synthetic backup. +This option is \f[I]required\f[] when recovering from an incremental +backup and has no effect otherwise. +.RS +.PP +Scope: Global/Server/Model. +.RE +.TP .B lock_directory_cleanup enables automatic cleaning up of the \f[C]barman_lock_directory\f[] from unused lock files. diff --git a/doc/barman.5.d/00-header.md b/doc/barman.5.d/00-header.md index d8675fb2a..178b229e9 100644 --- a/doc/barman.5.d/00-header.md +++ b/doc/barman.5.d/00-header.md @@ -1,3 +1,3 @@ -% BARMAN(5) Barman User manuals | Version 3.10.0 +% BARMAN(5) Barman User manuals | Version 3.11.1 % EnterpriseDB -% January 24, 2024 +% August 22, 2024 diff --git a/doc/barman.5.d/50-aws_await_snapshots_timeout.md b/doc/barman.5.d/50-aws_await_snapshots_timeout.md new file mode 100644 index 000000000..81cb296d1 --- /dev/null +++ b/doc/barman.5.d/50-aws_await_snapshots_timeout.md @@ -0,0 +1,6 @@ +aws_await_snapshots_timeout +: The length of time in seconds to wait for snapshots to be created in AWS + before timing out. + Positive integer, default 3600. + + Scope: Global/Server/Model. diff --git a/doc/barman.5.d/50-keepalive-interval.md b/doc/barman.5.d/50-keepalive-interval.md new file mode 100644 index 000000000..182843440 --- /dev/null +++ b/doc/barman.5.d/50-keepalive-interval.md @@ -0,0 +1,6 @@ +keepalive_interval +: An interval, in seconds, at which a hearbeat query will be sent to the + server to keep the libpq connection alive during an Rsync backup. Default + is 60. A value of 0 disables it. + + Scope: Global/Server/Model. diff --git a/doc/barman.5.d/50-local_staging_path.md b/doc/barman.5.d/50-local_staging_path.md new file mode 100644 index 000000000..f17ba36b0 --- /dev/null +++ b/doc/barman.5.d/50-local_staging_path.md @@ -0,0 +1,8 @@ +local_staging_path +: A path to a location on the local host where incremental backups will + be combined during the recovery. This location must have enough + available space to temporarily hold the new synthetic backup. This + option is *required* when recovering from an incremental backup and + has no effect otherwise. + + Scope: Global/Server/Model. diff --git a/doc/manual/00-head.en.md b/doc/manual/00-head.en.md index 5fdf07d39..4ce3c06d7 100644 --- a/doc/manual/00-head.en.md +++ b/doc/manual/00-head.en.md @@ -1,6 +1,6 @@ % Barman Manual % EnterpriseDB UK Limited -% January 24, 2024 (3.10.0) +% August 22, 2024 (3.11.1) **Barman** (Backup and Recovery Manager) is an open-source administration tool for disaster recovery of PostgreSQL servers written in Python. It allows your organisation to perform remote backups of multiple servers in business critical environments to reduce risk and help DBAs during the recovery phase. diff --git a/doc/manual/10-design.en.md b/doc/manual/10-design.en.md index 685c3023e..07cd87ca2 100644 --- a/doc/manual/10-design.en.md +++ b/doc/manual/10-design.en.md @@ -64,9 +64,9 @@ Barman is able to take backups using either Rsync, which uses SSH as a transport Choosing one of these two methods is a decision you will need to make, however for general usage we recommend using streaming replication for all currently supported versions of PostgreSQL. > **IMPORTANT:** \newline -> Because Barman transparently makes use of `pg_basebackup`, features such as incremental backup, parallel backup, and deduplication are currently not available. In this case, bandwidth limitation has some restrictions - compared to the traditional method via `rsync`. +> Because Barman transparently makes use of `pg_basebackup`, features such as parallel backup are currently not available. In this case, bandwidth limitation has some restrictions - compared to the traditional method via `rsync`. -Backup using `rsync`/SSH is recommended in all cases where `pg_basebackup` limitations occur (for example, a very large database that can benefit from incremental backup and deduplication). +Backup using `rsync`/SSH is recommended in cases where `pg_basebackup` limitations pose an issue for you. The reason why we recommend streaming backup is that, based on our experience, it is easier to setup than the traditional one. Also, streaming backup allows you to backup a PostgreSQL server on Windows[^windows], and makes life easier when working with Docker. diff --git a/doc/manual/15-system_requirements.en.md b/doc/manual/15-system_requirements.en.md index 683274000..25aaff87f 100644 --- a/doc/manual/15-system_requirements.en.md +++ b/doc/manual/15-system_requirements.en.md @@ -5,7 +5,7 @@ - Linux/Unix - Python >= 3.6 - Python modules: - - argcomplete + - argcomplete (optional) - psycopg2 >= 2.4.2 - python-dateutil - setuptools diff --git a/doc/manual/21-preliminary_steps.en.md b/doc/manual/21-preliminary_steps.en.md index e64b39810..1bc92fe97 100644 --- a/doc/manual/21-preliminary_steps.en.md +++ b/doc/manual/21-preliminary_steps.en.md @@ -29,9 +29,8 @@ postgres@pg$ createuser -P barman ``` ``` sql -GRANT EXECUTE ON FUNCTION pg_start_backup(text, boolean, boolean) to barman; -GRANT EXECUTE ON FUNCTION pg_stop_backup() to barman; -GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean, boolean) to barman; +GRANT EXECUTE ON FUNCTION pg_backup_start(text, boolean) to barman; +GRANT EXECUTE ON FUNCTION pg_backup_stop(boolean) to barman; GRANT EXECUTE ON FUNCTION pg_switch_wal() to barman; GRANT EXECUTE ON FUNCTION pg_create_restore_point(text) to barman; @@ -39,19 +38,20 @@ GRANT pg_read_all_settings TO barman; GRANT pg_read_all_stats TO barman; ``` -In the PostgreSQL 15 beta and any subsequent PostgreSQL versions the functions -`pg_start_backup` and `pg_stop_backup` have been renamed and have different -signatures. You will therefore need to replace the first three lines in the +In the case of using PostgreSQL version 14 or a prior version, the functions +`pg_backup_start` and `pg_backup_stop` had different names and different +signatures. You will therefore need to replace the first two lines in the above block with: ``` sql -GRANT EXECUTE ON FUNCTION pg_backup_start(text, boolean) to barman; -GRANT EXECUTE ON FUNCTION pg_backup_stop(boolean) to barman; +GRANT EXECUTE ON FUNCTION pg_start_backup(text, boolean, boolean) to barman; +GRANT EXECUTE ON FUNCTION pg_stop_backup() to barman; +GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean, boolean) to barman; ``` It is worth noting that with PostgreSQL version 13 and below without a real superuser, the `--force` option of the `barman switch-wal` command will not work. -If you are running PostgreSQL version 14 or above, you can grant the `pg_checkpoint` +If you are running PostgreSQL version 15 or above, you can grant the `pg_checkpoint` role, so you can use this feature without a superuser: ``` sql diff --git a/doc/manual/25-streaming_backup.en.md b/doc/manual/25-streaming_backup.en.md index 9bd2ea4dc..23861750b 100644 --- a/doc/manual/25-streaming_backup.en.md +++ b/doc/manual/25-streaming_backup.en.md @@ -1,7 +1,9 @@ ## Streaming backup Barman can backup a PostgreSQL server using the streaming connection, -relying on `pg_basebackup`. +relying on `pg_basebackup`. Since version 3.11, Barman also supports block-level +incremental backups using the streaming connection, for more information +consult the _"Features in detail"_ section. > **IMPORTANT:** Barman requires that `pg_basebackup` is installed in > the same server. It is recommended to install the last available diff --git a/doc/manual/26-rsync_backup.en.md b/doc/manual/26-rsync_backup.en.md index f988a3a4c..acb0f1a7c 100644 --- a/doc/manual/26-rsync_backup.en.md +++ b/doc/manual/26-rsync_backup.en.md @@ -1,9 +1,10 @@ ## Backup with `rsync`/SSH -The backup over `rsync` was the only available method before 2.0, and -is currently the only backup method that supports the incremental -backup feature. Please consult the _"Features in detail"_ section for -more information. +The backup over `rsync` was the only method for backups in Barman before +version 2.0, and before 3.11 it was the only method that supported incremental +backups. Current Barman supports file-level as well as block-level incremental backups. +Backups using `rsync` implements the file-level backup feature. Please consult the +_"Features in detail"_ section for more information. To take a backup using `rsync` you need to put these parameters inside the Barman server configuration file: @@ -34,3 +35,11 @@ To take a backup use the `barman backup` command: barman@backup$ barman backup pg ``` +> **NOTE:** +> Starting with Barman 3.11.0, Barman uses a keep-alive mechanism when taking +> rsync-based backups. It keeps sending a simple `SELECT 1` query over the +> libpq connection where Barman runs `pg_backup_start`/`pg_backup_stop` +> low-level API functions, and it's in place to reduce the probability of a firewall or +> a router dropping that connection as it can be idle for a long time while the base +> backup is being copied. You can control the interval of the hearbeats, or even +> disable the mechanism, through the `keepalive_interval` configuration option. diff --git a/doc/manual/28-snapshots.en.md b/doc/manual/28-snapshots.en.md index 493334f96..326e8cb1e 100644 --- a/doc/manual/28-snapshots.en.md +++ b/doc/manual/28-snapshots.en.md @@ -157,6 +157,7 @@ The following optional parameters can be set when using AWS: ``` ini aws_region = AWS_REGION aws_profile = AWS_PROFILE_NAME +aws_await_snapshots_timeout = TIMEOUT_IN_SECONDS ``` If `aws_profile` is used it should be set to the name of a section in the AWS credentials file. @@ -165,6 +166,8 @@ If no credentials file exists then credentials will be sourced from the environm If `aws_region` is specified it will override any region that may be defined in the AWS profile. +If `aws_await_snapshots_timeout` is not set, the default of `3600` seconds will be used. + ### Taking a snapshot backup Once the configuration options are set and appropriate credentials are available to Barman, backups can be taken using the [barman backup](#backup) command. diff --git a/doc/manual/42-server-commands.en.md b/doc/manual/42-server-commands.en.md index fa9caabd2..a6406cb6d 100644 --- a/doc/manual/42-server-commands.en.md +++ b/doc/manual/42-server-commands.en.md @@ -39,6 +39,9 @@ barman backup > You can use `barman backup ` to sequentially > backup both `` and `` servers. +For information on how to take incremental backups in Barman, please check the +[incremental backup section]{#incremental-backup}. + Barman 2.10 introduces the `-w`/`--wait` option for the `backup` command. When set, Barman temporarily saves the state of the backup to `WAITING_FOR_WALS`, then waits for all the required WAL files to be @@ -280,7 +283,8 @@ barman list-backups > **TIP:** You can request a full list of the backups of all servers > using `all` as the server name. -To have a machine-readable output you can use the `--minimal` option. +To get a machine-readable output you can use the `--minimal` option, +and to get the output in JSON format you can use the `--format=json` option. ## `rebuild-xlogdb` diff --git a/doc/manual/43-backup-commands.en.md b/doc/manual/43-backup-commands.en.md index 80d5e64b5..fd31ada6c 100644 --- a/doc/manual/43-backup-commands.en.md +++ b/doc/manual/43-backup-commands.en.md @@ -67,6 +67,11 @@ barman delete The `delete` command accepts any [shortcut](#backup-id-shortcuts) to identify backups. +> **IMPORTANT:** +> If the specified backup has dependent block-level incremental backups, +> those backups and all their dependents will also be deleted during this operation +> as they would effectively become unusable for recovery with a missing parent in its chain. + ## `keep` If you have a backup which you wish to keep beyond the retention policy of @@ -76,6 +81,21 @@ the server then you can make it an archival backup with: barman keep [--target TARGET, --status, --release] ``` +> **NOTE:** +> To ensure the integrity of your backup system, block-level incremental backups +> cannot use the keep annotation in Barman. This restriction is due to +> the way block-level incremental backups depend on each other. Using the keep +> annotation on such backups could result in orphaned backups, +> which means that certain backups might exist without their necessary +> parent backups. +> +> In simpler terms, if you were allowed to apply the keep annotation to +> a block-level incremental backup, there would be a risk that parts of the backup +> chain would be retained without their required predecessors. This +> situation could create backups that would be no longer be useful or +> complete, as they would be missing the essential parent backups needed +> to restore them properly. + Possible values for `TARGET` are: * `full`: The backup can always be used to recover to the latest point in @@ -178,6 +198,16 @@ command. The recovery command has several options that modify the command behavior. +> **IMPORTANT:** +> If you're unsure whether the backup you want to recover from will be +> automatically deleted by retention policies during recovery, +> mark it issuing the [keep command](#keep) using the appropriate target. +> Once the recovery process is complete and your new PostgreSQL instance +> has reached the desired recovery target, if you don’t want to +> keep the backup beyond the retention policy, you can remove `keep` +> annotation issuing the [keep command](#keep) again using the +> `--release` option. + ### Remote recovery Add the `--remote-ssh-command ` option to the invocation @@ -251,6 +281,10 @@ the following mutually exclusive options: > the start and the end of a backup, you must recover from the > previous backup in the catalogue. +> **IMPORTANT:** +> If no timezone is specified when using `--target-time`, the timezone of the Barman +> host will be used. + You can use the `--exclusive` option to specify whether to stop immediately before or immediately after the recovery target. @@ -315,6 +349,17 @@ behaviour defined by `recovery_options`. Use `--get-wal` with `barman recover` to enable the fetching of WALs from the Barman server, alternatively use `--no-get-wal` to disable it. +> **IMPORTANT:** +> When recovering with `--no-get-wal` in conjunction with any of these +> targets [`--target-xid`, `--target-name`, `--target-time`], Barman +> will copy the whole WAL archive from the Barman host to the recovery host. +> By doing that, and assuming that all the WALs required for reaching +> the configured target were already archived into Barman, we guarantee +> that at least these WALs will be made available to Postgres. +> This happens because currently there is no reliable and/or performant +> way of determining in Barman which WALs are needed by Postgres to +> reach those kinds of recovery targets. + ### Recovering compressed backups If a backup has been compressed using the `backup_compression` option @@ -339,6 +384,37 @@ the `--recovery-staging-path` option with the `barman recover` command. If you do neither of these things and attempt to recover a compressed backup then Barman will fail rather than try to guess a suitable location. +### Recovering block-level incremental backups + +If a backup is a block-level incremental, `barman recover` is able to combine the chain +of backups on recovery through `pg_combinebackup`. +A chain of backups is the tree branch that goes from the full backup +to the one requested for the recovery. This is a multi-step process: + +1. The chain of backups is combined into a new synthetic backup. A + folder named with the ID of the incremental backup being recovered is + created inside a given staging directory on the local server using + `pg_combinebackup`. For any type of recover (local or remote), the + synthetic backup is created locally in the barman server. +2. If it's a remote recover, the content is copied to the final destination + using Rsync. Otherwise, when it's a local recover, the content is just + moved to the final destination. +3. The folder named with the ID of the incremental backup being recovered, which + was created inside the provided staging directory, is removed at the end of the + recovery process. + +When recovering from a block-level incremental backup, you *must* therefore +either set `local_staging_path` in the global/server config *or* use +the `--local-staging-path` option with the `barman recover` command. If +you do neither of these things and attempt to recover such backup +then Barman fails rather than trying to guess a suitable location. + +> **IMPORTANT:** +> If any of the backups in the chain were taken with checksums +> disabled, but the final backup was taken with checksums enabled, the +> resulting directory may contain pages with invalid checksums. +> [Follow up the `limitations` section in pg_basebackup documentation](https://www.postgresql.org/docs/17/app-pgcombinebackup.html). + ## `show-backup` You can retrieve all the available information for a particular backup of diff --git a/doc/manual/50-feature-details.en.md b/doc/manual/50-feature-details.en.md index 60f7063a0..2dc5b4d5c 100644 --- a/doc/manual/50-feature-details.en.md +++ b/doc/manual/50-feature-details.en.md @@ -13,24 +13,25 @@ common patterns. ### Incremental backup -Barman implements **file-level incremental backup**. Incremental -backup is a type of full periodic backup which only saves data changes -from the latest full backup available in the catalog for a specific -PostgreSQL server. It must not be confused with differential backup, +Incremental backup is a type of backup which uses an already existing backup as reference +for copying only necessary data changes from the PostgreSQL server. It must not be confused with differential backup, which is implemented by _WAL continuous archiving_. -> **NOTE:** Block level incremental backup will be available in -> future versions. - -> **IMPORTANT:** The `reuse_backup` option can't be used with the -> `postgres` backup method at this time. - The main goals of incremental backups in Barman are: - Reduce the time taken for the full backup process - Reduce the disk space occupied by several periodic backups (**data deduplication**) +Barman currently supports **file-level incremental backups** (using `rsync`) as well +as **block-level incremental backups** (using `pg_basebackup`). + +> **NOTE:** Incremental backups of different backup types are currently not compatible +> i.e. a block-level incremental backup can not be taken upon an `rsync` backup and +> a file-level incremental backup can not be taken upon a streaming backup (taken with `pg_basebackup`). + +### File-level incremental backups + This feature heavily relies on `rsync` and [hard links][8], which must therefore be supported by both the underlying operating system and the file system where the backup data resides. @@ -41,15 +42,15 @@ relevant savings in disk usage. This is particularly true of VLDB contexts and of those databases containing a high percentage of _read-only historical tables_. -Barman implements incremental backup through a global/server option +Rsync incremental backups can be enabled through a global/server option called `reuse_backup`, that transparently manages the `barman backup` command. It accepts three values: - `off`: standard full backup (default) -- `link`: incremental backup, by reusing the last backup for a server +- `link`: file-level incremental backup, by reusing the last backup for a server and creating a hard link of the unchanged files (for backup space and time reduction) -- `copy`: incremental backup, by reusing the last backup for a server +- `copy`: file-level incremental backup, by reusing the last backup for a server and creating a copy of the unchanged files (just for backup time reduction) @@ -73,6 +74,54 @@ incremental backup as follows: barman backup --reuse-backup=link ``` +> **NOTE:** Unlike Postgres block-level incremental backups, Rsync file-level incremental backups are independent +> on their own, meaning that a backup that reused a previous backup for deduplication is not compromised +> in any way if the parent backup is deleted. As mentioned, deduplication in Rsync backups is implemented with the use +> of hard links, so when a previously reused backup is deleted, files shared with other backups will +> still remain on disk, only being removed when the last backup using those files is also deleted. It also +> means that there’s no need to take full backups when a previous full backup is deleted, each backup +> will still have all files and can be reused without any concerns. +> Along the same lines, there is no need to ever take a full backup with rsync by using `reuse_backup = off`. If it is the first backup being taken with `reuse_backup = link`, in essence +> it behaves like `off` because there are no existing files to create hard-links on. + +> **IMPORTANT:** The `reuse_backup` option must be used along with +> `rsync` or `local-rsync` as backup method. + + +### Block-level incremental backups + +Since version 3.11, Barman introduces support for block-level incremental +backups, leveraging the native [incremental backup support introduced in PostgreSQL 17](https://www.postgresql.org/docs/17/continuous-archiving.html#BACKUP-INCREMENTAL-BACKUP). + +With block-level incremental backups, deduplication occurs at the data block level +(pages in PostgreSQL). This means that only those pages modified since the +last backup will need to be stored, making it a more efficient option, +especially for large databases with spread write patterns. In PostgreSQL, this feature is +implemented with the use of [WAL Summarization](https://www.postgresql.org/docs/17/runtime-config-wal.html#RUNTIME-CONFIG-WAL-SUMMARIZATION), therefore `summarize_wal` must be enabled +on your database server in order to use it. + +You can perform block-level incremental backups in Barman using the `--incremental` +option when running a backup command. It accepts a backup id or backup ID shortcut as argument, +which references a previous backup (full or incremental) in the catalog to be used as a parent +for deduplication. In addition, you can also use `last-full` or `latest-full` to reference the latest +eligible full-backup in the catalog. + +``` bash +barman backup --incremental +``` + +To be able to perform a block-level incremental backup in Barman you must: + +- Have PostgreSQL 17 or later. +- Have `summarize_wal` enabled. +- Have `postgres` as your backup method. + +> **NOTE:** Compressed backups are not **yet** eligible for block-level incremental backups in Barman. + +> **IMPORTANT:** If you decide to enable `data_checksums` between block-level incremental backups, +> it is adivised to take a new full-backup as divergent checkum configurations can potentially +> lead to issues during recovery. + ### Limiting bandwidth usage It is possible to limit the usage of I/O bandwidth through the @@ -539,6 +588,24 @@ Retention policy based on recovery window backups required to allow point-in-time recovery back to 9:30 AM on the previous Friday. +> **IMPORTANT:** +> Block-level incremental backups are not considered during the retention +> policy processing. This is because this kind of incremental backups +> depends on all of its parent backups, up to the full backup which generates +> the chain, in order to be recoverable. To maintain the consistency of backup +> chains, only full backups are taken into account when applying retention +> policies. +> +> How It Works +> +> When the retention policy is applied, Barman ignores block-level incremental backups +> and focuses only on the status of the full backups. +> +> If the full backup is marked as KEEP:FULL, KEEP:STANDALONE, or VALID, +> the status VALID is marked to all dependent block-level incremental backups. +> If the full backup is marked as OBSOLETE, then all block-level incremental backups +> that depend on it will also be marked as OBSOLETE and removed. + #### Scope Retention policies can be defined for: @@ -1208,6 +1275,9 @@ Backup 20230123T131430: PostgreSQL Version : 140006 PGDATA directory : /opt/postgres/data + Server information: + Checksums : on + Snapshot information: provider : gcp project : project_id diff --git a/doc/manual/66-about.en.md b/doc/manual/66-about.en.md index e62380d30..bc8ba2a23 100644 --- a/doc/manual/66-about.en.md +++ b/doc/manual/66-about.en.md @@ -51,15 +51,20 @@ You can use GitHub's pull requests system for this purpose. In alphabetical order: -* Abhijit Menon-Sen -* Didier Michel -* Michael Wallace +* Andre Marchesini +* Barbara Leidens +* Giulio Calacoci +* Gustavo Oliveira +* Israel Barth +* Martín Marqués Past contributors (in alphabetical order): +* Abhijit Menon-Sen (architect) * Anna Bellandi (QA/testing) * Britt Cole (documentation reviewer) * Carlo Ascani (developer) +* Didier Michel (developer) * Francesco Canovai (QA/testing) * Gabriele Bartolini (architect) * Gianni Ciolli (QA/testing) @@ -69,6 +74,7 @@ Past contributors (in alphabetical order): * Jonathan Battiato (QA/testing) * Leonardo Cecchi (developer) * Marco Nenciarini (project leader) +* Michael Wallace (developer) * Niccolò Fei (QA/testing) * Rubens Souza (QA/testing) * Stefano Bianucci (developer) diff --git a/setup.py b/setup.py index d966f878c..0869c3d83 100755 --- a/setup.py +++ b/setup.py @@ -45,7 +45,6 @@ install_requires = [ "psycopg2 >= 2.4.2", "python-dateutil", - "argcomplete", ] barman = {} @@ -100,12 +99,12 @@ long_description="\n".join(__doc__.split("\n")[2:]), install_requires=install_requires, extras_require={ - "cloud": ["boto3"], + "argcomplete": ["argcomplete"], "aws-snapshots": ["boto3"], "azure": ["azure-identity", "azure-storage-blob"], "azure-snapshots": ["azure-identity", "azure-mgmt-compute"], - "snappy": ["python-snappy"], "zstd": ["zstandard"], + "cloud": ["boto3"], "google": [ "google-cloud-storage", ], @@ -113,6 +112,11 @@ "grpcio", "google-cloud-compute", # requires minimum python3.7 ], + "snappy": [ + 'python-snappy==0.6.1; python_version<"3.7"', + 'python-snappy; python_version>="3.7"', + 'cramjam >= 2.7.0; python_version>="3.7"', + ], }, platforms=["Linux", "Mac OS X"], classifiers=[ diff --git a/sonar-project.properties b/sonar-project.properties deleted file mode 100644 index d15c283c7..000000000 --- a/sonar-project.properties +++ /dev/null @@ -1,4 +0,0 @@ -sonar.python.coverage.reportPaths=coverage-reports/coverage.xml -sonar.python.xunit.reportPath=coverage-reports/results.xml -#sonar.python.xunit.skipDetails=true -sonar.coverage.exclusions=**/tests/**/*.py,**/sphinx/**/*.py,setup.py diff --git a/sonarqube/configure-env.sh b/sonarqube/configure-env.sh deleted file mode 100755 index 551b289de..000000000 --- a/sonarqube/configure-env.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - - - ########################## - # Setup build environment - ########################## - SetupEnv(){ - echo "current place :" pwd - sudo apt-get -y install python3-pip libpq-dev python3-dev - sudo python3 -m pip install --upgrade pip setuptools wheel - - echo "Install dependencies" - sudo python3 -m pip install -r tests/requirements_dev.txt - sudo python3 -m pip install pytest-cov - } - - - ################## - # Add build steps - ################## - GenerateReports(){ - echo "Create Coverage report" - python3 -m py.test --cov barman --cov-report xml:coverage-reports/coverage.xml --junitxml=coverage-reports/results.xml - } - - - ######## - # Main - ######## - SetupEnv - GenerateReports diff --git a/tests/test_backup.py b/tests/test_backup.py index 3454573fc..541b5f872 100644 --- a/tests/test_backup.py +++ b/tests/test_backup.py @@ -17,6 +17,7 @@ # along with Barman. If not, see . import errno +import itertools import os import re from datetime import datetime, timedelta @@ -25,7 +26,7 @@ import dateutil.tz import mock import pytest -from mock import Mock, patch +from mock import Mock, patch, call from barman.backup import BackupManager from barman.lockfile import ServerBackupIdLock @@ -33,6 +34,7 @@ from barman.annotations import KeepManager from barman.config import BackupOptions from barman.exceptions import ( + BackupException, CompressionIncompatibility, RecoveryInvalidTargetException, CommandFailedException, @@ -246,22 +248,7 @@ def test_delete_backup(self, mock_available_backups, tmpdir, caplog): "fake_backup_id": b_info, } - # Test 1: minimum redundancy not satisfied - caplog_reset(caplog) - backup_manager.server.config.minimum_redundancy = 2 - b_info.set_attribute("backup_version", 1) - build_backup_directories(b_info) - backup_manager.delete_backup(b_info) - assert re.search("WARNING .* Skipping delete of backup ", caplog.text) - assert "ERROR" not in caplog.text - assert os.path.exists(pg_data.strpath) - assert not os.path.exists(pg_data_v2.strpath) - assert os.path.exists(wal_file.strpath) - assert os.path.exists(wal_history_file02.strpath) - assert os.path.exists(wal_history_file03.strpath) - assert os.path.exists(wal_history_file04.strpath) - - # Test 2: normal delete expecting no errors (old format) + # Test 1: normal delete expecting no errors (old format) caplog_reset(caplog) backup_manager.server.config.minimum_redundancy = 1 b_info.set_attribute("backup_version", 1) @@ -277,7 +264,7 @@ def test_delete_backup(self, mock_available_backups, tmpdir, caplog): assert os.path.exists(wal_history_file03.strpath) assert os.path.exists(wal_history_file04.strpath) - # Test 3: delete the backup again, expect a failure in log + # Test 2: delete the backup again, expect a failure in log caplog_reset(caplog) backup_manager.delete_backup(b_info) assert re.search("ERROR .* Failure deleting backup fake_backup_id", caplog.text) @@ -288,7 +275,7 @@ def test_delete_backup(self, mock_available_backups, tmpdir, caplog): assert os.path.exists(wal_history_file03.strpath) assert os.path.exists(wal_history_file04.strpath) - # Test 4: normal delete expecting no errors (new format) + # Test 3: normal delete expecting no errors (new format) caplog_reset(caplog) b_info.set_attribute("backup_version", 2) build_backup_directories(b_info) @@ -302,7 +289,7 @@ def test_delete_backup(self, mock_available_backups, tmpdir, caplog): assert os.path.exists(wal_history_file03.strpath) assert os.path.exists(wal_history_file04.strpath) - # Test 5: normal delete of first backup no errors and no skip + # Test 4: normal delete of first backup no errors and no skip # removing one of the two backups present (new format) # and all the previous wal caplog_reset(caplog) @@ -318,7 +305,7 @@ def test_delete_backup(self, mock_available_backups, tmpdir, caplog): assert os.path.exists(wal_history_file03.strpath) assert os.path.exists(wal_history_file04.strpath) - # Test 6: normal delete of first backup no errors and no skip + # Test 5: normal delete of first backup no errors and no skip # removing one of the two backups present (new format) # the previous wal is retained as on a different timeline caplog_reset(caplog) @@ -336,7 +323,7 @@ def test_delete_backup(self, mock_available_backups, tmpdir, caplog): assert os.path.exists(wal_history_file03.strpath) assert os.path.exists(wal_history_file04.strpath) - # Test 7: simulate an error deleting the backup. + # Test 6: simulate an error deleting the backup. with patch( "barman.backup.BackupManager.delete_backup_data" ) as mock_delete_data: @@ -353,26 +340,165 @@ def test_delete_backup(self, mock_available_backups, tmpdir, caplog): assert os.path.exists(wal_history_file03.strpath) assert os.path.exists(wal_history_file04.strpath) - @patch("barman.backup.BackupManager.should_keep_backup") - def test_cannot_delete_keep_backup(self, mock_should_keep_backup, caplog): - """Verify that we cannot delete backups directly if they have a keep""" - # Setup of the test backup_manager - backup_manager = build_backup_manager() - backup_manager.server.config.name = "TestServer" - backup_manager.server.config.backup_options = [] + # Test 7: ensure a child backup has its referenced removed from + # the parent when removed successfully + parent_backup = build_test_backup_info( + backup_id="parent_backup_id", + server=backup_manager.server, + ) + build_backup_directories(parent_backup) + child_backup = build_test_backup_info( + backup_id="child_backup_id", + server=backup_manager.server, + parent_backup_id=parent_backup.backup_id, + ) + build_backup_directories(child_backup) + parent_backup.set_attribute( + "children_backup_ids", [child_backup.backup_id, "another_backup_id"] + ) + mock_available_backups.return_value = { + parent_backup.backup_id: parent_backup, + child_backup.backup_id: child_backup, + } + with patch("barman.infofile.LocalBackupInfo.get_parent_backup_info") as mock: + mock.return_value = parent_backup + deleted = backup_manager.delete_backup(child_backup) - mock_should_keep_backup.return_value = True + assert deleted is True + assert child_backup.backup_id not in parent_backup.children_backup_ids - b_info = build_test_backup_info( - backup_id="fake_backup_id", + # Test 8: Update next rsync backup information + given_backup = build_test_backup_info( + backup_id="rsync_backup_id", server=backup_manager.server, ) - assert backup_manager.delete_backup(b_info) is False - assert ( - "Skipping delete of backup fake_backup_id for server TestServer as it " - "has a current keep request. If you really want to delete this backup " - "please remove the keep and try again." in caplog.text + build_backup_directories(given_backup) + next_backup = build_test_backup_info( + backup_id="next_rsync_backup_id", + server=backup_manager.server, ) + build_backup_directories(next_backup) + mock_available_backups.return_value = { + given_backup.backup_id: given_backup, + next_backup.backup_id: next_backup, + } + with patch("barman.backup.BackupManager.get_next_backup") as get_next_backup: + with patch( + "barman.backup.BackupManager._set_backup_sizes" + ) as set_backup_sizes: + get_next_backup.return_value = next_backup + deleted = backup_manager.delete_backup(given_backup) + assert deleted is True + set_backup_sizes.assert_called_once_with(next_backup) + + @patch("os.stat") + @patch("barman.backup.fsync_file") + @patch("barman.backup.fsync_dir") + @patch("os.walk") + @pytest.mark.parametrize("fsync", [True, False]) + def test_set_backup_sizes( + self, + mock_walk, + mock_fsync_dir, + mock_fsync_file, + mock_stat, + fsync, + ): + """ + Test that the _set_backup_sizes method correctly sets the backup sizes + and optionally performs fsync. + """ + # Set up the mocks + backup_manager = build_backup_manager() + mock_stat.reset_mock() + mock_backup_info = Mock() + + # Mock os.walk to return a predefined directory structure + mock_walk.return_value = [ + ("/root", ["dir1", "dir2"], ["file1.txt"]), + ("/root/dir1", [], ["file2.txt"]), + ("/root/dir2", ["subdir"], []), + ("/root/dir2/subdir", [], ["file3.txt"]), + ] + + # Define the mock return values for os.stat + def mock_stat_return_value(backup): + return_values = { + "/root/file1.txt": { + "size": 1024, + "nlink": 3, + }, + "/root/dir1/file2.txt": { + "size": 2048, + "nlink": 2, + }, + "/root/dir2/subdir/file3.txt": { + "size": 4096, + "nlink": 1, + }, + } + return Mock( + st_size=return_values[backup]["size"], + st_nlink=return_values[backup]["nlink"], + ) + + mock_stat.side_effect = mock_stat_return_value + + # Define the mock return values for fsync_file + def mock_fsync_file_return_value(file_path): + return mock_stat_return_value(file_path) + + mock_fsync_file.side_effect = mock_fsync_file_return_value + + # Call the method under test + backup_manager._set_backup_sizes(mock_backup_info, fsync) + + # Assertions for both with and without fsync cases + mock_walk.assert_called_once_with( + mock_backup_info.get_basebackup_directory.return_value, + ) + assert mock_backup_info.set_attribute.call_count == 2 + mock_backup_info.set_attribute.assert_has_calls( + [ + call("size", 7168), + call("deduplicated_size", 4096), + ] + ) + mock_backup_info.save.assert_called_once() + + # Assertions when called with fsync + if fsync: + mock_stat.assert_not_called() + assert mock_fsync_dir.call_count == 4 + mock_fsync_dir.assert_has_calls( + [ + call("/root"), + call("/root/dir1"), + call("/root/dir2"), + call("/root/dir2/subdir"), + ] + ) + assert mock_fsync_file.call_count == 3 + mock_fsync_file.assert_has_calls( + [ + call("/root/file1.txt"), + call("/root/dir1/file2.txt"), + call("/root/dir2/subdir/file3.txt"), + ] + ) + + # Assertions without fsync (standard case) + else: + mock_fsync_dir.assert_not_called() + mock_fsync_file.assert_not_called() + assert mock_stat.call_count == 3 + mock_stat.assert_has_calls( + [ + call("/root/file1.txt"), + call("/root/dir1/file2.txt"), + call("/root/dir2/subdir/file3.txt"), + ] + ) def test_available_backups(self, tmpdir): """ @@ -824,6 +950,275 @@ def test_backup_without_name( # THEN backup name is None in the backup_info assert backup_info.backup_name is None + @patch("barman.backup.LocalBackupInfo.save") + @patch("barman.backup.output") + def test_backup_without_parent_backup_id( + self, + _mock_output, + _mock_backup_info_save, + ): + """ + Verify that information about parent and children are not updated when no parent + backup ID is specified. + """ + # GIVEN a backup manager + backup_manager = build_backup_manager() + backup_manager.executor.backup = Mock() + backup_manager.executor.copy_start_time = datetime.now() + + # WHEN a backup is taken with no parent backup ID + backup_info = backup_manager.backup() + + # THEN parent backup ID is None in the backup_info + assert backup_info.parent_backup_id is None + + @patch("barman.backup.LocalBackupInfo.save") + @patch("barman.backup.output") + def test_backup_with_parent_backup_id( + self, + _mock_output, + _mock_backup_info_save, + ): + """ + Verify that information about parent and children are updated when a parent + backup ID is specified. + """ + # GIVEN a backup manager + backup_manager = build_backup_manager() + backup_manager.executor.backup = Mock() + backup_manager.executor.copy_start_time = datetime.now() + + # WHEN a backup is taken with a parent backup ID which contains no children + with patch("barman.infofile.LocalBackupInfo.get_parent_backup_info") as mock: + mock.return_value.children_backup_ids = None + backup_info = backup_manager.backup( + parent_backup_id="SOME_PARENT_BACKUP_ID", + ) + + # THEN parent backup ID is filled in the backup_info + assert backup_info.parent_backup_id == "SOME_PARENT_BACKUP_ID" + + # AND children backup IDs is set in the parent backup_info + assert mock.return_value.children_backup_ids == [backup_info.backup_id] + + # WHEN a backup is taken with a parent backup ID which contains a child + with patch("barman.infofile.LocalBackupInfo.get_parent_backup_info") as mock: + mock.return_value.children_backup_ids = ["SOME_CHILD_BACKUP_ID"] + backup_info = backup_manager.backup( + parent_backup_id="SOME_PARENT_BACKUP_ID", + ) + + # THEN parent backup ID is filled in the backup_info + assert backup_info.parent_backup_id == "SOME_PARENT_BACKUP_ID" + + # AND children backup IDs is set in the parent backup_info + assert mock.return_value.children_backup_ids == [ + "SOME_CHILD_BACKUP_ID", + backup_info.backup_id, + ] + + @patch("barman.backup.BackupManager._validate_incremental_backup_configs") + def test_validate_backup_args(self, mock_validate_incremental): + """ + Test the validate_backup_args method, ensuring that validations are passed + correctly to all responsible methods according to the parameters received. + """ + backup_manager = build_backup_manager(global_conf={"backup_method": "postgres"}) + + # incremental backup validation is skipped when no parent backup is present + incremental_kwargs = {} + backup_manager.validate_backup_args(**incremental_kwargs) + mock_validate_incremental.assert_not_called() + + # incremental backup validation is called when a parent backup is present + mock_validate_incremental.reset_mock() + incremental_kwargs = {"parent_backup_id": Mock()} + backup_manager.validate_backup_args(**incremental_kwargs) + mock_validate_incremental.assert_called_once() + + def test_validate_incremental_backup_configs_pg_version(self): + """ + Test Postgres incremental backup validations for Postgres + server version + """ + backup_manager = build_backup_manager(global_conf={"backup_method": "postgres"}) + + # mock the postgres object to set server version + mock_postgres = Mock() + backup_manager.executor.server.postgres = mock_postgres + + # mock enabled summarize_wal option + backup_manager.executor.server.postgres.get_setting.side_effect = ["on"] + + # ensure no exception is raised when Postgres version >= 17 + mock_postgres.configure_mock(server_version=180500) + backup_manager._validate_incremental_backup_configs() + + # ensure BackupException is raised when Postgres version is < 17 + mock_postgres.configure_mock(server_version=160000) + with pytest.raises(BackupException): + backup_manager._validate_incremental_backup_configs() + + def test_validate_incremental_backup_configs_summarize_wal(self): + """ + Test that summarize_wal is enabled on Postgres incremental backup + """ + backup_manager = build_backup_manager(global_conf={"backup_method": "postgres"}) + + # mock the postgres object to set server version + mock_postgres = Mock() + backup_manager.executor.server.postgres = mock_postgres + mock_postgres.configure_mock(server_version=170000) + + # change the behavior of get_setting("summarize_wal") function call + backup_manager.executor.server.postgres.get_setting.side_effect = [ + None, + "off", + "on", + ] + + err_msg = "'summarize_wal' option has to be enabled in the Postgres server " + "to perform an incremental backup using the Postgres backup method" + + # ensure incremental backup with summarize_wal disabled raises exception + with pytest.raises(BackupException, match=err_msg): + backup_manager._validate_incremental_backup_configs() + with pytest.raises(BackupException, match=err_msg): + backup_manager._validate_incremental_backup_configs() + # no exception is raised when summarize_wal is on + backup_manager._validate_incremental_backup_configs() + + @pytest.mark.parametrize( + ("parent_backup_compression", "backup_compression"), + list(itertools.product(("gzip", None), ("gzip", None))), + ) + @patch("barman.backup.BackupManager.get_backup") + def test_validate_incremental_backup_configs_backup_compression( + self, + mock_get_backup, + parent_backup_compression, + backup_compression, + ): + """ + Test the behaviour of backups taken with backup_compression set + for incremental backups and/or parent backups. + """ + # set backup_compression option in global config + backup_manager = build_backup_manager( + global_conf={ + "backup_method": "postgres", + "backup_compression": backup_compression, + } + ) + + # mock the postgres object to set server version + mock_postgres = Mock() + backup_manager.executor.server.postgres = mock_postgres + mock_postgres.configure_mock(server_version=170000) + + # mock enabled summarize_wal option + backup_manager.executor.server.postgres.get_setting.side_effect = ["on"] + err_msg = "" + + mock_get_backup.return_value = build_test_backup_info( + compression=parent_backup_compression, summarize_wal="on" + ) + # ensure incremental backup with backup_compression set raises exception + if backup_compression: + err_msg = "Incremental backups cannot be taken with " + "'backup_compression' set in the configuration options." + with pytest.raises(BackupException, match=err_msg): + backup_manager._validate_incremental_backup_configs() + elif parent_backup_compression: + err_msg = ( + "The specified backup cannot be a parent for an " + "incremental backup. Reason: " + "Compressed backups are not eligible as parents of incremental backups." + ) + with pytest.raises(BackupException, match=err_msg): + backup_manager._validate_incremental_backup_configs() + else: + # no exception is raised when backup_compression is None + backup_manager._validate_incremental_backup_configs() + + @patch("barman.backup.BackupManager.get_available_backups") + def test_get_last_full_backup_id(self, get_available_backups): + """ + Test that the function returns the correct last full backup + """ + backup_manager = build_backup_manager(global_conf={"backup_method": "postgres"}) + + available_backups = { + "20241010T120000": "20241009T131000", + "20241010T131000": None, + "20241010T140202": "20241010T131000", + "20241010T150000": "20241010T140202", + "20241010T160000": None, + "20241010T180000": "20241010T160000", + "20241011T180000": None, + "20241012T180000": "20241011T180000", + "20241013T180000": "20241012T180000", + } + + backups = dict( + ( + bkp_id, + build_test_backup_info( + server=backup_manager.server, + backup_id=bkp_id, + parent_backup_id=par_bkp_id, + summarize_wal="on", + ), + ) + for bkp_id, par_bkp_id in available_backups.items() + ) + get_available_backups.return_value = backups + + last_full_backup = backup_manager.get_last_full_backup_id() + get_available_backups.assert_called_once() + assert last_full_backup == "20241011T180000" + + @patch("barman.backup._logger") + @patch("barman.backup.output") + @patch("barman.backup.BackupManager._set_backup_sizes") + def test_backup_fsync_and_set_sizes( + self, + mock_set_backup_sizes, + mock_output, + mock_logger, + ): + """ + Test the function for correct backup size and deduplication ratio + setting and logging. + """ + backup_manager = build_backup_manager() + backup_manager.executor.current_action = "calculating backup size" + backup_info = Mock() + backup_info.size = 0 + + # Test case with no deduplication ratio output + backup_manager.backup_fsync_and_set_sizes(backup_info) + mock_logger.debug.assert_called_once_with("calculating backup size") + mock_set_backup_sizes.assert_called_with(backup_info, fsync=True) + mock_output.info.assert_called_with("Backup size: %s" % "0 B") + + # Reset mocks + mock_logger.reset_mock() + mock_set_backup_sizes.reset_mock() + mock_output.reset_mock() + + # Test case when reuse_backup == "link" + backup_manager.config.reuse_backup = "link" + backup_info.size = 1000 + backup_info.deduplicated_size = 800 + backup_manager.backup_fsync_and_set_sizes(backup_info) + mock_logger.debug.assert_called_once_with("calculating backup size") + mock_set_backup_sizes.assert_called_with(backup_info, fsync=True) + mock_output.info.assert_called_once_with( + "Backup size: %s. Actual size on disk: %s (-%s deduplication ratio)." + % ("1000 B", "800 B", "20.00%") + ) + class TestWalCleanup(object): """Test cleanup of WALs by BackupManager""" diff --git a/tests/test_barman_cloud_backup_delete.py b/tests/test_barman_cloud_backup_delete.py index 656bfeb34..d9bc9400f 100644 --- a/tests/test_barman_cloud_backup_delete.py +++ b/tests/test_barman_cloud_backup_delete.py @@ -103,12 +103,57 @@ def _get_mock_backup_files(self, file_paths): for name, path in file_paths.items() ) + def _create_backup_info( + self, + backup_id, + begin_wal=None, + end_wal=None, + backup_name=None, + parent_backup_id=None, + children_backup_ids=None, + is_incremental=False, + has_children=False, + ): + """ + Helper for tests which creates mock BackupInfo objects. + + If *begin_wal* is set then the backup's begin_wal and + timeline values will be set according to the wal name. + + This is used by tests for two purposes: + 1. Helper to create a backup_metadata and mock CloudBackupCatalog. + 2. Providing the data needed to create backup_metadata. + """ + backup_info = mock.MagicMock(name="backup_info", snapshots_info=None) + backup_info.backup_id = backup_id + if backup_name: + backup_info.backup_name = backup_name + backup_info.status = "DONE" + backup_info.end_time = datetime.datetime.strptime( + backup_id, "%Y%m%dT%H%M%S" + ) + datetime.timedelta(hours=1) + + backup_info.begin_wal = begin_wal + backup_info.timeline = None + if backup_info.begin_wal: + backup_info.timeline = int(backup_info.begin_wal[:8]) + backup_info.end_wal = end_wal + backup_info.is_incremental = is_incremental + backup_info.has_children = has_children + backup_info.parent_backup_id = parent_backup_id + backup_info.children_backups_ids = children_backup_ids + return backup_info + def _create_backup_metadata( - self, backup_ids, begin_wals=None, end_wals=None, is_snapshot_backup=False + self, + backup_ids, + begin_wals=None, + end_wals=None, + is_snapshot_backup=False, ): """ - Helper for tests which creates mock BackupFileInfo and BackupInfo objects - which are returned in a dict keyed by backup_id. + Helper for tests which creates mock BackupFileInfo which are + returned in a dict keyed by backup_id. If begin_wals has an entry for a given backup_id then the begin_wal and timeline values will be set according to the wal name in begin_wals. @@ -121,6 +166,7 @@ def _create_backup_metadata( begin_wals = {} if end_wals is None: end_wals = {} + backup_metadata = {} for backup in backup_ids: backup_name = None @@ -141,23 +187,12 @@ def _create_backup_metadata( } ) backup_metadata[backup_id]["files"] = mock_backup_files - backup_info = mock.MagicMock(name="backup_info", snapshots_info=None) - backup_info.backup_id = backup_id - if backup_name: - backup_info.backup_name = backup_name - backup_info.status = "DONE" - backup_info.end_time = datetime.datetime.strptime( - backup_id, "%Y%m%dT%H%M%S" - ) + datetime.timedelta(hours=1) - try: - backup_info.begin_wal = begin_wals[backup_id] - backup_info.timeline = int(backup_info.begin_wal[:8]) - except KeyError: - pass - try: - backup_info.end_wal = end_wals[backup_id] - except KeyError: - pass + backup_info = self._create_backup_info( + backup_id, + begin_wals.get(backup_id), + end_wals.get(backup_id), + backup_name, + ) backup_metadata[backup_id]["info"] = backup_info return backup_metadata diff --git a/tests/test_barman_cloud_backup_show.py b/tests/test_barman_cloud_backup_show.py index 81e61281a..51b547866 100644 --- a/tests/test_barman_cloud_backup_show.py +++ b/tests/test_barman_cloud_backup_show.py @@ -43,7 +43,9 @@ def cloud_backup_catalog(self, backup_id): begin_wal="000000010000000000000002", end_time=datetime.datetime(2038, 1, 19, 4, 14, 8), end_wal="000000010000000000000004", - size=None, + size=2048, + data_checksums="on", + summarize_wal="on", snapshots_info=GcpSnapshotsInfo( project="test_project", snapshots=[ @@ -64,6 +66,8 @@ def cloud_backup_catalog(self, backup_id): ], ), version=150000, + cluster_size=2048, + deduplicated_size=1024, ) backup_info.mode = "concurrent" cloud_backup_catalog = mock.Mock() @@ -96,6 +100,11 @@ def test_cloud_backup_show( " Status : DONE\n" " PostgreSQL Version : 150000\n" " PGDATA directory : /pgdata/location\n" + " Estimated Cluster Size : 2.0 KiB\n" + "\n" + " Server information:\n" + " Checksums : on\n" + " WAL summarizer : on\n" "\n" " Snapshot information:\n" " provider : gcp\n" @@ -118,6 +127,8 @@ def test_cloud_backup_show( " tbs2 : /another/location (oid: 16405)\n" "\n" " Base backup information:\n" + " Backup Method : concurrent\n" + " Backup Size : 1.0 KiB\n" " Timeline : 1\n" " Begin WAL : 000000010000000000000002\n" " End WAL : 000000010000000000000004\n" @@ -157,10 +168,13 @@ def test_cloud_backup_show_json( "begin_time": "Tue Jan 19 03:14:08 2038", "begin_wal": "000000010000000000000002", "begin_xlog": "0/2000028", + "children_backup_ids": None, + "cluster_size": 2048, "compression": None, "config_file": "/pgdata/location/postgresql.conf", "copy_stats": None, - "deduplicated_size": None, + "data_checksums": "on", + "deduplicated_size": 1024, "end_offset": 184, "end_time": "Tue Jan 19 04:14:08 2038", "end_wal": "000000010000000000000004", @@ -170,9 +184,10 @@ def test_cloud_backup_show_json( "ident_file": "/pgdata/location/pg_ident.conf", "included_files": None, "mode": "concurrent", + "parent_backup_id": None, "pgdata": "/pgdata/location", "server_name": "main", - "size": None, + "size": 2048, "snapshots_info": { "provider": "gcp", "provider_info": { @@ -204,6 +219,7 @@ def test_cloud_backup_show_json( ], }, "status": "DONE", + "summarize_wal": "on", "systemid": None, "tablespaces": [ ["tbs1", 16387, "/fake/location"], diff --git a/tests/test_barman_cloud_wal_restore.py b/tests/test_barman_cloud_wal_restore.py index 521a357cd..6d0b35cf1 100644 --- a/tests/test_barman_cloud_wal_restore.py +++ b/tests/test_barman_cloud_wal_restore.py @@ -65,6 +65,52 @@ def test_succeeds_if_wal_is_found(self, get_cloud_interface_mock, caplog): assert caplog.text == "" cloud_interface_mock.download_file.assert_called_once() + @mock.patch("barman.clients.cloud_walrestore.get_cloud_interface") + def test_succeeds_if_wal_is_found_partial(self, get_cloud_interface_mock, caplog): + """If the WAL is found as partial we exit with status 0.""" + cloud_interface_mock = get_cloud_interface_mock.return_value + cloud_interface_mock.path = "testfolder/" + cloud_interface_mock.list_bucket.return_value = [ + "testfolder/test-server/wals/000000080000ABFF/000000080000ABFF000000C1.partial" + ] + cloud_walrestore.main( + [ + "s3://test-bucket/testfolder/", + "test-server", + "000000080000ABFF000000C1", + "/tmp/000000080000ABFF000000C1", + ] + ) + assert caplog.text == "" + cloud_interface_mock.download_file.assert_called_once() + + @mock.patch("barman.clients.cloud_walrestore.get_cloud_interface") + def test_fails_if_wal_is_found_partial_but_nopartial( + self, get_cloud_interface_mock, caplog + ): + """If the WAL is found as partial we exit with status 0.""" + cloud_interface_mock = get_cloud_interface_mock.return_value + cloud_interface_mock.path = "testfolder/" + cloud_interface_mock.list_bucket.return_value = [ + "testfolder/test-server/wals/000000080000ABFF/000000080000ABFF000000C1.partial" + ] + caplog.set_level(logging.INFO) + with pytest.raises(SystemExit) as exc: + cloud_walrestore.main( + [ + "--no-partial", + "s3://test-bucket/testfolder/", + "test-server", + "000000080000ABFF000000C1", + "/tmp/000000080000ABFF000000C1", + ] + ) + assert exc.value.code == 1 + assert ( + "WAL file 000000080000ABFF000000C1 for server test-server does not exists\n" + in caplog.text + ) + @mock.patch("barman.clients.cloud_walrestore.get_cloud_interface") def test_fails_if_wal_not_found(self, get_cloud_interface_mock, caplog): """If the WAL cannot be found we exit with status 1.""" diff --git a/tests/test_cli.py b/tests/test_cli.py index ca6b4698a..a11d6ef95 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -55,7 +55,9 @@ from testing_helpers import ( build_config_dictionary, build_config_from_dicts, + build_mocked_server, build_real_server, + build_test_backup_info, ) @@ -537,6 +539,7 @@ def mock_backup_info(self): backup_info.status = BackupInfo.DONE backup_info.tablespaces = [] backup_info.compression = None + backup_info.parent_backup_id = None return backup_info @pytest.fixture @@ -552,6 +555,8 @@ def mock_recover_args(self): args.target_time = None args.target_xid = None args.target_lsn = None + args.recovery_staging_path = None + args.local_staging_path = None return args @patch("barman.cli.parse_backup_id") @@ -713,6 +718,7 @@ def test_recover_get_wal( ): # GIVEN a backup parse_backup_id_mock.return_value = mock_backup_info + mock_backup_info.is_incremental = False # AND a configuration with the specified recovery options config = build_config_from_dicts( global_conf={"recovery_options": recovery_options} @@ -754,25 +760,42 @@ def test_recover_get_wal( "recovery_staging_path_config", "expected_recovery_staging_path", "should_error", + "error_substring", ), [ # If a backup is not compressed then recovery_staging_path is ignored - (False, None, None, None, False), + (False, None, None, None, False, None), # If a backup is compressed and no recovery_staging_path is provided # we expect an error - (True, None, None, None, True), + ( + True, + None, + None, + None, + True, + "backup is compressed with gzip compression but no recovery staging " + "path is provided.", + ), # If a backup is compressed and an argument is provided then it should # be set in the config - (True, "/from/arg", None, "/from/arg", False), + (True, "/from/arg", None, "/from/arg", False, None), # If a backup is compressed and a bad argument is provided then it should # error - (True, "from/arg", None, None, True), + ( + True, + "from/arg", + None, + None, + True, + "Cannot parse recovery staging path: Invalid value : 'from/arg' (must " + "be an absolute path)", + ), # If a backup is compressed and a config value is set then it should # be set in the config - (True, None, "/from/conf", "/from/conf", False), + (True, None, "/from/conf", "/from/conf", False, None), # If a backup is compressed and both arg and config are set then arg # takes precedence - (True, "/from/arg", "/from/conf", "/from/arg", False), + (True, "/from/arg", "/from/conf", "/from/arg", False, None), ], ) @patch("barman.cli.parse_backup_id") @@ -788,11 +811,14 @@ def test_recover_recovery_staging_path( recovery_staging_path_config, expected_recovery_staging_path, should_error, + error_substring, monkeypatch, capsys, ): # GIVEN a backup parse_backup_id_mock.return_value = mock_backup_info + # AND the backup is not incremental + mock_backup_info.is_incremental = False # AND the backup has the specified compression mock_backup_info.compression = backup_is_compressed and "gzip" or None # AND a configuration with the specified recovery_staging_path @@ -815,13 +841,165 @@ def test_recover_recovery_staging_path( # THEN if we expected an error the error was observed _, err = capsys.readouterr() + errors = [msg for msg in err.split("\n") if msg.startswith("ERROR: ")] if should_error: assert len(err) > 0 + assert any([error_substring in msg for msg in errors]) else: + assert len(errors) == 0 # AND if we expected success, the server config recovery staging # path matches expectations assert server.recovery_staging_path == expected_recovery_staging_path + @pytest.mark.parametrize( + ( + "backup_is_incremental", + "local_staging_path_arg", + "local_staging_path_config", + "expected_local_staging_path", + "should_error", + "error_substring", + ), + [ + # If a backup is not incremental then local_staging_path is ignored + (False, None, None, None, False, None), + # If a backup is incremental and no local_staging_path is provided + # we expect an error + ( + True, + None, + None, + None, + True, + "backup will be combined with pg_combinebackup in the barman host but " + "no local staging path is provided.", + ), + # If a backup is incremental and an argument is provided then it should + # be set in the config + (True, "/from/arg", None, "/from/arg", False, None), + # If a backup is incremental and a bad argument is provided then it should + # error + ( + True, + "from/arg", + None, + None, + True, + "Cannot parse local staging path: Invalid value : 'from/arg' (must " + "be an absolute path)", + ), + # If a backup is incremental and a config value is set then it should + # be set in the config + (True, None, "/from/conf", "/from/conf", False, None), + # If a backup is incremental and both arg and config are set then arg + # takes precedence + (True, "/from/arg", "/from/conf", "/from/arg", False, None), + ], + ) + @patch("barman.cli.parse_backup_id") + @patch("barman.cli.get_server") + def test_recover_local_staging_path( + self, + get_server_mock, + parse_backup_id_mock, + mock_backup_info, + mock_recover_args, + backup_is_incremental, + local_staging_path_arg, + local_staging_path_config, + expected_local_staging_path, + should_error, + error_substring, + monkeypatch, + capsys, + ): + # GIVEN a backup + parse_backup_id_mock.return_value = mock_backup_info + # AND the backup is incremental + mock_backup_info.is_incremental = backup_is_incremental + # AND a configuration with the specified local_staging_path + config = build_config_from_dicts( + global_conf={"local_staging_path": local_staging_path_config}, + ) + server = config.get_server("main") + get_server_mock.return_value.config = server + monkeypatch.setattr( + barman, + "__config__", + (config,), + ) + # WHEN recover is called with the specified --local-staging-path + mock_recover_args.local_staging_path = local_staging_path_arg + + # WITH a barman recover command + with pytest.raises(SystemExit): + recover(mock_recover_args) + + # THEN if we expected an error the error was observed + _, err = capsys.readouterr() + errors = [msg for msg in err.split("\n") if msg.startswith("ERROR: ")] + if should_error: + assert len(err) > 0 + assert any([error_substring in msg for msg in errors]) + else: + assert len(errors) == 0 + # AND if we expected success, the server config recovery staging + # path matches expectations + assert server.local_staging_path == expected_local_staging_path + + @pytest.mark.parametrize( + ("status", "should_error"), + [ + (BackupInfo.DONE, False), + (BackupInfo.WAITING_FOR_WALS, False), + (BackupInfo.FAILED, True), + (BackupInfo.EMPTY, True), + (BackupInfo.SYNCING, True), + (BackupInfo.STARTED, True), + ], + ) + @patch("barman.output.error") + @patch("barman.cli.parse_backup_id") + @patch("barman.cli.get_server") + def test_recover_backup_status( + self, + get_server_mock, + parse_backup_id_mock, + error_mock, + status, + should_error, + mock_recover_args, + ): + + server = build_mocked_server(name="test_server") + + get_server_mock.return_value = server + + backup_info = build_test_backup_info( + server=server, + backup_id="test_backup_id", + status=status, + ) + + parse_backup_id_mock.return_value = backup_info + mock_recover_args.backup_id = "test_backup_id" + mock_recover_args.snapshot_recovery_instance = None + + with pytest.raises( + SystemExit, + ): + recover(mock_recover_args) + + if should_error: + error_mock.assert_called_once_with( + "Cannot recover from backup '%s' of server " + "'%s': backup status is not DONE", + "test_backup_id", + "test_server", + ) + else: + error_mock.assert_not_called() + @pytest.mark.parametrize( ( "snapshots_info", @@ -1069,7 +1247,7 @@ def test_backup_immediate_checkpoint( mock_get_server_list.return_value = mock_server_list # WHEN backup is called with the immediate_checkpoint arg - mock_args = Mock(server_name=server_name) + mock_args = Mock(server_name=server_name, backup_id=None) if arg_value is not None: mock_args.immediate_checkpoint = arg_value else: @@ -1342,12 +1520,17 @@ def monkeypatch_config(self, monkeypatch): @patch("barman.cli.parse_backup_id") @patch("barman.cli.get_server") def test_barman_keep( - self, mock_get_server, mock_parse_backup_id, mock_args, monkeypatch_config + self, + mock_get_server, + mock_parse_backup_id, + mock_args, + monkeypatch_config, ): """Verify barman keep command calls keep_backup""" mock_args.target = "standalone" mock_parse_backup_id.return_value.backup_id = "test_backup_id" mock_parse_backup_id.return_value.status = BackupInfo.DONE + mock_parse_backup_id.return_value.is_incremental = False keep(mock_args) mock_get_server.return_value.backup_manager.keep_backup.assert_called_once_with( "test_backup_id", "standalone" @@ -1356,13 +1539,18 @@ def test_barman_keep( @patch("barman.cli.parse_backup_id") @patch("barman.cli.get_server") def test_barman_keep_fails_if_no_target_release_or_status_provided( - self, mock_get_server, mock_parse_backup_id, mock_args, capsys + self, + mock_get_server, + mock_parse_backup_id, + mock_args, + capsys, ): """ Verify barman keep command fails if none of --release, --status or --target are provided. """ mock_parse_backup_id.return_value.backup_id = "test_backup_id" + mock_parse_backup_id.return_value.is_incremental = False mock_parse_backup_id.return_value.status = BackupInfo.DONE with pytest.raises(SystemExit): keep(mock_args) @@ -1384,6 +1572,7 @@ def test_barman_keep_backup_not_done( """Verify barman keep command will not add keep if backup is not done""" mock_args.target = "standalone" mock_parse_backup_id.return_value.backup_id = "test_backup_id" + mock_parse_backup_id.return_value.is_incremental = False mock_parse_backup_id.return_value.status = BackupInfo.WAITING_FOR_WALS with pytest.raises(SystemExit): keep(mock_args) @@ -1397,7 +1586,11 @@ def test_barman_keep_backup_not_done( @patch("barman.cli.parse_backup_id") @patch("barman.cli.get_server") def test_barman_keep_release( - self, mock_get_server, mock_parse_backup_id, mock_args, monkeypatch_config + self, + mock_get_server, + mock_parse_backup_id, + mock_args, + monkeypatch_config, ): """Verify `barman keep --release` command calls release_keep""" mock_parse_backup_id.return_value.backup_id = "test_backup_id" @@ -1419,6 +1612,7 @@ def test_barman_keep_status( ): """Verify `barman keep --status` command prints get_keep_target output""" mock_parse_backup_id.return_value.backup_id = "test_backup_id" + mock_parse_backup_id.return_value.is_incremental = False mock_get_server.return_value.backup_manager.get_keep_target.return_value = ( "standalone" ) @@ -1442,6 +1636,7 @@ def test_barman_keep_status_nokeep( ): """Verify `barman keep --status` command prints get_keep_target output""" mock_parse_backup_id.return_value.backup_id = "test_backup_id" + mock_parse_backup_id.return_value.is_incremental = False mock_get_server.return_value.backup_manager.get_keep_target.return_value = None mock_args.status = True keep(mock_args) @@ -1451,6 +1646,45 @@ def test_barman_keep_status_nokeep( out, _err = capsys.readouterr() assert "nokeep" in out + @patch("barman.cli.parse_backup_id") + @patch("barman.cli.get_server") + def test_barman_keep_incremental_backup( + self, + mock_get_server, + mock_parse_backup_id, + mock_args, + capsys, + ): + """Verify barman keep command will not add keep if backup is incremental""" + mock_args.target = "standalone" + mock_parse_backup_id.return_value.backup_id = "test_backup_id" + mock_parse_backup_id.return_value.is_incremental = True + mock_parse_backup_id.return_value.status = BackupInfo.DONE + + with pytest.raises(SystemExit): + keep(mock_args) + _out, err = capsys.readouterr() + assert ( + "Unable to execute the keep command on backup test_backup_id: is an incremental backup.\n" + "Only full backups are eligible for the use of the keep command." + ) in err + mock_get_server.return_value.backup_manager.keep_backup.assert_not_called() + + @patch("barman.cli.parse_backup_id") + @patch("barman.cli.get_server") + def test_barman_keep_full_backup( + self, mock_get_server, mock_parse_backup_id, mock_args + ): + """Verify barman keep command will add keep if backup is not incremental""" + mock_parse_backup_id.return_value.backup_id = "test_backup_id" + mock_parse_backup_id.return_value.is_incremental = False + mock_parse_backup_id.return_value.status = BackupInfo.DONE + mock_args.release = True + keep(mock_args) + mock_get_server.return_value.backup_manager.release_keep.assert_called_once_with( + "test_backup_id" + ) + class TestCliHelp(object): """ diff --git a/tests/test_cloud_snapshot_interface.py b/tests/test_cloud_snapshot_interface.py index 7b23f8f0f..17993bf06 100644 --- a/tests/test_cloud_snapshot_interface.py +++ b/tests/test_cloud_snapshot_interface.py @@ -131,6 +131,27 @@ def test_from_config_azure_no_subscription_id(self): in str(exc.value) ) + @mock.patch("barman.cloud_providers.aws_s3.boto3") + def test_from_config_aws(self, mock_boto3): + """ + Verify aws-specific Barman config options are passed to snapshot interface. + """ + # GIVEN a server config with the aws snapshot provider and the specified + # parameters + mock_config = mock.Mock( + snapshot_provider="aws", + aws_region="us-east-2", + aws_profile="default", + aws_await_snapshots_timeout=7200, + ) + # WHEN get_snapshot_interface_from_server_config is called + snapshot_interface = get_snapshot_interface_from_server_config(mock_config) + # THEN the config values are passed to the snapshot interface + assert isinstance(snapshot_interface, AwsCloudSnapshotInterface) + assert snapshot_interface.region == "us-east-2" + assert snapshot_interface.await_snapshots_timeout == 7200 + mock_boto3.Session.assert_called_once_with(profile_name="default") + @pytest.mark.parametrize( ("snapshot_provider", "interface_cls"), [ @@ -341,6 +362,27 @@ def test_from_args_azure_no_subscription(self): "cloud provider is azure-blob-storage" ) == str(exc.value) + @mock.patch("barman.cloud_providers.aws_s3.boto3") + def test_from_args_aws(self, mock_boto3): + """ + Verify aws-specific barman-cloud args are passed to the snapshot interface. + """ + # GIVEN a cloud config with the aws snapshot provider and the specified + # parameters + mock_config = mock.Mock( + cloud_provider="aws-s3", + aws_region="us-east-2", + aws_profile="default", + aws_await_snapshots_timeout=7200, + ) + # WHEN get_snapshot_interface is called + snapshot_interface = get_snapshot_interface(mock_config) + # THEN the config values are passed to the snapshot interface + assert isinstance(snapshot_interface, AwsCloudSnapshotInterface) + assert snapshot_interface.region == "us-east-2" + assert snapshot_interface.await_snapshots_timeout == 7200 + mock_boto3.Session.assert_called_once_with(profile_name="default") + class TestGcpCloudSnapshotInterface(object): """ @@ -2759,7 +2801,8 @@ def test_take_snapshot_backup(self, number_of_disks, mock_ec2_client): # THEN we waited for completion of all snapshots expected_snapshot_ids = [self._get_snapshot_id(disk) for disk in disks] mock_ec2_client.get_waiter.return_value.wait.assert_called_once_with( - Filters=[{"Name": "snapshot-id", "Values": expected_snapshot_ids}] + Filters=[{"Name": "snapshot-id", "Values": expected_snapshot_ids}], + WaiterConfig={"Delay": 15, "MaxAttempts": 240}, ) # AND the backup_info is updated with the expected snapshot metadata @@ -2831,6 +2874,62 @@ def test_take_snapshot_backup_disks_not_attached(self, mock_ec2_client): ) ) + @pytest.mark.parametrize( + ("await_snapshots_timeout", "expected_wait_config"), + ( + # No timeout specified, default values should be used + (None, {"Delay": 15, "MaxAttempts": 240}), + # Timeout of zero specified, only one attempt should be used + (0, {"Delay": 15, "MaxAttempts": 1}), + # Timeout less than delay, only one attempt should be used + (10, {"Delay": 15, "MaxAttempts": 1}), + # Timeout greater than delay, but less than 2*delay, two attempts should be used + (20, {"Delay": 15, "MaxAttempts": 2}), + # Large timeout value should result in many attempts + (7200, {"Delay": 15, "MaxAttempts": 480}), + ), + ) + def test_take_snapshot_backup_wait( + self, await_snapshots_timeout, expected_wait_config, mock_ec2_client + ): + """ + Verify that take_snapshot_backup waits for completion of all snapshots and + updates the backup_info when complete. + """ + # GIVEN a set of disks, represented as VolumeMetadata + number_of_disks = 2 + disks = self.aws_disks[:number_of_disks] + assert len(disks) == number_of_disks + volumes = self._get_mock_volumes(disks) + # AND a backup_info for a given server name and backup ID + backup_info = mock.Mock(backup_id=self.backup_id, server_name=self.server_name) + # AND a mock EC2 client which returns an instance with the required disks + # attached + mock_ec2_client.describe_instances.return_value = ( + self._get_mock_describe_instances_resp(disks) + ) + # AND the mock EC2 client returns successful create_snapashot responses + mock_ec2_client.create_snapshot.side_effect = self._get_mock_create_snapshot( + disks + ) + # AND a new AwsCloudSnapshotInterface + kwargs = {"region": self.aws_region} + if await_snapshots_timeout is not None: + kwargs["await_snapshots_timeout"] = await_snapshots_timeout + snapshot_interface = AwsCloudSnapshotInterface(**kwargs) + + # WHEN take_snapshot_backup is called + snapshot_interface.take_snapshot_backup( + backup_info, self.aws_instance_id, volumes + ) + + # THEN we waited for completion of all snapshots with the expected WaiterConfig + expected_snapshot_ids = [self._get_snapshot_id(disk) for disk in disks] + mock_ec2_client.get_waiter.return_value.wait.assert_called_once_with( + Filters=[{"Name": "snapshot-id", "Values": expected_snapshot_ids}], + WaiterConfig=expected_wait_config, + ) + aws_live_states = ["pending", "running", "shutting-down", "stopping", "stopped"] def test_get_instance_metadata_by_id(self, mock_ec2_client): diff --git a/tests/test_command_wrappers.py b/tests/test_command_wrappers.py index 82507f6ce..57ff70c9e 100644 --- a/tests/test_command_wrappers.py +++ b/tests/test_command_wrappers.py @@ -1226,6 +1226,46 @@ def test_simple_invocation(self, popen, pipe_processor_loop, caplog): assert ("PgBaseBackup", DEBUG, out) in caplog.record_tuples assert ("PgBaseBackup", WARNING, err) in caplog.record_tuples + @pytest.mark.parametrize( + ("parent_backup_manifest_path", "expected_arg", "unexpected_arg"), + [ + ( + "/path/to/backup_manifest/file", + "--incremental=/path/to/backup_manifest/file", + None, + ), + (None, None, "--incremental"), + ], + ) + def test_incremental_backup_arg( + self, parent_backup_manifest_path, expected_arg, unexpected_arg + ): + """ + Asserts that incremental backup opions are passed correctly to the + pg_basebackup command. Only cares whether the correct arguments are + created and does not verify the behaviour of the command itself. + """ + connection_mock = mock.MagicMock() + connection_mock.get_connection_string.return_value = "fake_connstring" + compression_mock = None + cmd = command_wrappers.PgBaseBackup( + destination="/fake/target", + command=self.pg_basebackup_path, + connection=connection_mock, + version="17", + app_name="test_app_name", + compression=compression_mock, + parent_backup_manifest_path=parent_backup_manifest_path, + ) + + # Assert that the expected argument is present + if expected_arg is not None: + assert expected_arg in cmd.args + + # Assert that the unexpected argument is not present, no matter its value + if unexpected_arg is not None: + assert all(not arg.startswith(unexpected_arg) for arg in cmd.args) + @pytest.mark.parametrize( ("compression_config", "expected_args", "unexpected_args"), [ @@ -1717,6 +1757,169 @@ def test_init_simple(self, which): assert pg_verify_backup.out_handler +class TestPgCombineBackup(object): + """ + Class for testing of the :class:`PgCombineBackup` obj + """ + + pg_combinebackup_path = "/usr/bin/pg_combinebackup" + + def test_init_simple(self, which): + """ + Test class build + """ + connection_mock = mock.MagicMock() + connection_mock.get_connection_string.return_value = "test_conn" + pgcombinebackup = command_wrappers.PgCombineBackup( + destination="/fake/path", + command=self.pg_combinebackup_path, + connection=connection_mock, + version="17", + app_name="fake_app_name", + ) + assert pgcombinebackup.args == [ + "--output=/fake/path", + ] + assert pgcombinebackup.cmd == self.pg_combinebackup_path + assert pgcombinebackup.check is True + assert pgcombinebackup.close_fds is True + assert pgcombinebackup.allowed_retval == (0,) + assert pgcombinebackup.err_handler + assert pgcombinebackup.out_handler + + connection_mock.conn_parameters = { + "host": "fake host", + "port": "fake_port", + "user": "fake_user", + } + pgcombinebackup = command_wrappers.PgCombineBackup( + destination="/fake/target", + command=self.pg_combinebackup_path, + connection=connection_mock, + version="17", + app_name="fake_app_name", + ) + assert pgcombinebackup.args == [ + "--output=/fake/target", + ] + + which.return_value = None + which.side_effect = None + with pytest.raises(CommandFailedException): + # Expect an exception for pg_combinebackup not in path + command_wrappers.PgCombineBackup( + destination="/fake/target", + connection=connection_mock, + command="fake/path", + version="17", + app_name="fake_app_name", + ) + + def test_init_args(self): + """ + Test class build with additional arguments + """ + connection_mock = mock.MagicMock() + connection_mock.get_connection_string.return_value = "test_connstring" + backup_paths = ["/path/to/full_backup", "/path/to/incremental_backup"] + pg_combinebackup = command_wrappers.PgCombineBackup( + command="/path/to/pg_combinebackup", + connection=connection_mock, + version="17", + destination="/dest/dir", + args=backup_paths, + ) + assert ( + pg_combinebackup.args + == [ + "--output=/dest/dir", + ] + + backup_paths + ) + assert pg_combinebackup.cmd == "/path/to/pg_combinebackup" + assert pg_combinebackup.check is True + assert pg_combinebackup.close_fds is True + assert pg_combinebackup.allowed_retval == (0,) + assert pg_combinebackup.err_handler + assert pg_combinebackup.out_handler + + @mock.patch("barman.command_wrappers.Command.pipe_processor_loop") + @mock.patch("barman.command_wrappers.subprocess.Popen") + def test_simple_invocation(self, popen, pipe_processor_loop, caplog): + # See all logs + caplog.set_level(0) + + ret = 0 + out = "out" + err = "err" + + pipe = _mock_pipe(popen, pipe_processor_loop, ret, out, err) + connection_mock = mock.MagicMock() + connection_mock.get_connection_string.return_value = "fake_connstring" + cmd = command_wrappers.PgCombineBackup( + destination="/fake/target", + command=self.pg_combinebackup_path, + connection=connection_mock, + version="17", + app_name="fake_app_name", + ) + result = cmd.execute() + + popen.assert_called_with( + [ + self.pg_combinebackup_path, + "--output=/fake/target", + ], + close_fds=True, + env=None, + preexec_fn=mock.ANY, + shell=False, + stdout=mock.ANY, + stderr=mock.ANY, + stdin=mock.ANY, + ) + assert not pipe.stdin.write.called + pipe.stdin.close.assert_called_once_with() + assert result == ret + assert cmd.ret == ret + assert cmd.out is None + assert cmd.err is None + assert ("PgCombineBackup", DEBUG, out) in caplog.record_tuples + assert ("PgCombineBackup", WARNING, err) in caplog.record_tuples + + @pytest.mark.parametrize( + ("tbs_mapping", "expected_args"), + [ + ({"tbs1": "/dest1"}, ["--tablespace-mapping=tbs1=/dest1"]), + ( + {"tbs1": "/dest1", "tbs2": "/dest2"}, + [ + "--tablespace-mapping=tbs1=/dest1", + "--tablespace-mapping=tbs2=/dest2", + ], + ), + ], + ) + def test_tablespace_mapping(self, tbs_mapping, expected_args): + """ + Test that tablespace mappings are correctly passed + """ + connection_mock = mock.MagicMock() + connection_mock.get_connection_string.return_value = "fake_connstring" + cmd = command_wrappers.PgCombineBackup( + destination="/fake/target", + command=self.pg_combinebackup_path, + connection=connection_mock, + version="17", + app_name="fake_app_name", + tbs_mapping=tbs_mapping, + ) + + # Assert that the expected arguments are present + for expected_arg in expected_args: + assert expected_arg in cmd.args + + # noinspection PyMethodMayBeStatic class TestBarmanSubProcess(object): """ diff --git a/tests/test_config.py b/tests/test_config.py index bf884048b..23c065fb6 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -40,7 +40,7 @@ parse_backup_compression_format, parse_backup_compression_location, parse_si_suffix, - parse_recovery_staging_path, + parse_staging_path, parse_slot_name, parse_snapshot_disks, parse_time_interval, @@ -644,14 +644,14 @@ def test_populate_servers_and_models_following_symlink(self, tmpdir): symlink += 1 assert symlink == 1 - def test_parse_recovery_staging_path(self): + def test_parse_staging_path(self): """ - Test the parse_recovery_staging_path method + Test the parse_staging_path method """ - assert parse_recovery_staging_path(None) is None - assert parse_recovery_staging_path("/any/path") == "/any/path" + assert parse_staging_path(None) is None + assert parse_staging_path("/any/path") == "/any/path" with pytest.raises(ValueError): - parse_recovery_staging_path("here/it/is") + parse_staging_path("here/it/is") def test_parse_slot_name(self): """ @@ -1335,6 +1335,7 @@ def test_to_json(self, model_config): "archiver": None, "archiver_batch_size": None, "autogenerate_manifest": None, + "aws_await_snapshots_timeout": None, "aws_profile": None, "aws_region": None, "azure_credential": None, @@ -1364,9 +1365,11 @@ def test_to_json(self, model_config): "gcp_project": None, "gcp_zone": None, "immediate_checkpoint": None, + "keepalive_interval": None, "last_backup_maximum_age": None, "last_backup_minimum_size": None, "last_wal_maximum_age": None, + "local_staging_path": None, "max_incoming_wals_queue": None, "minimum_redundancy": None, "model": True, @@ -1421,6 +1424,7 @@ def test_to_json_with_config_source(self, model_config): "archiver": {"source": "SOME_SOURCE", "value": None}, "archiver_batch_size": {"source": "SOME_SOURCE", "value": None}, "autogenerate_manifest": {"source": "SOME_SOURCE", "value": None}, + "aws_await_snapshots_timeout": {"source": "SOME_SOURCE", "value": None}, "aws_profile": {"source": "SOME_SOURCE", "value": None}, "aws_region": {"source": "SOME_SOURCE", "value": None}, "azure_credential": {"source": "SOME_SOURCE", "value": None}, @@ -1450,9 +1454,11 @@ def test_to_json_with_config_source(self, model_config): "gcp_project": {"source": "SOME_SOURCE", "value": None}, "gcp_zone": {"source": "SOME_SOURCE", "value": None}, "immediate_checkpoint": {"source": "SOME_SOURCE", "value": None}, + "keepalive_interval": {"source": "SOME_SOURCE", "value": None}, "last_backup_maximum_age": {"source": "SOME_SOURCE", "value": None}, "last_backup_minimum_size": {"source": "SOME_SOURCE", "value": None}, "last_wal_maximum_age": {"source": "SOME_SOURCE", "value": None}, + "local_staging_path": {"source": "SOME_SOURCE", "value": None}, "max_incoming_wals_queue": {"source": "SOME_SOURCE", "value": None}, "minimum_redundancy": {"source": "SOME_SOURCE", "value": None}, "model": {"source": "SOME_SOURCE", "value": True}, @@ -2079,6 +2085,55 @@ def test_receive_config_changes_warnings(self, mock_warning, tmpdir): ] ) + @patch("barman.config.output.warning") + def test_receive_config_changes_with_empty_or_malformed_queue_file( + self, mock_warning, tmpdir + ): + # test it throws the expected warnings when invalid requests are issued + # and that it ignores the changes instead of applying them + config = Mock() + queue_file = tmpdir.join("cfg_changes.queue") + queue_file.ensure(file=True) + config.barman_home = tmpdir.strpath + config.config_changes_queue = queue_file.strpath + config._config.get_config_source.return_value = "default" + config.get_server.side_effect = [Mock()] * 2 + [None] + config.get_model.side_effect = [None] * 2 + [Mock()] + processor = ConfigChangesProcessor(config) + + changes = [ + { + "server_name": "main", + "key1": "value1", + "key2": "value2", + "scope": "server", + } + ] + + processor.receive_config_changes(changes) + + with ConfigChangesQueue(config.config_changes_queue) as chgs_queue: + assert len(chgs_queue.queue) == 1 + assert isinstance(chgs_queue.queue[0], ConfigChangeSet) + + assert len(chgs_queue.queue[0].changes_set) == 2 + assert isinstance(chgs_queue.queue[0].changes_set[0], ConfigChange) + assert isinstance(chgs_queue.queue[0].changes_set[1], ConfigChange) + assert chgs_queue.queue[0].section == "main" + assert chgs_queue.queue[0].changes_set[0].key == "key1" + assert chgs_queue.queue[0].changes_set[0].value == "value1" + assert chgs_queue.queue[0].changes_set[1].key == "key2" + assert chgs_queue.queue[0].changes_set[1].value == "value2" + + mock_warning.assert_has_calls( + [ + call( + "Malformed or empty configuration change queue: %s" + % str(queue_file) + ) + ] + ) + @patch("barman.config.output.info") def test_process_conf_changes_queue(self, mock_info, tmpdir): config = Mock() @@ -2117,8 +2172,6 @@ def test_process_conf_changes_queue(self, mock_info, tmpdir): ), ] - # processor.applied_changes = changes - changes = [ { "server_name": "main", @@ -2214,6 +2267,29 @@ def test_config_changes_queue(self, tmpdir): saved_queue = json.load(file, object_hook=ConfigChangeSet.from_dict) assert saved_queue == change_sets + @patch("barman.config.output.warning") + def test_config_changes_queue_with_empty_or_malformed_queue_file( + self, mock_warning, tmpdir + ): + config = Mock() + queue_file = tmpdir.join("cfg_changes.queue") + queue_file.ensure(file=True) + config.barman_home = tmpdir.strpath + config.config_changes_queue = queue_file.strpath + config._config.get_config_source.return_value = "default" + # Initialize the ConfigChangesQueue + with ConfigChangesQueue(config.config_changes_queue) as queue_w: + # Ensure the queue is initially empty + assert queue_w.queue == [] + mock_warning.assert_has_calls( + [ + call( + "Malformed or empty configuration change queue: %s" + % str(queue_file) + ) + ] + ) + class TestConfigChangeSet: def test_config_change_set_from_dict(self): diff --git a/tests/test_executor.py b/tests/test_executor.py index 0f2833717..c63eda03d 100644 --- a/tests/test_executor.py +++ b/tests/test_executor.py @@ -24,11 +24,12 @@ import mock import pytest from dateutil import tz -from mock import Mock, PropertyMock, patch +from mock import Mock, PropertyMock, patch, call from barman.backup_executor import ( ExclusiveBackupStrategy, PostgresBackupExecutor, + PostgresBackupStrategy, RsyncBackupExecutor, SnapshotBackupExecutor, ) @@ -204,6 +205,9 @@ def test_backup(self, rwbb_mock, gpb_mock, backup_copy_mock, capsys, tmpdir): "backup_options": "exclusive_backup", } ) + # mocks the keep-alive query + backup_manager.server.postgres.send_heartbeat_query.return_value = True, None + backup_info = LocalBackupInfo(backup_manager.server, backup_id="fake_backup_id") backup_info.begin_xlog = "0/2000028" backup_info.begin_wal = "000000010000000000000002" @@ -589,7 +593,8 @@ def test_exclusive_start_backup(self): """ # Build a backup_manager using a mocked server server = build_mocked_server( - main_conf={"backup_options": BackupOptions.EXCLUSIVE_BACKUP} + main_conf={"backup_options": BackupOptions.EXCLUSIVE_BACKUP}, + pg_version=170000, ) backup_manager = build_backup_manager(server=server) @@ -614,7 +619,6 @@ def test_exclusive_start_backup(self): "file_offset": 11845848, "timestamp": start_time, } - # Build a test empty backup info backup_info = LocalBackupInfo(server=backup_manager.server, backup_id="fake_id") @@ -644,7 +648,8 @@ def test_start_backup_for_old_pg(self): # Test: start concurrent backup # Build a backup_manager using a mocked server server = build_mocked_server( - main_conf={"backup_options": BackupOptions.CONCURRENT_BACKUP} + main_conf={"backup_options": BackupOptions.CONCURRENT_BACKUP}, + pg_version=170000, ) backup_manager = build_backup_manager(server=server) # Simulate old Postgres version @@ -664,7 +669,8 @@ def test_concurrent_start_backup(self): # Test: start concurrent backup # Build a backup_manager using a mocked server server = build_mocked_server( - main_conf={"backup_options": BackupOptions.CONCURRENT_BACKUP} + main_conf={"backup_options": BackupOptions.CONCURRENT_BACKUP}, + pg_version=90600, # this is a postgres 9.6 ) backup_manager = build_backup_manager(server=server) # Mock server.get_pg_setting('data_directory') call @@ -678,9 +684,6 @@ def test_concurrent_start_backup(self): # Mock server.get_pg_tablespaces() call tablespaces = [Tablespace._make(("test_tbs", 1234, "/tbs/test"))] server.postgres.get_tablespaces.return_value = tablespaces - # this is a postgres 9.6 - server.postgres.server_version = 90600 - # Mock call to new api method start_time = datetime.datetime.now() server.postgres.start_concurrent_backup.return_value = { @@ -761,13 +764,12 @@ def test_concurrent_stop_backup(self, tbs_map_mock): """ # Build a backup info and configure the mocks server = build_mocked_server( - main_conf={"backup_options": BackupOptions.CONCURRENT_BACKUP} + main_conf={"backup_options": BackupOptions.CONCURRENT_BACKUP}, + pg_version=90600, # This is a postgres 9.6 ) backup_manager = build_backup_manager(server=server) stop_time = datetime.datetime.now() - # This is a pg 9.6 - server.postgres.server_version = 90600 # Mock stop backup call for the new api method start_time = datetime.datetime.now(tz.tzlocal()).replace(microsecond=0) server.postgres.stop_concurrent_backup.return_value = { @@ -831,6 +833,75 @@ def test_exclusive_check(self, server_version, expected_message, capsys): else: assert check_result.status is True + @pytest.mark.parametrize( + ("server_version", "expected_value"), + [(160000, None), (170000, "on")], + ) + def test__pg_get_metadata(self, server_version, expected_value): + # Given a PostgreSQL connection of the specified version + mock_postgres = mock.Mock() + mock_postgres.server_version = server_version + # Mock the get_setting("data_directory"), get_setting("data_checksums") and + # get_setting("summarize_wal") calls, respectively + mock_postgres.get_setting.side_effect = [ + "data_directory", + "off", + expected_value, + ] + + # Mock postgres server.get_configuration_files() call + mock_postgres.get_configuration_files.return_value = dict( + config_file="/etc/postgresql.conf", + hba_file="/pg/pg_hba.conf", + ident_file="/pg/pg_ident.conf", + ) + # Mock postgres server.get_tablespaces() call + tablespaces = [Tablespace._make(("test_tbs", 1234, "/tbs/test"))] + mock_postgres.get_tablespaces.return_value = tablespaces + + mock_postgres.current_size = 2048 + mock_postgres.xlog_segment_size = 16 + strategy = PostgresBackupStrategy(mock_postgres, "test server") + backup_info = build_test_backup_info() + + # default values from build_test_backup_info() to verify that the values + # set in this method were changed inplace after calling the method + assert backup_info.pgdata == "/pgdata/location" + assert backup_info.version == 90302 + assert backup_info.xlog_segment_size == 16777216 + assert backup_info.tablespaces == [ + Tablespace(name="tbs1", oid=16387, location="/fake/location"), + Tablespace(name="tbs2", oid=16405, location="/another/location"), + ] + assert backup_info.summarize_wal is None + assert backup_info.cluster_size == 2048 + + strategy._pg_get_metadata(backup_info) + + mock_postgres.get_tablespaces.assert_called_once() + mock_postgres.get_configuration_files.assert_called_once() + if mock_postgres.server_version < 170000: + calls = [call("data_directory"), call("data_checksums")] + mock_postgres.get_setting.assert_has_calls(calls, any_order=False) + assert mock_postgres.get_setting.call_count == 2 + assert backup_info.summarize_wal is None + assert backup_info.version == 160000 + else: + calls = [ + call("data_directory"), + call("data_checksums"), + call("summarize_wal"), + ] + assert mock_postgres.get_setting.call_count == 3 + mock_postgres.get_setting.assert_has_calls(calls, any_order=False) + assert backup_info.summarize_wal == "on" + assert backup_info.version == 170000 + + assert backup_info.pgdata == "data_directory" + assert backup_info.xlog_segment_size == 16 + assert backup_info.tablespaces == tablespaces + assert backup_info.cluster_size == 2048 + class TestPostgresBackupExecutor(object): """ @@ -881,6 +952,7 @@ def test_backup(self, gpb_mock, pbc_mock, capsys, tmpdir): begin_offset=28, copy_stats=dict(copy_time=100, total_time=105), ) + backup_manager.server.postgres.server_version = backup_info.version current_xlog_timestamp = datetime.datetime(2015, 10, 26, 14, 38) backup_manager.server.postgres.current_xlog_info = dict( location="0/12000090", @@ -1097,6 +1169,7 @@ def test_backup_copy(self, remote_mock, pg_basebackup_mock, tmpdir, capsys): compression=None, err_handler=mock.ANY, out_handler=mock.ANY, + parent_backup_manifest_path=None, ), mock.call()(), ] @@ -1135,6 +1208,7 @@ def test_backup_copy(self, remote_mock, pg_basebackup_mock, tmpdir, capsys): compression=None, err_handler=mock.ANY, out_handler=mock.ANY, + parent_backup_manifest_path=None, ), mock.call()(), ] @@ -1172,6 +1246,7 @@ def test_backup_copy(self, remote_mock, pg_basebackup_mock, tmpdir, capsys): compression=None, err_handler=mock.ANY, out_handler=mock.ANY, + parent_backup_manifest_path=None, ), mock.call()(), ] @@ -1204,6 +1279,7 @@ def test_backup_copy(self, remote_mock, pg_basebackup_mock, tmpdir, capsys): compression=None, err_handler=mock.ANY, out_handler=mock.ANY, + parent_backup_manifest_path=None, ), mock.call()(), ] @@ -1218,16 +1294,62 @@ def test_backup_copy(self, remote_mock, pg_basebackup_mock, tmpdir, capsys): with pytest.raises(DataTransferFailure): backup_manager.executor.backup_copy(backup_info) + # Check incremental backups with Postgres 17 onward + remote_mock.reset_mock() + pg_basebackup_mock.reset_mock() + pg_basebackup_mock.return_value.side_effect = None + backup_manager.executor._remote_status = None + remote_mock.return_value = { + "pg_basebackup_version": "17", + "pg_basebackup_path": "/fake/path", + "pg_basebackup_bwlimit": True, + } + backup_manager.executor.config.immediate_checkpoint = True + backup_manager.executor.config.streaming_conninfo = "fake=connstring" + mock_parent_backup_info = Mock() + mock_parent_backup_info.get_backup_manifest_path.return_value = "/SOME/MANIFEST" + with patch("barman.infofile.LocalBackupInfo.get_parent_backup_info") as mock_gp: + mock_gp.return_value = mock_parent_backup_info + backup_manager.executor.backup_copy(backup_info) + out, err = capsys.readouterr() + assert out == "" + assert err == "" + # Check that expected parameter was passed to pg_basebackup to identify + # the parent backup + assert pg_basebackup_mock.mock_calls == [ + mock.call.make_logging_handler(logging.INFO), + mock.call( + connection=mock.ANY, + version="17", + app_name="barman_streaming_backup", + destination=mock.ANY, + command="/fake/path", + tbs_mapping=mock.ANY, + bwlimit=1, + immediate=True, + retry_times=0, + retry_sleep=30, + retry_handler=mock.ANY, + path=mock.ANY, + compression=None, + err_handler=mock.ANY, + out_handler=mock.ANY, + parent_backup_manifest_path="/SOME/MANIFEST", + ), + mock.call()(), + ] + def test_postgres_start_backup(self): """ Test concurrent backup using pg_basebackup """ # Test: start concurrent backup backup_manager = build_backup_manager(global_conf={"backup_method": "postgres"}) - # Mock server.get_pg_setting('data_directory') call + # Mock the get_setting("data_directory") and get_setting("data_checksums") calls postgres_mock = backup_manager.server.postgres postgres_mock.get_setting.side_effect = [ "/test/fake_data_dir", + "off", ] # Mock server.get_pg_configuration_files() call postgres_mock.get_configuration_files.return_value = dict( diff --git a/tests/test_fs.py b/tests/test_fs.py index 5f3d12aa0..0fd07e6da 100644 --- a/tests/test_fs.py +++ b/tests/test_fs.py @@ -16,6 +16,7 @@ # You should have received a copy of the GNU General Public License # along with Barman. If not, see . +import sys import pytest from mock import call, patch @@ -553,6 +554,121 @@ def test_findmnt_unexpected_output(self, command_mock, command_output): # AND the exception has the expected message assert str(exc.value) == "Unexpected findmnt output: {}".format(command_output) + @patch("barman.fs.Command") + @patch("barman.fs.UnixLocalCommand.cmd") + def test_get_system_info(self, cmd_mock, command_mock): + """Basic test for the get_system_info method.""" + # For this test, we mock everything as if we are on an Ubuntu distro + # the lsb_release command succededs + cmd_mock.return_value = 0 + # mock the internal_cmd.out.rstrip() calls, in sequence + command_mock.return_value.out.rstrip.side_effect = [ + # lsb_release -a output + "Ubuntu Linux 20.04.1 LTS", + # uname -a output + "Linux version 5.4.0-54-generic (buildd@lgw01-amd64)", + # ssh -V output + "OpenSSH_8.2p1 Ubuntu-4ubuntu0.3", + ] + # rsync --version output + command_mock.return_value.out.splitlines.return_value = ["Rsync version 3.2.3"] + + result = UnixLocalCommand().get_system_info() + + assert result == { + "release": "Ubuntu Linux 20.04.1 LTS", + "python_ver": f"Python {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}", + "python_executable": sys.executable, + "kernel_ver": "Linux version 5.4.0-54-generic (buildd@lgw01-amd64)", + "rsync_ver": "Rsync version 3.2.3", + "ssh_ver": "OpenSSH_8.2p1 Ubuntu-4ubuntu0.3", + } + + @patch("barman.fs.Command") + @patch("barman.fs.UnixLocalCommand.cmd") + @patch("barman.fs.UnixLocalCommand.exists") + def test_get_system_info_release_cases(self, exists_mock, cmd_mock, command_mock): + """ + Test all possible cases for the release ouput in the system info. + Other configs not related to the release are abstracted in this test. + """ + # Case 1: We are on an Ubuntu system + # the lsb_release command succededs + cmd_mock.return_value = 0 + # mock the internal_cmd.out.rstrip() calls, in sequence + command_mock.return_value.out.rstrip.side_effect = [ + "Ubuntu Linux 20.04.1 LTS", # output of lsb_release -a + "Some output of `uname -a` command", + "Some output of `ssh -V` command", + ] + command_mock.return_value.out.splitlines.return_value = ["Some Rsync version"] + result = UnixLocalCommand().get_system_info()["release"] + assert result == "Ubuntu Linux 20.04.1 LTS" + + # Case 2: We are on a Ubuntu system, but the lsb_release command does not exist + cmd_mock.reset_mock(), command_mock.reset_mock() + # the lsb_release command does not succeded + cmd_mock.return_value = 1 + # The /etc/lsb-release path exists + exists_mock.return_value = True + # mock the internal_cmd.out.rstrip() calls, in sequence + command_mock.return_value.out.rstrip.side_effect = [ + "22.04.1 LTS", # ouput of cat /etc/lsb-release + "Some output of `uname -a` command", + "Some output of `ssh -V` command", + ] + command_mock.return_value.out.splitlines.return_value = ["Some Rsync version"] + result = UnixLocalCommand().get_system_info()["release"] + assert result == "Ubuntu Linux 22.04.1 LTS" + + # Case 3: We are on a Debian system + cmd_mock.reset_mock(), command_mock.reset_mock(), exists_mock.reset_mock() + # the lsb_release command does not succeded + cmd_mock.return_value = 1 + # /etc/lsb-release does not exist, /etc/debian_version exists + exists_mock.side_effect = [False, True] + # mock the internal_cmd.out.rstrip() calls, in sequence + command_mock.return_value.out.rstrip.side_effect = [ + "10.7", # output of cat /etc/debian_version + "Some output of `uname -a` command", + "Some output of `ssh -V` command", + ] + command_mock.return_value.out.splitlines.return_value = ["Some Rsync version"] + result = UnixLocalCommand().get_system_info()["release"] + assert result == "Debian GNU/Linux 10.7" + + # Case 4: We are on a RHEL system + cmd_mock.reset_mock(), command_mock.reset_mock(), exists_mock.reset_mock() + # the lsb_release command does not succeded + cmd_mock.return_value = 1 + # /etc/lsb-release does not exist, /etc/debian_version does not exist, /etc/redhat-release exists + exists_mock.side_effect = [False, False, True] + # mock the internal_cmd.out.rstrip() calls, in sequence + command_mock.return_value.out.rstrip.side_effect = [ + "7.9.2009 (Core)", # output of cat /etc/redhat-release + "Some output of `uname -a` command", + "Some output of `ssh -V` command", + ] + command_mock.return_value.out.splitlines.return_value = ["Some Rsync version"] + result = UnixLocalCommand().get_system_info()["release"] + assert result == "RedHat Linux 7.9.2009 (Core)" + + # Case 5: We are on a MacOs system + cmd_mock.reset_mock(), command_mock.reset_mock(), exists_mock.reset_mock() + # the lsb_release command does not succeded, but all rest succeeds + cmd_mock.side_effect = [1, 0, 0, 0, 0] + # None of the releas efiles checked previously exists + exists_mock.side_effect = [False, False, False] + # mock the internal_cmd.out.rstrip() calls, in sequence + command_mock.return_value.out.rstrip.side_effect = [ + "macOS 11.1", # output of sw_vers + "Some output of `uname -a` command", + "Some output of `ssh -V` command", + ] + command_mock.return_value.out.splitlines.return_value = ["Some Rsync version"] + result = UnixLocalCommand().get_system_info()["release"] + assert result == "macOS 11.1" + class TestFileMatchingRules(object): def test_match_dirs_not_anchored(self): diff --git a/tests/test_infofile.py b/tests/test_infofile.py index fe436764b..d19ad589c 100644 --- a/tests/test_infofile.py +++ b/tests/test_infofile.py @@ -16,6 +16,7 @@ # You should have received a copy of the GNU General Public License # along with Barman. If not, see . +import copy import json import os import warnings @@ -24,6 +25,7 @@ import mock import pytest from dateutil.tz import tzlocal, tzoffset +from mock import PropertyMock, patch from barman.cloud_providers.aws_s3 import AwsSnapshotMetadata, AwsSnapshotsInfo from barman.cloud_providers.azure_blob_storage import ( AzureSnapshotMetadata, @@ -39,16 +41,25 @@ Field, FieldListFile, LocalBackupInfo, + SyntheticBackupInfo, WalFileInfo, load_datetime_tz, + dump_backup_ids, + load_backup_ids, +) +from testing_helpers import ( + build_backup_manager, + build_mocked_server, + build_real_server, + build_test_backup_info, ) -from testing_helpers import build_backup_manager, build_mocked_server, build_real_server BASE_BACKUP_INFO = """backup_label=None begin_offset=40 begin_time=2014-12-22 09:25:22.561207+01:00 begin_wal=000000010000000000000004 begin_xlog=0/4000028 +children_backup_ids=None config_file=/fakepath/postgresql.conf end_offset=184 end_time=2014-12-22 09:25:27.410470+01:00 @@ -58,6 +69,7 @@ hba_file=/fakepath/pg_hba.conf ident_file=/fakepath/pg_ident.conf mode=default +parent_backup_id=None pgdata=/fakepath/data server_name=fake-9.4-server size=20935690 @@ -89,6 +101,36 @@ def test_load_datetime_tz(): load_datetime_tz("Invalid datetime") +@pytest.mark.parametrize( + ("input", "expected"), + [ + (None, None), + (["SOME_BACKUP_ID"], "SOME_BACKUP_ID"), + (["SOME_BACKUP_ID_1", "SOME_BACKUP_ID_2"], "SOME_BACKUP_ID_1,SOME_BACKUP_ID_2"), + ], +) +def test_dump_backup_ids(input, expected): + """ + Unit tests for :func:`dump_backup_ids`. + """ + assert dump_backup_ids(input) == expected + + +@pytest.mark.parametrize( + ("input", "expected"), + [ + (None, None), + ("SOME_BACKUP_ID", ["SOME_BACKUP_ID"]), + ("SOME_BACKUP_ID_1,SOME_BACKUP_ID_2", ["SOME_BACKUP_ID_1", "SOME_BACKUP_ID_2"]), + ], +) +def test_load_backup_ids(input, expected): + """ + Unit tests for :func:`load_backup_ids`. + """ + assert load_backup_ids(input) == expected + + # noinspection PyMethodMayBeStatic class TestField(object): def test_field_creation(self): @@ -478,6 +520,24 @@ def test_backup_info_from_backup_id(self, tmpdir): assert b_info.tablespaces[0].oid == 16384 assert b_info.tablespaces[0].location == "/fake_tmp/tbs" + @pytest.mark.parametrize( + ("mode", "parent_backup_id", "expected_backup_type"), + [ + ("rsync", None, "rsync"), + ("postgres", "some_id", "incremental"), + ("postgres", None, "full"), + ("snapshot", None, "snapshot"), + ], + ) + def test_backup_type(self, mode, parent_backup_id, expected_backup_type): + """ + Ensure :meth:`LocalBackupInfo.backup_type` returns the correct backup type label. + """ + backup_info = build_test_backup_info(parent_backup_id=parent_backup_id) + backup_info.mode = mode + + assert backup_info.backup_type == expected_backup_type + def test_backup_info_save(self, tmpdir): """ Test the save method of a BackupInfo object @@ -874,3 +934,437 @@ def test_with_no_snapshots_info(self, tmpdir): assert "snapshots_info" not in infofile.read() # AND the backup name is not included in the JSON output assert "snapshots_info" not in b_info.to_json().keys() + + +class TestLocalBackupInfo: + """ + Unit tests for :class:`LocalBackupInfo`. + """ + + @pytest.fixture + def backup_info(self, tmpdir): + """ + Create a new instance of :class:`LocalBackupInfo`. + + :return LocalBackupInfo: an instance of a local backup info. + """ + infofile = tmpdir.join("backup.info") + infofile.write(BASE_BACKUP_INFO) + # Mock the server, we don't need it at the moment + server = build_mocked_server() + # load the data from the backup.info file + return LocalBackupInfo(server, info_file=infofile.strpath) + + @mock.patch("barman.infofile.LocalBackupInfo.get_data_directory") + def test_get_backup_manifest_path(self, mock_get_data_dir, backup_info): + """ + Ensure :meth:`LocalBackupInfo.get_backup_manifest_path` returns the expected + path for its ``backup_manifest`` file. + """ + expected = "/some/random/path/backup_manifest" + mock_get_data_dir.return_value = "/some/random/path" + assert backup_info.get_backup_manifest_path() == expected + + @patch("barman.infofile.BackupInfo.is_incremental", new_callable=PropertyMock) + def test_get_parent_backup_info_no_parent(self, mock_is_incremental, backup_info): + """ + Ensure :meth:`LocalBackupInfo.get_parent_backup_info` returns ``None`` if the + backup is not incremental. + """ + mock_is_incremental.return_value = False + assert backup_info.get_parent_backup_info() is None + + @patch("barman.infofile.BackupInfo.is_incremental", new_callable=PropertyMock) + def test_get_parent_backup_info_empty_parent( + self, mock_is_incremental, backup_info + ): + """ + Ensure :meth:`LocalBackupInfo.get_parent_backup_info` returns ``None`` if the + backup is incremental and has a parent backup, but the parent backup is empty. + """ + mock_is_incremental.return_value = True + backup_info.parent_backup_id = "SOME_ID" + + with patch("barman.infofile.LocalBackupInfo") as mock: + mock.return_value.status = BackupInfo.EMPTY + assert backup_info.get_parent_backup_info() is None + mock.assert_called_once_with(backup_info.server, backup_id="SOME_ID") + + @patch("barman.infofile.BackupInfo.is_incremental", new_callable=PropertyMock) + def test_get_parent_backup_info_parent_ok(self, mock_is_incremental, backup_info): + """ + Ensure :meth:`LocalBackupInfo.get_parent_backup_info` returns the backup info + object of the parent. + """ + mock_is_incremental.return_value = True + backup_info.parent_backup_id = "SOME_ID" + + with patch("barman.infofile.LocalBackupInfo") as mock: + mock.return_value.status = BackupInfo.DONE + assert backup_info.get_parent_backup_info() is mock.return_value + mock.assert_called_once_with(backup_info.server, backup_id="SOME_ID") + + def test_get_child_backup_info_no_parent(self, backup_info): + """ + Ensure :meth:`LocalBackupInfo.get_child_backup_info` returns ``None`` if the + backup doesn't have children backups. + """ + backup_info.children_backup_ids = None + assert backup_info.get_child_backup_info("SOME_ID") is None + + def test_get_child_backup_info_not_a_child(self, backup_info): + """ + Ensure :meth:`LocalBackupInfo.get_child_backup_info` returns ``None`` if the + backup has children, but requested ID is not a child. + """ + backup_info.children_backup_ids = ["SOME_CHILD_ID_1", "SOME_CHILD_ID_2"] + assert backup_info.get_child_backup_info("SOME_ID") is None + + def test_get_child_backup_info_empty_child(self, backup_info): + """ + Ensure :meth:`LocalBackupInfo.get_child_backup_info` returns ``None`` if the + backup has children, but requested ID is from an empty child. + """ + backup_info.children_backup_ids = ["SOME_CHILD_ID_1", "SOME_CHILD_ID_2"] + + with patch("barman.infofile.LocalBackupInfo") as mock: + mock.return_value.status = BackupInfo.EMPTY + assert backup_info.get_child_backup_info("SOME_CHILD_ID_1") is None + mock.assert_called_once_with( + backup_info.server, + backup_id="SOME_CHILD_ID_1", + ) + + def test_get_child_backup_info_child_ok(self, backup_info): + """ + Ensure :meth:`LocalBackupInfo.get_child_backup_info` returns the backup info + object of the requested child. + """ + backup_info.children_backup_ids = ["SOME_CHILD_ID_1", "SOME_CHILD_ID_2"] + + with patch("barman.infofile.LocalBackupInfo") as mock: + mock.return_value.status = BackupInfo.DONE + assert ( + backup_info.get_child_backup_info("SOME_CHILD_ID_1") + is mock.return_value + ) + mock.assert_called_once_with( + backup_info.server, + backup_id="SOME_CHILD_ID_1", + ) + + def test_walk_to_root(self, backup_info): + """ + Unit test for :meth:`LocalBackupInfo.walk_to_root` method. + + This test checks if the method correctly walks through all the parent backups + of the current backup and returns a generator of :class:`LocalBackupInfo` + objects for each parent backup. + """ + # Create a LocalBackupInfo used as a model for the parent backups + # inside the side_effect function `provide_parent_backup_info` + model_backup_info = LocalBackupInfo( + backup_info.server, backup_id="model_backup" + ) + + def provide_parent_backup_info(server, backup_id): + """ + Helper function to provide a :class:`LocalBackupInfo` object for a given + backup ID. + """ + next_backup_id = int(backup_id[-1]) + 1 + parent_backup_info = copy.copy(model_backup_info) + parent_backup_info.backup_id = backup_id + parent_backup_info.status = BackupInfo.DONE + if next_backup_id < 4: + parent_backup_info.parent_backup_id = ( + "parent_backup_id%s" % next_backup_id + ) + else: + parent_backup_info.parent_backup_id = None + return parent_backup_info + + # Create parent backup info objects + backup_info.parent_backup_id = "parent_backup_id1" + with mock.patch( + "barman.infofile.LocalBackupInfo", + side_effect=provide_parent_backup_info, + ): + # Call the walk_to_root method + result = list(backup_info.walk_to_root(return_self=False)) + + # Check if the method correctly walks through all the parent backups + # in the correct order + assert len(result) == 3 + assert result[0].backup_id == "parent_backup_id1" + assert result[1].backup_id == "parent_backup_id2" + assert result[2].backup_id == "parent_backup_id3" + + # Test case for when the method is set to also return the current backup + backup_info.backup_id = "incremental_backup_id" + with mock.patch( + "barman.infofile.LocalBackupInfo", + side_effect=provide_parent_backup_info, + ): + # Call the walk_to_root method with include_self=True + result = list(backup_info.walk_to_root()) + + # Check if the method includes the current backup and walks through all + # the parent backups in the correct order and ALSO yields the current + # backup + assert len(result) == 4 + assert result[0].backup_id == "incremental_backup_id" + assert result[1].backup_id == "parent_backup_id1" + assert result[2].backup_id == "parent_backup_id2" + assert result[3].backup_id == "parent_backup_id3" + + def test_walk_backups_tree(self): + """ + Unit test for the :meth:`LocalBackupInfo.walk_backups_tree` method. + """ + # Create a mock server + server = build_mocked_server() + # Create a LocalBackupInfo used as a model for the parent backups + # inside the side_effect function `provide_parent_backup_info` + model_backup_info = LocalBackupInfo(server, backup_id="model_backup") + + def provide_child_backup_info(server, backup_id): + """ + Helper function to provide a :class:`LocalBackupInfo` object for a given + backup ID of a child backup. + """ + if backup_id == "child_backup1": + child1_backup_info = copy.copy(model_backup_info) + child1_backup_info.backup_id = "child_backup1" + child1_backup_info.status = BackupInfo.DONE + child1_backup_info.parent_backup_id = "root_backup" + child1_backup_info.children_backup_ids = ["child_backup3"] + return child1_backup_info + if backup_id == "child_backup2": + child2_backup_info = copy.copy(model_backup_info) + child2_backup_info.backup_id = "child_backup2" + child2_backup_info.status = BackupInfo.DONE + child2_backup_info.parent_backup_id = "root_backup" + return child2_backup_info + if backup_id == "child_backup3": + child3_backup_info = copy.copy(model_backup_info) + child3_backup_info.backup_id = "child_backup3" + child3_backup_info.status = BackupInfo.DONE + child3_backup_info.parent_backup_id = "child_backup1" + child3_backup_info.children_backup_ids = [] + return child3_backup_info + + # Create a root backup info object + # the final structure of the backups tree is: + # root_backup + # / \ + # child_backup1 child_backup2 + # / + # child_backup3 + root_backup_info = LocalBackupInfo(server, backup_id="root_backup") + root_backup_info.status = BackupInfo.DONE + root_backup_info.children_backup_ids = ["child_backup1", "child_backup2"] + + # Mock the `LocalBackupInfo` constructor to return the corresponding backup info objects + with patch( + "barman.infofile.LocalBackupInfo", + side_effect=provide_child_backup_info, + ): + # Call the `walk_backups_tree` method on the root backup info + backups = list(root_backup_info.walk_backups_tree()) + # Assert that the backups are returned in the correct order + # We want to walk through the tree in a depth-first post order, + # so leaf nodes are visited first, then their parent, and so on. + assert len(backups) == 4 + assert backups[0].backup_id == "child_backup3" + assert backups[1].backup_id == "child_backup1" + assert backups[2].backup_id == "child_backup2" + assert backups[3].backup_id == "root_backup" + + # Call the `walk_backups_tree` method on the root backup info + backups = list(root_backup_info.walk_backups_tree(return_self=False)) + # Assert that the backups are returned in the correct order + # We want to walk through the tree in a depth-first post order, + # so leaf nodes are visited first, then their parent, and so on. + assert len(backups) == 3 + assert backups[0].backup_id == "child_backup3" + assert backups[1].backup_id == "child_backup1" + assert backups[2].backup_id == "child_backup2" + + @pytest.mark.parametrize( + # The following are data_checksum configurations for 4 different backups of a chain + # root_backup incremental1 incremental2 incremental3 expected_result + ("conf_root", "conf_inc1", "conf_inc2", "conf_inc3", "expected"), + ( + # Case 1: checksums were disabled in the most recent backup + # so the chain is considered consistent automatically + ("on", "on", "on", "off", True), + # Case 2: checksums were enabled in the most recent backup and on all + # previous backups in the chain, so the chain is considered consistent + ("on", "on", "on", "on", True), + # Case 3: checksums were enabled in the most recent backup and disabled in + # one or more backups in the chain, so the chain is considered inconsistent + ("off", "off", "on", "on", False), + ), + ) + @patch("barman.infofile.BackupInfo.is_incremental", new_callable=PropertyMock) + def test_is_checksum_consistent( + self, mock_is_incremental, conf_root, conf_inc1, conf_inc2, conf_inc3, expected + ): + """ + Test checksum configuration consistency between Postgres incremental backups of a chain + """ + mock_is_incremental.return_value = True + backup_manager = build_backup_manager(main_conf={"backup_method": "postgres"}) + root_backup = build_test_backup_info( + server=backup_manager.server, data_checksums=conf_root + ) + incremental1 = build_test_backup_info( + server=backup_manager.server, + data_checksums=conf_inc1, + parent_backup_id=root_backup.backup_id, + ) + incremental2 = build_test_backup_info( + server=backup_manager.server, + data_checksums=conf_inc2, + parent_backup_id=incremental1.backup_id, + ) + incremental3 = build_test_backup_info( + server=backup_manager.server, + data_checksums=conf_inc3, + parent_backup_id=incremental2.backup_id, + ) + + with patch("barman.infofile.LocalBackupInfo.walk_to_root") as walk_mock: + walk_mock.return_value = (incremental2, incremental1, root_backup) + assert incremental3.is_checksum_consistent() is expected + + @patch("barman.infofile.BackupInfo.is_incremental", new_callable=PropertyMock) + def test_true_is_full_and_eligible_for_incremental(self, mock_is_incremental): + """ + Test that the function applies the correct conditions for a full backup + that is eligible for incremental mode. The backup_method should be `postgres`, + the summarize_wal should be ``on`` and backup should be incremental. + """ + mock_is_incremental.return_value = False + backup_method = "postgres" + summarize_wal = "on" + + pg_backup_manager = build_backup_manager( + main_conf={"backup_method": backup_method} + ) + + backup_info = build_test_backup_info( + server=pg_backup_manager.server, + backup_id="12345", + summarize_wal=summarize_wal, + ) + + assert backup_info.is_full_and_eligible_for_incremental() + + @pytest.mark.parametrize( + ("backup_method", "summarize_wal", "is_incremental"), + [ + ("postgres", "on", True), + ("postgres", "off", True), + ("postgres", "off", False), + ("rsync", "on", True), + ("rsync", "on", False), + ("rsync", "off", True), + ("rsync", "off", False), + ], + ) + @patch("barman.infofile.BackupInfo.is_incremental", new_callable=PropertyMock) + def test_false_is_full_and_eligible_for_incremental( + self, mock_is_incremental, backup_method, summarize_wal, is_incremental + ): + """ + Test that the function applies the correct conditions for a full backup + that is eligible for incremental mode. The backup_method should be `postgres`, + the summarize_wal should be `on` and backup should be incremental. + """ + mock_is_incremental.return_value = is_incremental + + backup_manager = build_backup_manager( + main_conf={"backup_method": backup_method} + ) + + backup_info = build_test_backup_info( + server=backup_manager.server, + backup_id="12345", + summarize_wal=summarize_wal, + ) + + assert not backup_info.is_full_and_eligible_for_incremental() + + +class TestSyntheticBackupInfo: + """ + this class tests the methods of the SyntheticBackupInfo object + """ + + def test_init_synthetic_backup_info_with_backup_id(self): + """ + Unit test for the __init__ method using backup_id. + + Create mock server and a SyntheticBackupInfo object. + + This unit tests checks: + * base_directory parameter + * backup_id parameter + * instance type + """ + server = build_mocked_server() + base_directory = "fake/path/" + backup_id = "fake_name" + obj = SyntheticBackupInfo( + server=server, + base_directory=base_directory, + backup_id=backup_id, + info_file=None, + ) + assert obj.base_directory == base_directory + assert obj.backup_id == backup_id + assert isinstance(obj, SyntheticBackupInfo) + + def test_init_synthetic_backup_info_with_info_file(self, tmpdir): + """ + Unit test for the __init__ method using info_file. + + Create mock server and a SyntheticBackupInfo object. + + This unit tests checks: + * base_directory parameter + * filename parameter + * instance type + """ + server = build_mocked_server() + base_directory = "fake/path/" + backup_id = "fake_name" + infofile = tmpdir.mkdir(backup_id).join("backup.info") + infofile.write(BASE_BACKUP_INFO) + obj = SyntheticBackupInfo( + server=server, + base_directory=base_directory, + backup_id=None, + info_file=infofile.strpath, + ) + assert obj.base_directory == base_directory + assert obj.filename == infofile.strpath + assert isinstance(obj, SyntheticBackupInfo) + + def test_get_basebackup_directory(self): + """ + Unit test for the get_basebackup_directory. + + Create mock server and a SyntheticBackupInfo object. + + This unit tests checks if the method returns the correct path based on + base_directory and backup_id. + """ + server = build_mocked_server() + backup_info = SyntheticBackupInfo( + server=server, base_directory="/fake/path/", backup_id="fake_name" + ) + directory = backup_info.get_basebackup_directory() + assert directory == "/fake/path/fake_name" diff --git a/tests/test_output.py b/tests/test_output.py index 26bad140b..203e4806c 100644 --- a/tests/test_output.py +++ b/tests/test_output.py @@ -19,9 +19,10 @@ import json import mock +from mock import PropertyMock import pytest -from datetime import datetime +import datetime from dateutil import tz from barman import output @@ -30,8 +31,13 @@ GcpSnapshotsInfo, ) from barman.infofile import BackupInfo -from barman.utils import BarmanEncoder, pretty_size -from testing_helpers import build_test_backup_info, find_by_attr, mock_backup_ext_info +from barman.utils import BarmanEncoder, human_readable_timedelta, pretty_size +from testing_helpers import ( + build_backup_manager, + build_test_backup_info, + find_by_attr, + mock_backup_ext_info, +) # Color output constants RED = "\033[31m" @@ -623,6 +629,10 @@ def test_close_and_exit_with_error(self, exit_mock): # noinspection PyMethodMayBeStatic class TestConsoleWriter(object): + row = " {:<23}: {}" + header_row = " {}:" + nested_row = " {:<21}: {}" + def test_debug(self, capsys): writer = output.ConsoleOutputWriter(debug=True) @@ -1144,8 +1154,6 @@ def test_result_list_backup(self, capsys): assert bi.server_name in out assert bi.backup_id in out assert str(bi.end_time.ctime()) in out - for name, _, location in bi.tablespaces: - assert "%s:%s" % (name, location) assert "Size: " + pretty_size(backup_size) in out assert "WAL Size: " + pretty_size(wal_size) in out assert err == "" @@ -1157,9 +1165,7 @@ def test_result_list_backup(self, capsys): writer.close() (out, err) = capsys.readouterr() assert not writer.minimal - assert bi.server_name in out - assert bi.backup_id in out - assert bi.status in out + assert "%s %s - S - %s" % (bi.server_name, bi.backup_id, bi.status) in out def test_result_list_backup_with_backup_name(self, capsys): # GIVEN a backup info with a backup_name @@ -1180,17 +1186,39 @@ def test_result_list_backup_with_backup_name(self, capsys): out, _err = capsys.readouterr() assert "%s %s '%s'" % (bi.server_name, bi.backup_id, bi.backup_name) in out - def test_result_show_backup(self, capsys): - # mock the backup ext info - wal_per_second = 0.01 - ext_info = mock_backup_ext_info( - children_timelines=(mock.Mock(tli="1"),), - copy_stats={"analysis_time": 2, "copy_time": 1}, + def test_result_show_backup_any_mode(self, capsys): + """ + Unit test for the show-backup command that display information in the + console that are common between any type of backup method. + + This unit tests checks if the fields common to any backup mode/method are + correctly rendered to the standard output. + + :param capsys: mock fixture for stdout/stderr + """ + backup_manager = build_backup_manager( + main_conf={"backup_options": "concurrent_backup"} + ) + backup_info = build_test_backup_info( + server=backup_manager.server, + backup_id="12345", + summarize_wal="on", + cluster_size=2048, deduplicated_size=1234, - status=BackupInfo.DONE, systemid="systemid", + data_checksums="on", + ) + + wal_per_second = 0.01 + ext_info = mock_backup_ext_info( + backup_info=backup_info, + children_timelines=(mock.Mock(tli="1"), mock.Mock(tli="2")), wal_until_next_compression_ratio=1.5, wals_per_second=wal_per_second, + est_dedup_size=1024, + deduplication_ratio=0.99, + wal_rate=wal_per_second * 3600, + wal_compression_ratio=0.5, ) writer = output.ConsoleOutputWriter() @@ -1199,18 +1227,230 @@ def test_result_show_backup(self, capsys): writer.result_show_backup(ext_info) writer.close() (out, err) = capsys.readouterr() - assert ext_info["server_name"] in out - assert ext_info["backup_id"] in out - assert ext_info["status"] in out - assert str(ext_info["end_time"]) in out - assert ext_info["systemid"] in out + assert f'Backup {ext_info["backup_id"]}' in out + + # Header rows + header_rows = [ + "Server information", + "Tablespaces", + "Base backup information", + "WAL information", + "Catalog information", + ] + for h_row in header_rows: + assert TestConsoleWriter.header_row.format(h_row) in out + + # Rows + rows = [ + ("Server Name", ext_info["server_name"]), + ("System Id", ext_info["systemid"]), + ("Status", ext_info["status"]), + ("PostgreSQL Version", ext_info["version"]), + ("PGDATA directory", ext_info["pgdata"]), + ("Estimated Cluster Size", pretty_size(ext_info["cluster_size"])), + ] + + for _row in rows: + assert TestConsoleWriter.row.format(_row[0], _row[1]) in out + + backup_size_output = "{} ({} with WALs)".format( + pretty_size(ext_info["deduplicated_size"]), + pretty_size(ext_info["deduplicated_size"] + ext_info["wal_size"]), + ) + wal_compression_output = "{percent:.2%}".format( + percent=ext_info["wal_compression_ratio"] + ) + compression_rate_output = "{percent:.2%}".format( + percent=ext_info["wal_until_next_compression_ratio"] + ) + + # Nested rows + nested_rows = [ + ("Checksums", ext_info["data_checksums"]), + ("WAL summarizer", ext_info["summarize_wal"]), + ("Backup Method", ext_info["mode"]), + ("Backup Size", backup_size_output), + ("WAL Size", pretty_size(ext_info["wal_size"])), + ("Timeline", str(ext_info["timeline"])), + ("Begin WAL", ext_info["begin_wal"]), + ("End WAL", ext_info["end_wal"]), + ("WAL number", ext_info["wal_num"]), + ("WAL compression ratio", wal_compression_output), + ("Begin time", str(ext_info["begin_time"])), + ("End time", str(ext_info["end_time"])), + ("Begin Offset", str(ext_info["begin_offset"])), + ("End Offset", str(ext_info["end_offset"])), + ("Begin LSN", str(ext_info["begin_xlog"])), + ("End LSN", str(ext_info["end_xlog"])), + ("No of files", ext_info["wal_until_next_num"]), + ("Disk usage", pretty_size(ext_info["wal_until_next_size"])), + ("WAL rate", "%0.2f/hour" % (wal_per_second * 3600)), + ("Compression ratio", compression_rate_output), + ("Last available", ext_info["wal_last"]), + ("Reachable timelines", "1, 2"), + ("Retention Policy", "not enforced"), + ("Previous Backup", "- (this is the oldest base backup)"), + ("Next Backup", "- (this is the latest base backup)"), + ] for name, _, location in ext_info["tablespaces"]: - assert "{:<21}: {}".format(name, location) in out - assert (pretty_size(ext_info["size"] + ext_info["wal_size"])) in out - assert (pretty_size(ext_info["deduplicated_size"])) in out - assert (pretty_size(ext_info["wal_until_next_size"])) in out - assert "WAL rate : %0.2f/hour" % (wal_per_second * 3600) in out - # TODO: this test can be expanded + nested_rows.append((name, location)) + + for n_row in nested_rows: + assert TestConsoleWriter.nested_row.format(n_row[0], n_row[1]) in out + assert err == "" + + @pytest.mark.parametrize( + ("backup_method", "backup_type", "parent_backup_id", "chidren_backup_ids"), + [ + ("postgres", "full", None, ["1234", "123456"]), + ("postgres", "incremental", 12345, ["1234", "123456"]), + ("rsync", None, None, None), + ], + ) + def test_result_show_backup_specific_fields_by_backup_type( + self, backup_method, backup_type, parent_backup_id, chidren_backup_ids, capsys + ): + """ + Unit test for the show-backup command that display specific information + in the console for each type of the following backups: 'rsync', + 'incremental' and 'full'. + + This unit tests checks if the fields that are related to those types of + backups are correctly rendered to the standard output. + + :param backup_method: backup_method mock parameter + :param backup_type: backup_type mock parameter + :param parent_backup_id: parent_backup_id mock parameter + :param chidren_backup_ids: chidren_backup_ids mock parameter + :param capsys: mock fixture for stdout/stderr + """ + backup_manager = build_backup_manager( + main_conf={ + "backup_method": backup_method, + "backup_options": "concurrent_backup", + } + ) + backup_info = build_test_backup_info( + server=backup_manager.server, + backup_id="1", + summarize_wal="on", + cluster_size=2048, + deduplicated_size=1234, + systemid="systemid", + parent_backup_id=parent_backup_id, + children_backup_ids=chidren_backup_ids, + ) + + wal_per_second = 0.01 + ext_info = mock_backup_ext_info( + backup_info=backup_info, + root_backup_id="root", + chain_size="2", + est_dedup_size=1024, + deduplication_ratio=0.99, + wal_rate=wal_per_second * 3600, + backup_type=backup_type, + ) + + writer = output.ConsoleOutputWriter() + + writer.result_show_backup(ext_info) + writer.close() + (out, err) = capsys.readouterr() + + resources_saved_output = "{} ({})".format( + pretty_size(ext_info["est_dedup_size"]), + "{percent:.2%}".format(percent=ext_info["deduplication_ratio"]), + ) + + # Output for both 'rsync' and 'incremental' backup type + if ext_info["backup_type"] in ("rsync", "incremental"): + assert ( + TestConsoleWriter.nested_row.format( + "Resources saved", resources_saved_output + ) + in out + ) + # Output for 'postgres' backup method + if ext_info["mode"] == "postgres": + # Output for 'full' and 'incremental' backup type + full = [ + ("Backup Type", ext_info["backup_type"]), + ("Children Backup(s)", ext_info["children_backup_ids"]), + ] + for n_row in full: + assert TestConsoleWriter.nested_row.format(n_row[0], n_row[1]) in out + + # Output only for 'incremental' backup type + incremental = [ + ("Root Backup", ext_info["root_backup_id"]), + ("Parent Backup", ext_info["parent_backup_id"]), + ("Backup chain size", ext_info["chain_size"]), + ] + if ext_info["backup_type"] == "incremental": + for n_row in incremental: + assert ( + TestConsoleWriter.nested_row.format(n_row[0], n_row[1]) in out + ) + assert err == "" + + def test_result_show_backup_any_mode_copy_stats(self, capsys): + """ + Unit test for the show-backup command that display information in the + console about copy statistics. + + This unit tests checks if the copy statistics are correctly rendered + to the standard output. + + :param capsys: mock fixture for stdout/stderr + """ + backup_manager = build_backup_manager( + main_conf={"backup_options": "concurrent_backup"} + ) + backup_info = build_test_backup_info( + server=backup_manager.server, + backup_id="12345", + cluster_size=2048, + copy_stats={"analysis_time": 2, "copy_time": 1, "number_of_workers": 1}, + deduplicated_size=1234, + systemid="systemid", + ) + ext_info = mock_backup_ext_info( + backup_info=backup_info, + est_dedup_size=1024, + deduplication_ratio=0.99, + copy_time=backup_info.copy_stats["copy_time"], + analysis_time=backup_info.copy_stats["analysis_time"], + number_of_workers=backup_info.copy_stats["number_of_workers"], + estimated_throughput=1, + ) + + writer = output.ConsoleOutputWriter() + + writer.result_show_backup(ext_info) + writer.close() + (out, err) = capsys.readouterr() + + copy_time_output = human_readable_timedelta( + datetime.timedelta(seconds=ext_info["copy_time"]) + ) + copy_time_output += " + {} startup".format( + human_readable_timedelta( + datetime.timedelta(seconds=ext_info["analysis_time"]) + ) + ) + assert TestConsoleWriter.nested_row.format("Copy time", copy_time_output) in out + est_throughput_output = "{}/s".format( + pretty_size(ext_info["estimated_throughput"]) + ) + if "number_of_workers" in ext_info and ext_info["number_of_workers"] > 1: + est_throughput_output += " (%s jobs)" % ext_info["number_of_workers"] + assert ( + TestConsoleWriter.nested_row.format( + "Estimated throughput", est_throughput_output + ) + in out + ) assert err == "" def test_result_show_backup_with_backup_name(self, capsys): @@ -1219,6 +1459,8 @@ def test_result_show_backup_with_backup_name(self, capsys): backup_name="named backup", status=BackupInfo.DONE, wals_per_second=0.1, + est_dedup_size=1024, + deduplication_ratio=0.5, ) # WHEN the list_backup output is generated in Plain form @@ -1257,6 +1499,9 @@ def test_result_show_backup_with_snapshots_info_gcp(self, capsys): snapshots_info=snapshots_info, status=BackupInfo.DONE, wals_per_second=0.1, + cluster_size=2048, + est_dedup_size=1024, + deduplication_ratio=0.5, ) # WHEN the show output is generated in Plain form @@ -1370,9 +1615,9 @@ def test_readact_passwords_in_json(self, capsys): # noinspection PyMethodMayBeStatic class TestJsonWriter(object): # Fixed start and end timestamps for backup/recovery timestamps - begin_time = datetime(2022, 7, 4, 9, 15, 35, tzinfo=tz.tzutc()) + begin_time = datetime.datetime(2022, 7, 4, 9, 15, 35, tzinfo=tz.tzutc()) begin_epoch = "1656926135" - end_time = datetime(2022, 7, 4, 9, 22, 37, tzinfo=tz.tzutc()) + end_time = datetime.datetime(2022, 7, 4, 9, 22, 37, tzinfo=tz.tzutc()) end_epoch = "1656926557" def test_debug(self, capsys): @@ -1816,10 +2061,6 @@ def test_result_list_backup(self, capsys): assert bi.backup_id == backup["backup_id"] assert str(bi.end_time.ctime()) == backup["end_time"] assert self.end_epoch == backup["end_time_timestamp"] - for name, _, location in bi.tablespaces: - tablespace = find_by_attr(backup["tablespaces"], "name", name) - assert name == tablespace["name"] - assert location == tablespace["location"] assert pretty_size(backup_size) == backup["size"] assert pretty_size(wal_size) == backup["wal_size"] assert err == "" @@ -1863,15 +2104,86 @@ def test_result_list_backup_with_backup_name(self, capsys): assert json_output[bi.server_name][0]["backup_id"] == bi.backup_id assert json_output[bi.server_name][0]["backup_name"] == bi.backup_name + def test_result_list_backup_types(self, capsys): + # GIVEN a backup info with specific backup types + backup_types = ["rsync", "incremental", "full", "snapshot"] + backup_size = 12345 + wal_size = 54321 + retention_status = "test status" + + for backup_type in backup_types: + bi = build_test_backup_info( + server_name="test_server", + backup_id="test_backup_id", + status="DONE", + ) + + # Mock the backup_type property + type_mock = PropertyMock(return_value=backup_type) + type(bi).backup_type = type_mock + + # WHEN the list_backup output is generated + json_writer = output.JsonOutputWriter() + json_writer.init_list_backup(bi.server_name, False) + json_writer.result_list_backup(bi, backup_size, wal_size, retention_status) + json_writer.close() + + # Capture the output + out, err = capsys.readouterr() + + # THEN the JSON output contains the correct backup type + json_output = json.loads(out) + backup = find_by_attr( + json_output[bi.server_name], "backup_id", bi.backup_id + ) + assert backup["backup_type"] == backup_type + assert err == "" + type_mock.assert_called_once() + @mock.patch.dict("os.environ", {"TZ": "US/Eastern"}) def test_result_show_backup(self, capsys): - # mock the backup ext info + """ + Unit test for the show-backup command that display information about a + backup with a json format. + + This unit tests checks if all fields of a backup are correctly in + the result dict to be rendered as json. + + :param capsys: mock fixture for stdout/stderr + """ + backup_manager = build_backup_manager( + main_conf={"backup_options": "concurrent_backup"} + ) + backup_info = build_test_backup_info( + server=backup_manager.server, + backup_id="12345", + summarize_wal="on", + cluster_size=2048, + deduplicated_size=1234, + systemid="systemid", + data_checksums="on", + begin_time=self.begin_time, + end_time=self.end_time, + ) + wal_per_second = 0.01 ext_info = mock_backup_ext_info( - status=BackupInfo.DONE, + backup_info=backup_info, + children_timelines=(mock.Mock(tli="1"), mock.Mock(tli="2")), + wal_until_next_compression_ratio=1.5, wals_per_second=wal_per_second, - begin_time=self.begin_time, - end_time=self.end_time, + est_dedup_size=1024, + deduplication_ratio=0.99, + wal_rate=wal_per_second * 3600, + wal_compression_ratio=0.5, + copy_time=1, + analysis_time=2, + number_of_workers=1, + backup_type="incremental", + root_backup_id="1234", + retention_policy_status="VALID", + previous_backup_id="12345", + next_backup_id="123456", ) server_name = ext_info["server_name"] @@ -1884,14 +2196,108 @@ def test_result_show_backup(self, capsys): base_information = json_output[server_name]["base_backup_information"] wal_information = json_output[server_name]["wal_information"] + server_information = json_output[server_name]["server_information"] + catalog_information = json_output[server_name]["catalog_information"] assert server_name in json_output assert ext_info["backup_id"] == json_output[server_name]["backup_id"] assert ext_info["status"] == json_output[server_name]["status"] + assert ext_info["systemid"] == json_output[server_name]["system_id"] + assert ext_info["version"] == json_output[server_name]["postgresql_version"] + assert ext_info["pgdata"] == json_output[server_name]["pgdata_directory"] + assert ( + pretty_size(ext_info["cluster_size"]) + == json_output[server_name]["cluster_size"] + ) + assert ( + ext_info["cluster_size"] == json_output[server_name]["cluster_size_bytes"] + ) + + assert ext_info["data_checksums"] == server_information["data_checksums"] + assert ext_info["summarize_wal"] == server_information["summarize_wal"] + + assert ext_info["mode"] == base_information["backup_method"] + if ext_info["backup_type"] == "incremental": + assert ext_info["root_backup_id"] == catalog_information["root_backup_id"] + assert ( + ext_info["parent_backup_id"] == catalog_information["parent_backup_id"] + ) + assert ext_info["chain_size"] == catalog_information["chain_size"] + if ext_info["mode"] == "postgres": + assert ext_info["backup_type"] == base_information["backup_type"] + assert ( + ext_info["children_backup_ids"] + == catalog_information["children_backup_ids"] + ) + + assert ( + pretty_size(ext_info["deduplicated_size"]) + == base_information["backup_size"] + ) + assert ext_info["deduplicated_size"] == base_information["backup_size_bytes"] + assert ( + pretty_size(ext_info["deduplicated_size"] + ext_info["wal_size"]) + == base_information["backup_size_with_wals"] + ) + assert ( + ext_info["deduplicated_size"] + ext_info["wal_size"] + == base_information["backup_size_with_wals_bytes"] + ) + assert pretty_size(ext_info["wal_size"]) == base_information["wal_size"] + assert ext_info["wal_size"] == base_information["wal_size_bytes"] + + assert ( + pretty_size(ext_info["est_dedup_size"]) + == base_information["resources_saved"] + ) + assert ext_info["est_dedup_size"] == base_information["resources_saved_bytes"] + resources_saved_percentage = "{percent:.2%}".format( + percent=ext_info["deduplication_ratio"] + ) + assert ( + resources_saved_percentage == base_information["resources_saved_percentage"] + ) + + assert ext_info["timeline"] == base_information["timeline"] + assert ext_info["begin_wal"] == base_information["begin_wal"] + assert ext_info["end_wal"] == base_information["end_wal"] + assert ext_info["wal_num"] == base_information["wal_num"] + + wal_compression_ratio = "{percent:.2%}".format( + percent=ext_info["wal_compression_ratio"] + ) + assert wal_compression_ratio == base_information["wal_compression_ratio"] + + assert str(ext_info["begin_time"]) == base_information["begin_time"] assert str(ext_info["end_time"]) == base_information["end_time"] + assert self.end_epoch == base_information["end_time_timestamp"] assert self.begin_epoch == base_information["begin_time_timestamp"] + copy_time = human_readable_timedelta( + datetime.timedelta(seconds=ext_info["copy_time"]) + ) + analysis_time = human_readable_timedelta( + datetime.timedelta(seconds=ext_info["analysis_time"]) + ) + assert copy_time == base_information["copy_time"] + assert analysis_time == base_information["analysis_time"] + assert ext_info["copy_time"] == base_information["copy_time_seconds"] + throughput_output = "%s/s" % pretty_size( + ext_info["deduplicated_size"] / ext_info["copy_time"] + ) + assert throughput_output == base_information["throughput"] + assert ( + ext_info["deduplicated_size"] / ext_info["copy_time"] + == base_information["throughput_bytes"] + ) + assert ext_info["number_of_workers"] == base_information["number_of_workers"] + + assert ext_info["begin_offset"] == base_information["begin_offset"] + assert ext_info["end_offset"] == base_information["end_offset"] + assert ext_info["begin_xlog"] == base_information["begin_lsn"] + assert ext_info["end_xlog"] == base_information["end_lsn"] + for name, _, location in ext_info["tablespaces"]: tablespace = find_by_attr( json_output[server_name]["tablespaces"], "name", name @@ -1899,24 +2305,49 @@ def test_result_show_backup(self, capsys): assert name == tablespace["name"] assert location == tablespace["location"] + assert ext_info["wal_until_next_num"] == wal_information["no_of_files"] assert ( - pretty_size(ext_info["size"] + ext_info["wal_size"]) - ) == base_information["disk_usage_with_wals"] - assert (pretty_size(ext_info["wal_until_next_size"])) == wal_information[ - "disk_usage" - ] - assert "%0.2f/hour" % (wal_per_second * 3600) == wal_information["wal_rate"] + pretty_size(ext_info["wal_until_next_size"]) + == wal_information["disk_usage"] + ) + assert ext_info["wal_until_next_size"] == wal_information["disk_usage_bytes"] + + wal_rate = "%0.2f/hour" % (ext_info["wals_per_second"] * 3600) + assert wal_rate == wal_information["wal_rate"] + assert ext_info["wals_per_second"] == wal_information["wal_rate_per_second"] + + compression_ratio = "{percent:.2%}".format( + percent=ext_info["wal_until_next_compression_ratio"] + ) + assert compression_ratio == wal_information["compression_ratio"] + assert ext_info["wal_last"] == wal_information["last_available"] + + assert ( + ext_info["retention_policy_status"] + == catalog_information["retention_policy"] + ) + assert ext_info["previous_backup_id"] == catalog_information["previous_backup"] + assert ext_info["next_backup_id"] == catalog_information["next_backup"] assert err == "" def test_result_show_backup_with_backup_name(self, capsys): - # GIVEN a backup info with a backup_name - ext_info = mock_backup_ext_info( + backup_manager = build_backup_manager( + main_conf={"backup_options": "concurrent_backup"} + ) + backup_info = build_test_backup_info( backup_name="named backup", - status=BackupInfo.DONE, - wals_per_second=0.1, - begin_time=self.begin_time, - end_time=self.end_time, + server=backup_manager.server, + cluster_size=2048, + deduplicated_size=1234, + ) + + wal_per_second = 0.01 + ext_info = mock_backup_ext_info( + backup_info=backup_info, + est_dedup_size=1024, + deduplication_ratio=0.99, + wal_rate=wal_per_second * 3600, ) # WHEN the list_backup output is generated in JSON form @@ -1958,10 +2389,22 @@ def test_result_show_backup_with_snapshots_info(self, capsys): ), ], ) - ext_info = mock_backup_ext_info( + backup_manager = build_backup_manager( + main_conf={"backup_options": "concurrent_backup"} + ) + backup_info = build_test_backup_info( + server=backup_manager.server, + cluster_size=2048, + deduplicated_size=1234, snapshots_info=snapshots_info, - status=BackupInfo.DONE, - wals_per_second=0.1, + ) + + wal_per_second = 0.01 + ext_info = mock_backup_ext_info( + backup_info=backup_info, + est_dedup_size=1024, + deduplication_ratio=0.99, + wal_rate=wal_per_second * 3600, ) # WHEN the show backup output is generated in JSON form diff --git a/tests/test_postgres.py b/tests/test_postgres.py index 65236ca4a..0384607ab 100644 --- a/tests/test_postgres.py +++ b/tests/test_postgres.py @@ -17,7 +17,9 @@ # along with Barman. If not, see . import datetime +import time from multiprocessing import Queue +from unittest.mock import MagicMock try: from queue import Queue as SyncQueue @@ -31,6 +33,7 @@ from barman.exceptions import ( PostgresConnectionError, + PostgresConnectionLost, PostgresDuplicateReplicationSlot, PostgresException, PostgresInvalidReplicationSlot, @@ -42,6 +45,7 @@ ) from barman.postgres import ( PostgreSQLConnection, + PostgresKeepAlive, StandbyPostgreSQLConnection, PostgreSQL, ) @@ -1088,7 +1092,7 @@ def test_has_checkpoint_privileges( cursor_mock.fetchone.side_effect = [(False,)] assert not server.postgres.has_checkpoint_privileges cursor_mock.execute.assert_called_with( - "select pg_has_role(CURRENT_USER ,'pg_checkpoint', 'MEMBER');" + "select pg_has_role(CURRENT_USER ,'pg_checkpoint', 'USAGE');" ) # no superuser, pg_checkpoint -> True @@ -1097,7 +1101,7 @@ def test_has_checkpoint_privileges( cursor_mock.fetchone.side_effect = [(True,)] assert server.postgres.has_checkpoint_privileges cursor_mock.execute.assert_called_with( - "select pg_has_role(CURRENT_USER ,'pg_checkpoint', 'MEMBER');" + "select pg_has_role(CURRENT_USER ,'pg_checkpoint', 'USAGE');" ) # superuser, no pg_checkpoint -> True @@ -1714,11 +1718,11 @@ def test_has_monitoring_privileges( """ SELECT ( - pg_has_role(CURRENT_USER, 'pg_monitor', 'MEMBER') + pg_has_role(CURRENT_USER, 'pg_monitor', 'USAGE') OR ( - pg_has_role(CURRENT_USER, 'pg_read_all_settings', 'MEMBER') - AND pg_has_role(CURRENT_USER, 'pg_read_all_stats', 'MEMBER') + pg_has_role(CURRENT_USER, 'pg_read_all_settings', 'USAGE') + AND pg_has_role(CURRENT_USER, 'pg_read_all_stats', 'USAGE') ) ) """ @@ -1748,6 +1752,104 @@ def test_has_monitoring_privileges_exception(self, conn_mock, mock_is_superuser) # THEN a None value was returned assert has_monitoring is None + @pytest.mark.parametrize( + ("backup_privilege", "expected", "should_fetchone"), + [(False, None, False), (True, 2048, True)], + ) + @patch("barman.postgres.PostgreSQLConnection.connect") + @patch( + "barman.postgres.PostgreSQLConnection.has_backup_privileges", + new_callable=PropertyMock, + ) + def test_current_size( + self, + has_backup_privileges_mock, + conn_mock, + backup_privilege, + expected, + should_fetchone, + ): + """ + Test the current_size method + """ + any_size = 2048 + + has_backup_privileges_mock.return_value = backup_privilege + + # Build a server + server = build_real_server() + cursor_mock = conn_mock.return_value.cursor.return_value + cursor_mock.fetchone.side_effect = [[any_size]] + + result = server.postgres.current_size + assert result == expected + if should_fetchone: + cursor_mock.execute.assert_called_once_with( + "SELECT sum(pg_tablespace_size(oid)) FROM pg_tablespace" + ) + else: + cursor_mock.assert_not_called() + + @pytest.mark.parametrize( + "expected_error", (psycopg2.Error, PostgresConnectionError) + ) + @patch("barman.postgres._logger.debug") + @patch("barman.postgres.PostgreSQLConnection.connect") + @patch( + "barman.postgres.PostgreSQLConnection.has_backup_privileges", + new_callable=PropertyMock, + ) + def test_current_size_error( + self, has_backup_privileges_mock, conn_mock, logger_mock, expected_error + ): + """ + Test the current_size method + """ + has_backup_privileges_mock.return_value = True + + # Build a server + server = build_real_server() + cursor_mock = conn_mock.return_value.cursor.return_value + cursor_mock.fetchone.side_effect = expected_error + + result = server.postgres.current_size + logger_mock.assert_called_once_with( + "Error retrieving PostgreSQL total size: %s", "" + ) + assert result is None + + @patch("barman.postgres._logger.debug") + def test_send_heartbeat_query(self, logger_mock): + """Test sending a heartbeat query to the current connection""" + server = build_real_server() + # Mock an already opened connection + conn_mock = MagicMock() + server.postgres._conn = conn_mock + # Mock the cursor to be used + cursor_mock = MagicMock() + conn_mock.cursor.return_value.__enter__.return_value = cursor_mock + + # Case 1: the connection is working and the query executes successfully + result, ex = server.postgres.send_heartbeat_query() + cursor_mock.execute.assert_called_once_with("SELECT 1") + logger_mock.assert_called_once_with( + "Sent heartbeat query to maintain the current connection" + ) + assert result is True and ex is None + + cursor_mock.reset_mock() + logger_mock.reset_mock() + + # Case 2: a database error occurred when executing the query + ex_to_raise = psycopg2.DatabaseError("some database error") + cursor_mock.execute.side_effect = ex_to_raise + result, ex = server.postgres.send_heartbeat_query() + cursor_mock.execute.assert_called_once_with("SELECT 1") + logger_mock.assert_called_once_with( + "Failed to execute heartbeat query on the current connection: some database error" + ) + assert result is False and ex is ex_to_raise + # noinspection PyMethodMayBeStatic class TestStreamingConnection(object): @@ -2127,3 +2229,154 @@ def test_stop_backup( # AND mock_done_q.put is called exactly once with the argument True mock_done_q.put.assert_called_once_with(True) + + +class TestPostgresKeepAlive(object): + """Test the PostgresKeepAlive class""" + + def _something_that_takes_some_time(self): + time.sleep(0.1) + + def _something_that_takes_a_bunch_of_time(self): + time.sleep(1) + + @patch("barman.postgres.PostgreSQLConnection") + def test_keepalive_runs_successfully(self, postgres_mock): + """Basic test. Ensures the keep-alive works in the ideal scenario""" + # Mock a Postgres with a connection already opened + postgres_mock.has_connection = True + postgres_mock.send_heartbeat_query.return_value = True, None + keepalive = PostgresKeepAlive(postgres_mock, 1) + keepalive._thread = Mock(wraps=keepalive._thread) + keepalive._stop_thread = Mock(wraps=keepalive._stop_thread) + with keepalive: + # Simulate something that takes a while, just so it allows some context switch + self._something_that_takes_some_time() + # The thread has started + keepalive._thread.start.assert_called_once() + assert keepalive._thread.is_alive() is True + # It sent a heartbeat query + postgres_mock.send_heartbeat_query.assert_called() + # It slept after the query + keepalive._stop_thread.wait.assert_called_with(1) + # The context manager exited, the thread should now be terminated + # The stop-thread event has been set, the thread stopped doing its work + assert keepalive._stop_thread.is_set() + # The thread is indeed terminated + assert keepalive._thread.is_alive() is False + # The main thread joined and can continue its work alone from now on + keepalive._thread.join.assert_called_once() + + @patch("barman.postgres.PostgreSQLConnection") + def test_do_nothing_if_interval_less_or_equal_zero(self, postgres_mock): + """Ensures nothing happens if an interval <= ``0`` is given""" + postgres_mock.has_connection = True + postgres_mock.send_heartbeat_query.return_value = True, None + keepalive = PostgresKeepAlive(postgres_mock, 0) + keepalive._thread = Mock(wraps=keepalive._thread) + with keepalive: + self._something_that_takes_some_time() + # The thread was never started + keepalive._thread.start.assert_not_called() + assert keepalive._thread.is_alive() is False + # No queries were sent by the keep-alive + postgres_mock.send_heartbeat_query.assert_not_called() + + @patch("barman.postgres.PostgreSQLConnection") + def test_wait_connection_open_to_send_queries(self, postgres_mock): + """Ensures the keep-alive only starts when the connection is opnened on the main thread""" + postgres_mock.has_connection = False + postgres_mock.send_heartbeat_query.return_value = True, None + keepalive = PostgresKeepAlive(postgres_mock, 1) + keepalive._thread = Mock(wraps=keepalive._thread) + with keepalive: + self._something_that_takes_some_time() + # The thread has started but it's waiting for the connection to + # be opened so no query is executed yet + keepalive._thread.start.assert_called_once() + assert keepalive._thread.is_alive() is True + postgres_mock.send_heartbeat_query.assert_not_called() + # The connection opens and now it should start sending queries + postgres_mock.has_connection = True + # we simulate something that takes more time here, since the keep-alive sleeps + # for 1 second before checking the connection again + self._something_that_takes_a_bunch_of_time() + postgres_mock.send_heartbeat_query.assert_called() + + @patch("barman.postgres.PostgreSQLConnection") + def test_raise_exception_when_needed(self, postgres_mock): + """Ensures the keep-alive raises exceptions only when set to do so""" + # Mock a Postgres with a connection that fails in the first query + postgres_mock.has_connection = True + postgres_mock.send_heartbeat_query.return_value = ( + False, + psycopg2.OperationalError("some connection error"), + ) + # Case 1: Raises an exception when set do so + keepalive = PostgresKeepAlive(postgres_mock, 1, True) + keepalive._thread = Mock(wraps=keepalive._thread) + with pytest.raises(PostgresConnectionLost): + with keepalive: + self._something_that_takes_some_time() + keepalive._thread.start.assert_called_once() + assert keepalive._thread.is_alive() is True + # it sent a heartbeat query, but failed, the exception is raised + postgres_mock.send_heartbeat_query.assert_called() + + # Case 2: Don't raise any exceptions if not set. In this case, it will just + # ignore the errors and keep executing until the context exits + keepalive = PostgresKeepAlive(postgres_mock, 1) + keepalive._thread = Mock(wraps=keepalive._thread) + with keepalive: + self._something_that_takes_some_time() + keepalive._thread.start.assert_called_once() + assert keepalive._thread.is_alive() is True + # It sent a heartbeat query, but failed, no exception is raised + postgres_mock.send_heartbeat_query.assert_called() + + @patch("barman.postgres.PostgreSQLConnection") + def test_thread_is_always_terminated(self, postgres_mock): + """Ensures the keep-alive thread is stopped in all cases""" + postgres_mock.has_connection = True + postgres_mock.send_heartbeat_query.return_value = True, None + # Case 1: A normal execution, the stop-thread event is set when exiting + # the context manager and the thread is terminated + keepalive = PostgresKeepAlive(postgres_mock, 1) + keepalive._thread = Mock(wraps=keepalive._thread) + with keepalive: + self._something_that_takes_some_time() + keepalive._thread.start.assert_called_once() + assert keepalive._thread.is_alive() is True + assert keepalive._stop_thread.is_set() + assert keepalive._thread.is_alive() is False + + # Case 2: If something goes wrong on main, the thread should also + # stop its work as soon as the context manager exits + keepalive = PostgresKeepAlive(postgres_mock, 1) + keepalive._thread = Mock(wraps=keepalive._thread) + try: + with keepalive: + self._something_that_takes_some_time() + keepalive._thread.start.assert_called_once() + assert keepalive._thread.is_alive() is True + raise Exception # oops! something went wrong here + except Exception: + pass + assert keepalive._stop_thread.is_set() + assert keepalive._thread.is_alive() is False + + # Case 3: When the keep-alive itself raises an exception it should + # also stop itself immediately + postgres_mock.send_heartbeat_query.return_value = ( + False, + psycopg2.OperationalError("some connection error"), + ) + keepalive = PostgresKeepAlive(postgres_mock, 1, True) + keepalive._thread = Mock(wraps=keepalive._thread) + with pytest.raises(PostgresConnectionLost): + with keepalive: + self._something_that_takes_some_time() + keepalive._thread.start.assert_called_once() + assert keepalive._thread.is_alive() is True + assert keepalive._stop_thread.is_set() + assert keepalive._thread.is_alive() is False diff --git a/tests/test_recovery_executor.py b/tests/test_recovery_executor.py index b7f01dc54..0873684b4 100644 --- a/tests/test_recovery_executor.py +++ b/tests/test_recovery_executor.py @@ -19,13 +19,13 @@ from functools import partial import os import shutil -import time from contextlib import closing import dateutil +import dateutil.tz import mock import pytest -from mock import MagicMock +from mock import MagicMock, Mock, call import testing_helpers from barman import xlog @@ -38,7 +38,11 @@ RecoveryTargetActionException, SnapshotBackupException, ) -from barman.infofile import BackupInfo, WalFileInfo +from barman.infofile import ( + BackupInfo, + WalFileInfo, + SyntheticBackupInfo, +) from barman.recovery_executor import ( Assertion, RecoveryExecutor, @@ -47,6 +51,7 @@ TarballRecoveryExecutor, ConfigurationFileMangeler, recovery_executor_factory, + IncrementalRecoveryExecutor, ) @@ -389,7 +394,6 @@ def test_set_pitr_targets(self, tmpdir): recovery_info, backup_info, dest.strpath, "", "", "", "", "", False, None ) # Test with empty values (no PITR) - assert recovery_info["target_epoch"] is None assert recovery_info["target_datetime"] is None assert recovery_info["wal_dest"] == wal_dest.strpath @@ -407,12 +411,27 @@ def test_set_pitr_targets(self, tmpdir): None, ) target_datetime = dateutil.parser.parse("2015-06-03 16:11:03.710380+02:00") - target_epoch = time.mktime(target_datetime.timetuple()) + ( - target_datetime.microsecond / 1000000.0 + + assert recovery_info["target_datetime"] == target_datetime + assert recovery_info["wal_dest"] == dest.join("barman_wal").strpath + + # Test for PITR targets with implicit target time + executor._set_pitr_targets( + recovery_info, + backup_info, + dest.strpath, + "target_name", + "2015-06-03 16:11:03.71038", + "2", + None, + "", + False, + None, ) + target_datetime = dateutil.parser.parse("2015-06-03 16:11:03.710380") + target_datetime = target_datetime.replace(tzinfo=dateutil.tz.tzlocal()) assert recovery_info["target_datetime"] == target_datetime - assert recovery_info["target_epoch"] == target_epoch assert recovery_info["wal_dest"] == dest.join("barman_wal").strpath # Test for too early PITR target @@ -706,6 +725,7 @@ def test_generate_recovery_conf_pre12(self, rsync_pg_mock, tmpdir): "tempdir": tmpdir.strpath, "results": {"changes": [], "warnings": []}, "get_wal": False, + "target_datetime": "2015-06-03 16:11:03.71038+02", } backup_info = testing_helpers.build_test_backup_info() dest = tmpdir.mkdir("destination") @@ -721,7 +741,7 @@ def test_generate_recovery_conf_pre12(self, rsync_pg_mock, tmpdir): True, "remote@command", "target_name", - "2015-06-03 16:11:03.71038+02", + "2015-06-03 16:11:03.71038", "2", "", "", @@ -742,6 +762,9 @@ def test_generate_recovery_conf_pre12(self, rsync_pg_mock, tmpdir): assert "recovery_target_name" in recovery_conf assert "recovery_target" not in recovery_conf assert recovery_conf["recovery_end_command"] == "'rm -fr barman_wal'" + # what matters is the 'target_datetime', which always contain the target time + # with a time zone, even if the user specified no time zone through + # '--target-time'. assert recovery_conf["recovery_target_time"] == "'2015-06-03 16:11:03.71038+02'" assert recovery_conf["recovery_target_timeline"] == "2" assert recovery_conf["recovery_target_name"] == "'target_name'" @@ -859,6 +882,7 @@ def test_generate_recovery_conf(self, rsync_pg_mock, tmpdir): "tempdir": tmpdir.strpath, "results": {"changes": [], "warnings": []}, "get_wal": False, + "target_datetime": "2015-06-03 16:11:03.71038+02", } backup_info = testing_helpers.build_test_backup_info( version=120000, @@ -876,7 +900,7 @@ def test_generate_recovery_conf(self, rsync_pg_mock, tmpdir): True, "remote@command", "target_name", - "2015-06-03 16:11:03.71038+02", + "2015-06-03 16:11:03.71038", "2", "", "", @@ -900,6 +924,9 @@ def test_generate_recovery_conf(self, rsync_pg_mock, tmpdir): assert "recovery_target_name" in pg_auto_conf assert "recovery_target" in pg_auto_conf assert pg_auto_conf["recovery_end_command"] == "'rm -fr barman_wal'" + # what matters is the 'target_datetime', which always contain the target time + # with a time zone, even if the user specified no time zone through + # '--target-time'. assert pg_auto_conf["recovery_target_time"] == "'2015-06-03 16:11:03.71038+02'" assert pg_auto_conf["recovery_target_timeline"] == "2" assert pg_auto_conf["recovery_target_name"] == "'target_name'" @@ -1293,7 +1320,6 @@ def test_recovery( ), ], }, - "target_epoch": None, "configuration_files": ["postgresql.conf", "postgresql.auto.conf"], "target_datetime": None, "safe_horizon": None, @@ -1342,7 +1368,6 @@ def test_recovery( ), ], }, - "target_epoch": None, "configuration_files": ["postgresql.conf", "postgresql.auto.conf"], "target_datetime": None, "safe_horizon": None, @@ -1763,9 +1788,9 @@ def mock_resolve_mounted_volume(_cmd): attached_volumes["disk0"].mount_point = "/opt/disk0" attached_volumes["disk0"].mount_options = "rw,noatime" - attached_volumes[ - "disk0" - ].resolve_mounted_volume.side_effect = mock_resolve_mounted_volume + attached_volumes["disk0"].resolve_mounted_volume.side_effect = ( + mock_resolve_mounted_volume + ) mock_get_snapshot_interface.return_value.get_attached_volumes.return_value = ( attached_volumes ) @@ -2195,9 +2220,9 @@ def mock_resolve_mounted_volume(volume, mount_info, _cmd): # If resolved_mount_info should raise an exception then just set it as the # side effect if isinstance(resolved_mount_info, Exception): - attached_volumes[ - disk - ].resolve_mounted_volume.side_effect = resolved_mount_info + attached_volumes[disk].resolve_mounted_volume.side_effect = ( + resolved_mount_info + ) # Otherwise, create a partial which sets the mount point and options to the # values at the current index else: @@ -2244,25 +2269,40 @@ def mock_resolve_mounted_volume(volume, mount_info, _cmd): class TestRecoveryExecutorFactory(object): @pytest.mark.parametrize( - ("compression", "expected_executor", "snapshots_info", "should_error"), + ( + "compression", + "is_incremental", + "expected_executor", + "snapshots_info", + "should_error", + ), [ # No compression or snapshots_info should return RecoveryExecutor - (None, RecoveryExecutor, None, False), + (None, False, RecoveryExecutor, None, False), # Supported compression should return TarballRecoveryExecutor - ("gzip", TarballRecoveryExecutor, None, False), + ("gzip", False, TarballRecoveryExecutor, None, False), # Unrecognised compression should cause an error - ("snappy", None, None, True), + ("snappy", False, None, None, True), # A backup_info with snapshots_info should return SnapshotRecoveryExecutor - (None, SnapshotRecoveryExecutor, mock.Mock(), False), + (None, False, SnapshotRecoveryExecutor, mock.Mock(), False), + # A backup with a parent_backup_id should return IncrementalRecoveryExecutor + (None, True, IncrementalRecoveryExecutor, None, False), ], ) def test_recovery_executor_factory( - self, compression, expected_executor, snapshots_info, should_error + self, + compression, + is_incremental, + expected_executor, + snapshots_info, + should_error, ): mock_backup_manager = mock.Mock() mock_command = mock.Mock() mock_backup_info = mock.Mock( - compression=compression, snapshots_info=snapshots_info + compression=compression, + snapshots_info=snapshots_info, + is_incremental=is_incremental, ) # WHEN recovery_executor_factory is called with the specified compression @@ -2292,3 +2332,705 @@ def test_simple_file_mangeling(self, tmpdir): content = a_file.read() assert len(mangeled) == 1 assert "#BARMAN#recovery_target=something" in content + + +class TestIncrementalRecoveryExecutor(object): + """ + This class tests the methods of the :class:`IncrementalRecoveryExecutor` class. + """ + + @pytest.fixture + def server(self): + """ + Server mock fixture to be used in the tests below. + """ + backup_manager = mock.Mock() + backup_manager.get_keep_target.return_value = None + server = testing_helpers.build_mocked_server() + server.backup_manager = backup_manager + yield server + + @pytest.fixture + def executor(self): + """ + Executor mock fixture to be used in the tests below. + """ + backup_manager = testing_helpers.build_backup_manager() + executor = IncrementalRecoveryExecutor(backup_manager=backup_manager) + return executor + + @pytest.fixture + def synthetic_backup_info(self, server): + backup_info = SyntheticBackupInfo( + server=server, + base_directory="fake_path", + backup_id="backup_id", + version=170000, + ) + return backup_info + + @mock.patch("barman.recovery_executor.IncrementalRecoveryExecutor._combine_backups") + def test_recover(self, mock__combine_backups, executor, synthetic_backup_info): + """ + Unit test for the recover method. + + This unit test checks if the recover from the super class + is called with the required parameters. + + :param mock__combine_backups: _combine_backups method mock from + IncrementalRecoveryExecutor class + :param executor: executor mock fixture + :param synthetic_backup_info: synthetic_backup_info mock fixture + """ + mock_backup_info = Mock() + mock__combine_backups.return_value = synthetic_backup_info + executor.config.local_staging_path = "fake/staging/path" + + with mock.patch("barman.recovery_executor.RecoveryExecutor.recover") as mock_sr: + _ = executor.recover( + mock_backup_info, "fake/destination/path", remote_command=None + ) + + mock_sr.assert_called_once_with( + synthetic_backup_info, "fake/destination/path", remote_command=None + ) + + @mock.patch("barman.recovery_executor.PgCombineBackup") + @mock.patch( + "barman.recovery_executor.IncrementalRecoveryExecutor._get_backup_chain_paths" + ) + @mock.patch( + "barman.recovery_executor.IncrementalRecoveryExecutor._fetch_remote_status" + ) + @mock.patch( + "barman.recovery_executor.IncrementalRecoveryExecutor._prepare_destination" + ) + @mock.patch("barman.infofile.LocalBackupInfo.get_data_directory") + @mock.patch("barman.infofile.FieldListFile.load") + @mock.patch("barman.config.parse_staging_path") + def test__combine_backups( + self, + parse_local_staging_path, + mock_load_fields, + mock_get_data_dir, + mock__prepare_dest, + mock_fetch_remote_status, + mock_get_backup_chain_paths, + mock_pg_combinebackup, + executor, + server, + ): + """ + Unit test for the _combine_backups method. + + Create mock patches for the methods used inside _combine_backups. + + This unit tests checks if _prepare_destination and pg_combinebackup are + called with the correct parameters. + It also tests if the result is a SyntheticBackupInfo object. + + :param mock_parse_local_stg_path: parse_local_staging_path mock + :param mock_load_fields: load mock for backup_infos + :param mock_get_data_dir: get_data_directory method mock + :param mock__prepare_dest: _prepare_destination method mock + :param mock_fetch_remote_status: _fetch_remote_status method mock + :param mock_get_backup_chain_paths: _get_backup_chain_paths method mock + :param mock_pg_combinebackup: PgCombineBackup object mock + :param executor: executor mock fixture + :param server: server mock fixture + """ + parse_local_staging_path.return_value = "/home/fake/path/data" + + mock_backup_info = testing_helpers.build_test_backup_info( + backup_id="backup", + server=server, + tablespaces=[("tbs2", 16409, "/var/lib/pgsql/17/tablespaces2")], + ) + + mock_load_fields.side_effect = None + + def side_effect(tablespace_oid=None): + if tablespace_oid: + return f"/home/fake/path/data/{tablespace_oid}" + return "/home/fake/path/data" + + mock_get_data_dir.side_effect = side_effect + + mock_fetch_remote_status.return_value = { + "pg_combinebackup_installed": True, + "pg_combinebackup_path": "/fake/path", + "pg_combinebackup_version": "17", + } + + mock_get_backup_chain_paths.return_value = ( + "/some/barman/home/main/base/backup_%s/data/" % i for i in range(3) + ) + + tbs_map = {"/home/fake/path/data/16409": "/home/fake/path/data/16409"} + + mock_pg_combinebackup.side_effect = None + + result = executor._combine_backups( + mock_backup_info, "/home/mock/destination/to/combine" + ) + + calls = [ + mock.call(mock_get_data_dir()), + mock.call(mock_get_data_dir(16409)), + ] + + mock__prepare_dest.assert_has_calls(calls, any_order=False) + + mock_pg_combinebackup.assert_called_once_with( + destination=mock_get_data_dir(), + command=mock_fetch_remote_status.return_value["pg_combinebackup_path"], + version=mock_fetch_remote_status.return_value["pg_combinebackup_version"], + app_name=None, + tbs_mapping=tbs_map, + retry_times=0, + retry_sleep=30, + retry_handler=mock.ANY, + out_handler=mock.ANY, + args=mock_get_backup_chain_paths.return_value, + ) + + assert isinstance(result, SyntheticBackupInfo) + + @mock.patch( + "barman.recovery_executor.IncrementalRecoveryExecutor._move_to_destination" + ) + @mock.patch( + "barman.recovery_executor.IncrementalRecoveryExecutor._prepare_destination" + ) + @mock.patch("barman.infofile.LocalBackupInfo.get_data_directory") + def test__backup_copy_no_tablespaces( + self, mock_get_data_dir, mock_prepare_dest, mock_move_to_dest, executor, server + ): + """ + Unit test for the _backup_copy method without tablespaces. + + Create mock patches for the methods used inside _backup_copy. + + This unit tests checks if get_data_directory, _prepare_destination and + _move_to_destination are called (or not) with the correct parameters. + It tests one cenario: there are no tablespaces created in the postgres + server. + + :param mock_get_data_dir: get_data_directory method mock + :param mock_prepare_dest: _prepare_destination method mock + :param mock_move_to_dest: _move_to_destination method mock + :param executor: executor mock fixture + :param server: server mock fixture + """ + backup_info = testing_helpers.build_test_backup_info( + backup_id="backup", server=server, tablespaces=None + ) + mock_get_data_dir.return_value = "/some/barman/home/main/base/backup/data/" + + executor._backup_copy(backup_info, dest="destination/recover/path") + + mock_get_data_dir.assert_called_once() + mock_prepare_dest.assert_not_called() + mock_move_to_dest.assert_called_once_with( + source=mock_get_data_dir.return_value, + destination="destination/recover/path", + exclude_path_names={"pg_tblspc"}, + ) + + @mock.patch( + "barman.recovery_executor.IncrementalRecoveryExecutor._move_to_destination" + ) + @mock.patch( + "barman.recovery_executor.IncrementalRecoveryExecutor._prepare_destination" + ) + @mock.patch("barman.infofile.LocalBackupInfo.get_data_directory") + def test__backup_copy_with_tablespaces( + self, mock_get_data_dir, mock_prepare_dest, mock_move_to_dest, server, executor + ): + """ + Unit test for the _backup_copy method with tablespaces. + + Create mock patches for the methods used inside _backup_copy. + + This unit tests checks if get_data_directory, _prepare_destination and + _move_to_destination are called (or not) with the correct parameters. + It tests two cenario: there are two tablespaces created in the postgres + server. The first cenario has no tablespace mapping and the second has + tablespace mapping. + + :param mock_get_data_dir: get_data_directory method mock + :param mock_get_data_dest: _prepare_destination method mock + :param mock_move_to_dest: _move_to_destination method mock + :param executor: executor mock fixture + :param server: server mock fixture + """ + backup_info = testing_helpers.build_test_backup_info( + backup_id="backup", + server=server, + tablespaces=[ + ("tbs2", 16409, "/var/lib/pgsql/17/tablespaces2"), + ("tbs1", 16419, "/var/lib/pgsql/17/tablespaces"), + ], + ) + + def side_effect(tablespace_oid=None): + if tablespace_oid: + return f"/home/fake/path/data/{tablespace_oid}" + return "/home/fake/path/data" + + mock_get_data_dir.side_effect = side_effect + + executor._backup_copy(backup_info, dest="destination/recover/path") + + assert mock_get_data_dir.call_count == 3 + assert mock_prepare_dest.call_count == 2 + + tablespace_mapping = { + "tbs1": "/home/fake/path/tablespace1", + "tbs2": "/home/fake/path/tablespace2", + } + prepare_calls = { + "no_mapping": [ + mock.call(backup_info.tablespaces[0].location), + mock.call(backup_info.tablespaces[1].location), + ], + "with_mapping": [ + mock.call(tablespace_mapping["tbs2"]), + mock.call(tablespace_mapping["tbs1"]), + ], + } + + move_calls = { + "no_mapping": [ + mock.call( + source=mock_get_data_dir(tablespace_oid=16409), + destination=backup_info.tablespaces[0].location, + ), + mock.call( + source=mock_get_data_dir(tablespace_oid=16419), + destination=backup_info.tablespaces[1].location, + ), + mock.call( + source=mock_get_data_dir(), + destination="destination/recover/path", + exclude_path_names={"pg_tblspc"}, + ), + ], + "with_mapping": [ + mock.call( + source=mock_get_data_dir(tablespace_oid=16409), + destination=tablespace_mapping["tbs2"], + ), + mock.call( + source=mock_get_data_dir(tablespace_oid=16419), + destination=tablespace_mapping["tbs1"], + ), + mock.call( + source=mock_get_data_dir(), + destination="destination/recover/path", + exclude_path_names={"pg_tblspc"}, + ), + ], + } + + mock_prepare_dest.assert_has_calls(prepare_calls["no_mapping"], any_order=False) + mock_move_to_dest.assert_has_calls(move_calls["no_mapping"], any_order=False) + + executor._backup_copy( + backup_info, dest="destination/recover/path", tablespaces=tablespace_mapping + ) + + mock_prepare_dest.assert_has_calls( + prepare_calls["with_mapping"], any_order=False + ) + mock_move_to_dest.assert_has_calls(move_calls["with_mapping"], any_order=False) + + def test__backup_copy_remote(self, server, executor): + """ + Unit test for the _backup_copy method with remote option. + + Create mock patches for the methods used inside _backup_copy. + + This unit tests checks if get_data_directory, _prepare_destination and + _move_to_destination are called (or not) with the correct parameters. + It tests two cenario: there are two tablespaces created in the postgres + server. The first cenario has no tablespace mapping and the second has + tablespace mapping. + + :param mock_get_data_dir: get_data_directory method mock + :param mock_get_data_dest: _prepare_destination method mock + :param mock_move_to_dest: _move_to_destination method mock + :param executor: executor mock fixture + :param server: server mock fixture + """ + backup_info = testing_helpers.build_test_backup_info( + backup_id="backup", + server=server, + tablespaces=[ + ("tbs2", 16409, "/var/lib/pgsql/17/tablespaces2"), + ("tbs1", 16419, "/var/lib/pgsql/17/tablespaces"), + ], + ) + + dest = "destination/recover/path" + remote_command = "ssh pg" + tablespaces = {"tbs_name": "/destination/location"} + + with mock.patch( + "barman.recovery_executor.RecoveryExecutor._backup_copy" + ) as mock_super__backup_copy: + executor._backup_copy( + backup_info, + dest=dest, + remote_command=remote_command, + tablespaces=tablespaces, + ) + + mock_super__backup_copy.assert_called_once_with( + backup_info, + dest, + tablespaces, + remote_command, + ) + + @mock.patch("barman.infofile.LocalBackupInfo.walk_to_root") + def test__get_backup_chain_paths(self, mock_walk_to_root, executor, server): + """ + Unit test for the _get_backup_chain_paths method. + + Create mock patch for walk_to_root method used inside _get_backup_chain_paths. + + This unit tests checks if the result paths are returned in the corret order. + + :param mock_walk_to_root: walk_to_root method mock + :param executor: executor mock fixture + :param server: server mock fixture + """ + mock_walk_to_root.return_value = ( + testing_helpers.build_test_backup_info( + backup_id="b%s" % i, + server=server, + parent_backup_id=(None if i == 0 else "b" + str(i - 1)), + ) + for i in range(3) + ) + + backup_info = testing_helpers.build_test_backup_info( + backup_id="b2", + server=server, + parent_backup_id="b1", + ) + + basedir = "/some/barman/home/main/base/" + result = list(executor._get_backup_chain_paths(backup_info)) + + assert list(result) == [ + basedir + "b2/data", + basedir + "b1/data", + basedir + "b0/data", + ] + + @mock.patch("barman.command_wrappers.PostgreSQLClient.find_command") + def test__fetch_remote_status(self, find_command, executor): + """ + Unit test for the _fetch_remote_status method. + + Create mock patch for find_command. + + This unit tests checks the information for the pg_combinebackup client + of the server. + + :param find_command: find_command mock + :param executor: executor mock fixture + """ + # Simulate the absence of pg_combinebackup + find_command.side_effect = CommandFailedException + executor.backup_manager.server.postgres.server_major_version = "16" + remote = executor._fetch_remote_status() + assert remote["pg_combinebackup_installed"] is False + assert remote["pg_combinebackup_path"] is None + + # Simulate the presence of pg_combinebackup 17 and pg 17 + find_command.side_effect = None + find_command.return_value.cmd = "/fake/path" + find_command.return_value.out = "pg_combinebackup 17.0.0" + executor.server.postgres.server_major_version = "17" + executor.server.path = "fake/path2" + remote = executor._fetch_remote_status() + assert remote["pg_combinebackup_installed"] is True + assert remote["pg_combinebackup_path"] == "/fake/path" + assert remote["pg_combinebackup_version"] == "17.0.0" + + # Simulate the presence of pg_combinebackup 17 and no Pg + executor.server.postgres.server_major_version = None + find_command.reset_mock() + find_command.return_value.out = "pg_combinebackup 17.0.0" + remote = executor._fetch_remote_status() + assert remote["pg_combinebackup_installed"] is True + assert remote["pg_combinebackup_path"] == "/fake/path" + assert remote["pg_combinebackup_version"] == "17.0.0" + + @mock.patch("os.chmod") + @mock.patch("os.makedirs") + @mock.patch("shutil.rmtree") + def test__prepare_destination(self, mock_rmtree, mock_mkdir, mock_chmod, executor): + """ + Unit test for the _prepare_destination method. + + Create mock patch for shutil.rmtree, os.makedirs and os.chmod. + + This unit tests checks if all methods are called once with the correct + args and the number of calls. + + :param mock_rmtree: shutil.rmtree mock object + :param mock_mkdir: os.makedirs mock object + :param mock_chmod: os.chmod mock object + :param executor: executor mock fixture + """ + dest_dir = "/destination/directory" + executor._prepare_destination(dest_dir) + mock_rmtree.assert_called_once_with(dest_dir, ignore_errors=True) + mock_mkdir.assert_called_once_with(dest_dir) + mock_chmod.assert_called_once_with(dest_dir, 448) + + @mock.patch("shutil.move") + @mock.patch("os.path.join") + @mock.patch("os.listdir") + def test__move_to_destination( + self, + mock_listdir, + mock_path_join, + mock_sh_move, + executor, + ): + """ + Unit test for the _move_to_destination method. + + Create mock patch for os.listdir, os.path.join and oshutil.move. + + This unit tests checks if all methods are called with the correct args + and the number of calls. + + :param mock_listdir: os.listdir mock object + :param mock_path_join: os.path.join mock object + :param mock_sh_move: shutil.move mock object + :param executor: executor mock fixture + """ + mock_listdir.return_value = [ + "some/directory", + "another", + "i_am_a_file.py", + ] + + source_dir = "/source/destination" + dest_dir = "target/destination" + + def side_effect(source="/source/destination", file_or_dir=""): + return source + "/" + file_or_dir + + mock_path_join.side_effect = side_effect + + executor._move_to_destination( + source=source_dir, destination=dest_dir, exclude_path_names=set() + ) + + mock_listdir.assert_called_once_with(source_dir) + assert mock_path_join.call_count == 3 + assert mock_sh_move.call_count == 3 + + calls = [ + call("/source/destination/some/directory", dest_dir), + call("/source/destination/another", dest_dir), + call("/source/destination/i_am_a_file.py", dest_dir), + ] + mock_sh_move.assert_has_calls(calls, any_order=False) + + @mock.patch("shutil.move") + @mock.patch("os.path.join") + @mock.patch("os.listdir") + def test__move_to_destination_exclude_path( + self, + mock_listdir, + mock_path_join, + mock_sh_move, + executor, + ): + """ + Unit test for the _move_to_destination method excluding a path. + + Create mock patch for os.listdir, os.path.join and oshutil.move. + + This unit tests checks if all methods are called with the correct args + and the number of calls. + + :param mock_listdir: os.listdir mock object + :param mock_path_join: os.path.join mock object + :param mock_sh_move: shutil.move mock object + :param executor: executor mock fixture + """ + mock_listdir.return_value = [ + "some/directory", + "another", + "i_am_a_file.py", + ] + + source_dir = "/source/destination" + dest_dir = "target/destination" + + def side_effect(source="/source/destination", file_or_dir=""): + return source + "/" + file_or_dir + + mock_path_join.side_effect = side_effect + + executor._move_to_destination( + source=source_dir, + destination=dest_dir, + exclude_path_names={"i_am_a_file.py"}, + ) + + mock_listdir.assert_called_once_with(source_dir) + assert mock_path_join.call_count == 2 + assert mock_sh_move.call_count == 2 + assert mock_sh_move.call_count == 2 + + calls = [ + call("/source/destination/some/directory", dest_dir), + call("/source/destination/another", dest_dir), + ] + mock_sh_move.assert_has_calls(calls, any_order=False) + + @mock.patch("barman.output.error") + @mock.patch("shutil.move") + @mock.patch("os.path.join") + @mock.patch("os.listdir") + def test__move_to_destination_error( + self, + mock_listdir, + mock_path_join, + mock_sh_move, + mock_error, + executor, + ): + """ + Unit test for the _move_to_destination method with shutil.Error. + + Create mock patch for os.listdir, os.path.join, oshutil.move and + barman.output.error. + + This unit tests checks if an error is raised when shutil.move fails + and the error message that output.error is called with. + Also checks number of method calls. + + :param mock_listdir: os.listdir mock object + :param mock_path_join: os.path.join mock object + :param mock_sh_move: shutil.move mock object + :param mock_error: barman.output.error mock object + :param executor: executor mock fixture + """ + mock_listdir.return_value = [ + "some/directory", + "another/", + "i_am_a_file.py", + ] + + source_dir = "/source/destination" + dest_dir = "target/destination" + + def side_effect(source="/source/destination", file_or_dir=""): + return source + "/" + file_or_dir + + mock_path_join.side_effect = side_effect + + def move_side_effect(path=None, file_or_dir=None): + raise shutil.Error() + + mock_sh_move.side_effect = move_side_effect + with pytest.raises(SystemExit): + executor._move_to_destination( + source=source_dir, destination=dest_dir, exclude_path_names=set() + ) + + assert mock_path_join.call_count == 1 + assert mock_sh_move.call_count == 1 + mock_error.assert_called_once_with( + f"Destination directory '{dest_dir}' must be empty." + ) + + @mock.patch( + "barman.recovery_executor.IncrementalRecoveryExecutor._prepare_destination" + ) + @mock.patch("barman.output.warning") + def test__retry_handler(self, mock_warning, mock__prepare_dest, executor): + """ + Unit test for the _retry_handler method. + + Create mock patch for barman.output.warning and _prepare_destination. + + This unit tests checks number of calls, calls order and if the methods + are called with the correct args. + + :param mock_warning: barman.output.warning mock object + :param mock__prepare_dest: _prepare_destination mock object + :param executor: executor mock fixture + """ + dest_dirs = [ + "some/destination", + "another", + "i_am_a_file.py", + ] + executor._retry_handler(dest_dirs=dest_dirs, attempt=3) + + assert mock_warning.call_count == 2 + calls = [ + call("Failure combining backups using pg_combinebackup (attempt %s)", 3), + call( + "The files created so far will be removed and the combine process will restart in %s seconds", + "30", + ), + ] + mock_warning.assert_has_calls(calls, any_order=False) + + assert mock__prepare_dest.call_count == 3 + calls = [call("some/destination"), call("another"), call("i_am_a_file.py")] + mock__prepare_dest.assert_has_calls(calls, any_order=False) + + @mock.patch("barman.output.info") + def test__start_message(self, mock_info, executor, synthetic_backup_info): + """ + Unit test for the _start_message method. + + Create mock patch for barman.output.info. + + This unit tests checks if there is a call to the method with the correct + message. + . + :param mock_info: barman.output.info mock object + :param executor: executor mock fixture + :param synthetic_backup_info: synthetic_backup_info mock fixture + """ + executor._start_message(synthetic_backup_info) + mock_info.assert_called_once_with( + "Start combining backup via pg_combinebackup for backup %s on %s", + synthetic_backup_info.backup_id, + synthetic_backup_info.base_directory, + ) + + @mock.patch("barman.output.info") + def test__end_message(self, mock_info, executor, synthetic_backup_info): + """ + Unit test for the _end_message method. + + Create mock patch for barman.output.info. + + This unit tests checks if there is a call to the method with the correct + message. + + :param mock_info: barman.output.info mock object + :param executor: executor mock fixture + :param synthetic_backup_info: synthetic_backup_info mock fixture + """ + executor._end_message(synthetic_backup_info) + mock_info.assert_called_once_with( + "End combining backup via pg_combinebackup for backup %s", + synthetic_backup_info.backup_id, + ) diff --git a/tests/test_retention_policies.py b/tests/test_retention_policies.py index 0d9080e7f..097781bc2 100644 --- a/tests/test_retention_policies.py +++ b/tests/test_retention_policies.py @@ -16,6 +16,7 @@ # You should have received a copy of the GNU General Public License # along with Barman. If not, see . +import itertools import logging import re from datetime import datetime, timedelta @@ -35,6 +36,7 @@ class TestRetentionPolicies(object): + @pytest.fixture def server(self): backup_manager = mock.Mock() @@ -180,51 +182,91 @@ def test_backup_status(self, server): # Build a BackupInfo object with status to DONE backup_info = build_test_backup_info( - server=server, backup_id="test1", end_time=datetime.now(tzlocal()) + server=server, + backup_id="test_backup", + end_time=datetime.now(tzlocal()), + parent_backup_id=None, + children_backup_ids=["test_backup_child"], + ) + + # Build a CHILD BackupInfo object with status to DONE + child_backup_info = build_test_backup_info( + server=server, + backup_id="test_backup_child", + end_time=datetime.now(tzlocal()) + timedelta(days=1), + parent_backup_id="test_backup", + children_backup_ids=None, ) # instruct the get_available_backups method to return a map with # our mock as result and minimum_redundancy = 1 - server.get_available_backups.return_value = {"test_backup": backup_info} + server.get_available_backups.return_value = { + "test_backup": backup_info, + "test_backup_child": child_backup_info, + } server.config.minimum_redundancy = 1 - # execute retention policy report - report = rp.backup_status("test_backup") - assert report == "VALID" + # execute retention policy report on parent + report_parent = rp.backup_status("test_backup") + + assert report_parent == "VALID" + + # execute retention policy report on child + report_child = rp.backup_status("test_backup_child") + + assert report_child == "VALID" # Force context of retention policy for testing purposes. # Expect the method to return a BackupInfo.NONE value rp.context = "invalid" - empty_report = rp.backup_status("test_backup") + empty_report_parent = rp.backup_status("test_backup") + + assert empty_report_parent == BackupInfo.NONE - assert empty_report == BackupInfo.NONE + empty_report_child = rp.backup_status("test_backup_child") + + assert empty_report_child == BackupInfo.NONE rp = RetentionPolicyFactory.create( "retention_policy", "RECOVERY WINDOW OF 4 WEEKS", server=server ) assert isinstance(rp, RecoveryWindowRetentionPolicy) - # Build a BackupInfo object with status to DONE - backup_info = build_test_backup_info( - server=server, backup_id="test1", end_time=datetime.now(tzlocal()) - ) - # instruct the get_available_backups method to return a map with # our mock as result and minimum_redundancy = 1 - server.get_available_backups.return_value = {"test_backup": backup_info} + server.get_available_backups.return_value = { + "test_backup": backup_info, + "test_backup_child": child_backup_info, + } server.config.minimum_redundancy = 1 - # execute retention policy report - report = rp.backup_status("test_backup") - assert report == "VALID" + # execute retention policy report on parent + report_parent = rp.backup_status("test_backup") + + assert report_parent == "VALID" + + # execute retention policy report on child + report_child = rp.backup_status("test_backup_child") + + assert report_child == "VALID" # Force context of retention policy for testing purposes. # Expect the method to return a BackupInfo.NONE value rp.context = "invalid" - empty_report = rp.backup_status("test_backup") + empty_report_parent = rp.backup_status("test_backup") + + assert empty_report_parent == BackupInfo.NONE + + empty_report_child = rp.backup_status("test_backup_child") - assert empty_report == BackupInfo.NONE + assert empty_report_child == BackupInfo.NONE def test_first_backup(self, server): + """ + Basic unit test of method first_backup + + This method tests the retrieval of the first backup using both + RedundancyRetentionPolicy and RecoveryWindowRetentionPolicy + """ rp = RetentionPolicyFactory.create( "retention_policy", "RECOVERY WINDOW OF 4 WEEKS", server ) @@ -232,12 +274,21 @@ def test_first_backup(self, server): # Build a BackupInfo object with status to DONE backup_info = build_test_backup_info( - server=server, backup_id="test0", end_time=datetime.now(tzlocal()) + server=server, + backup_id="test0", + end_time=datetime.now(tzlocal()) - timedelta(days=1), + ) + # Build another BackupInfo object with status to DONE taken one day after + backup_info2 = build_test_backup_info( + server=server, backup_id="test1", end_time=datetime.now(tzlocal()) ) # instruct the get_available_backups method to return a map with # our mock as result and minimum_redundancy = 1 - server.get_available_backups.return_value = {"test_backup": backup_info} + server.get_available_backups.return_value = { + "test_backup": backup_info, + "test_backup2": backup_info2, + } server.config.minimum_redundancy = 1 # execute retention policy report report = rp.first_backup() @@ -249,14 +300,12 @@ def test_first_backup(self, server): ) assert isinstance(rp, RedundancyRetentionPolicy) - # Build a BackupInfo object with status to DONE - backup_info = build_test_backup_info( - server=server, backup_id="test1", end_time=datetime.now(tzlocal()) - ) - # instruct the get_available_backups method to return a map with # our mock as result and minimum_redundancy = 1 - server.get_available_backups.return_value = {"test_backup": backup_info} + server.get_available_backups.return_value = { + "test_backup": backup_info, + "test_backup2": backup_info2, + } server.config.minimum_redundancy = 1 # execute retention policy report @@ -264,6 +313,265 @@ def test_first_backup(self, server): assert report == "test_backup" + @pytest.mark.parametrize( + ("retention_policy", "retention_status"), + itertools.product( + ("RECOVERY WINDOW OF 4 WEEKS", "REDUNDANCY 2"), + ( + BackupInfo.OBSOLETE, + BackupInfo.VALID, + BackupInfo.POTENTIALLY_OBSOLETE, + BackupInfo.KEEP_FULL, + BackupInfo.KEEP_STANDALONE, + BackupInfo.NONE, + ), + ), + ) + @mock.patch("barman.retention_policies._logger.debug") + @mock.patch("barman.infofile.LocalBackupInfo.walk_backups_tree") + def test__propagate_retention_status_to_children( + self, + mock_walk_backups_tree, + mock_logger, + retention_policy, + retention_status, + server, + tmpdir, + ): + """ + Unit test of method _propagate_retention_status_to_children + """ + + # Use this to Build a chain of incrementals BackupInfo objects in + # post-order up to the root. + chain = { + "b3": """parent_backup_id=b2 + children_backup_ids=None + status=DONE""", + "b6": """parent_backup_id=b2 + children_backup_ids=None + status=DONE""", + "b2": """parent_backup_id=root + children_backup_ids=b3,b6 + status=DONE""", + "b5": """parent_backup_id=b4 + children_backup_ids=None + status=DONE""", + "b4": """parent_backup_id=root + children_backup_ids=b5 + status=DONE""", + "root": """parent_backup_id=None + children_backup_ids=b2,b4 + status=DONE""", + } + backup_chain = {} + for bkp in chain: + infofile = tmpdir.mkdir(bkp).join("backup.info") + infofile.write(chain[bkp]) + b_info = build_test_backup_info( + backup_id=bkp, + server=server, + ) + backup_chain[bkp] = b_info + + root = backup_chain["root"] + mock_walk_backups_tree.return_value = iter(list(backup_chain.values())[:-1]) + + rp = RetentionPolicyFactory.create( + "retention_policy", retention_policy, server=server + ) + + report = {} + rp._propagate_retention_status_to_children(root, report, retention_status) + + mock_walk_backups_tree.assert_called_once() + + assert mock_logger.call_count == 5 + # For full backups with status KEEP, we propagate VALID status to children + if retention_status in (BackupInfo.KEEP_FULL, BackupInfo.KEEP_STANDALONE): + retention_status = BackupInfo.VALID + for backup_id in report: + mock_logger.assert_any_call( + "Propagating %s retention status of backup root to %s." + % (retention_status, backup_id) + ) + + for backup in report: + assert report[backup] == retention_status + + def test_redundancy_report_with_incrementals(self, server, caplog): + """ + Test of the management of the minimum_redundancy parameter + into the backup_report method of the RedundancyRetentionPolicy class + + """ + rp = RetentionPolicyFactory.create( + "retention_policy", "REDUNDANCY 2", server=server + ) + assert isinstance(rp, RedundancyRetentionPolicy) + + backups_data = { + "20240628T000000": { + "parent_backup_id": None, + "children_backup_ids": ["20240628T120000"], + "end_time": datetime.now(tzlocal()) - timedelta(weeks=6, days=1), + }, + "20240628T120000": { + "parent_backup_id": "20240628T000000", + "children_backup_ids": None, + "end_time": datetime.now(tzlocal()) - timedelta(weeks=6), + }, + "20240629T000000": { + "parent_backup_id": None, + "children_backup_ids": ["20240629T120000"], + "end_time": datetime.now(tzlocal()) - timedelta(weeks=5, days=1), + }, + "20240629T120000": { + "parent_backup_id": "20240629T000000", + "children_backup_ids": None, + "end_time": datetime.now(tzlocal()) - timedelta(weeks=5), + }, + "20240630T000000": { + "parent_backup_id": None, + "children_backup_ids": None, + "end_time": datetime.now(tzlocal()), + }, + } + + available_backups = {} + for bkp_id, info in backups_data.items(): + available_backups[bkp_id] = build_test_backup_info( + backup_id=bkp_id, + server=server, + parent_backup_id=info["parent_backup_id"], + children_backup_ids=info["children_backup_ids"], + end_time=info["end_time"], + ) + + # instruct the get_available_backups method to return a map with + # our mock as result and minimum_redundancy = 1 + server.get_available_backups.return_value = available_backups + + server.config.minimum_redundancy = 1 + + # execute retention policy report + report = rp.report() + # check that our mock is valid for the retention policy because + # the total number of valid backups is lower than the retention policy + # redundancy. + assert report == { + "20240630T000000": "VALID", + "20240629T000000": "VALID", + "20240629T120000": "VALID", + "20240628T000000": "OBSOLETE", + "20240628T120000": "OBSOLETE", + } + + # Expect a ValueError if passed context is invalid + with pytest.raises(ValueError): + rp.report(context="invalid") + # Set a new minimum_redundancy parameter, enforcing the usage of the + # configuration parameter instead of the retention policy default + server.config.minimum_redundancy = 3 + # execute retention policy report + rp.report() + # Check for the warning inside the log + caplog.set_level(logging.WARNING) + + log = caplog.text + assert log.find( + "WARNING Retention policy redundancy (2) " + "is lower than the required minimum redundancy (3). " + "Enforce 3." + ) + + def test_recovery_window_report_with_incrementals(self, server, caplog): + """ + Test of the management of the minimum_redundancy parameter + into the backup_report method of the RecoveryWindowRetentionPolicy class + + """ + rp = RetentionPolicyFactory.create( + "retention_policy", "RECOVERY WINDOW OF 4 WEEKS", server=server + ) + assert isinstance(rp, RecoveryWindowRetentionPolicy) + + backups_data = { + "20240628T000000": { + "parent_backup_id": None, + "children_backup_ids": ["20240628T120000"], + "end_time": datetime.now(tzlocal()) - timedelta(weeks=6, days=1), + }, + "20240628T120000": { + "parent_backup_id": "20240628T000000", + "children_backup_ids": None, + "end_time": datetime.now(tzlocal()) - timedelta(weeks=6), + }, + "20240629T000000": { + "parent_backup_id": None, + "children_backup_ids": ["20240629T120000"], + "end_time": datetime.now(tzlocal()) - timedelta(weeks=5, days=1), + }, + "20240629T120000": { + "parent_backup_id": "20240629T000000", + "children_backup_ids": None, + "end_time": datetime.now(tzlocal()) - timedelta(weeks=5), + }, + "20240630T000000": { + "parent_backup_id": None, + "children_backup_ids": None, + "end_time": datetime.now(tzlocal()), + }, + } + + available_backups = {} + for bkp_id, info in backups_data.items(): + available_backups[bkp_id] = build_test_backup_info( + backup_id=bkp_id, + server=server, + parent_backup_id=info["parent_backup_id"], + children_backup_ids=info["children_backup_ids"], + end_time=info["end_time"], + ) + + # instruct the get_available_backups method to return a map with + # our mock as result and minimum_redundancy = 1 + server.get_available_backups.return_value = available_backups + + server.config.minimum_redundancy = 1 + server.config.name = "test" + + # execute retention policy report + report = rp.report() + # check that our mock is valid for the retention policy because + # the total number of valid backups is lower than the retention policy + # redundancy. + assert report == { + "20240630T000000": "VALID", + "20240629T000000": "VALID", + "20240629T120000": "VALID", + "20240628T000000": "OBSOLETE", + "20240628T120000": "OBSOLETE", + } + + # Expect a ValueError if passed context is invalid + with pytest.raises(ValueError): + rp.report(context="invalid") + # Set a new minimum_redundancy parameter, enforcing the usage of the + # configuration parameter instead of the retention policy default + server.config.minimum_redundancy = 4 + # execute retention policy report + rp.report() + # Check for the warning inside the log + caplog.set_level(logging.WARNING) + log = caplog.text + warn = ( + r"WARNING .*Keeping obsolete backup 20240628T000000 for " + r"server test \(older than .*\) due to minimum redundancy " + r"requirements \(4\)\n" + ) + assert re.search(warn, log) + class TestRedundancyRetentionPolicyWithKeepAnnotation(object): """ diff --git a/tests/test_server.py b/tests/test_server.py index 95a25c918..9bfe0787f 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -26,6 +26,7 @@ import dateutil.tz from io import BytesIO +import mock import pytest from mock import MagicMock, Mock, PropertyMock, patch from psycopg2.tz import FixedOffsetTimezone @@ -283,7 +284,15 @@ def test_get_wal_full_path(self, tmpdir): assert full_path == str(tmpdir.join("wals").join(wal_hash).join(wal_name)) @pytest.mark.parametrize( - ["wal_info_files", "target_tlis", "target_time", "expected_indices"], + [ + "wal_info_files", + "target_tlis", + "target_time", + "target_xid", + "target_lsn", + "target_immediate", + "expected_indices", + ], [ ( # GIVEN The following WALs @@ -299,6 +308,12 @@ def test_get_wal_full_path(self, tmpdir): (None, 2, "current"), # AND no target_time None, + # AND no target_xid + None, + # AND no target_lsn + None, + # AND target_immediate is False + False, # WHEN get_required_xlog_files runs for a backup on tli 2 # the WAL on tli 2 is returned along with all history files [1, 2, 3, 5], @@ -318,6 +333,12 @@ def test_get_wal_full_path(self, tmpdir): (None, 2, "current"), # AND no target_time None, + # AND no target_xid + None, + # AND no target_lsn + None, + # AND target_immediate is False + False, # WHEN get_required_xlog_files runs for a backup on tli 2 # all WALs on tli 2 are returned along with all history files [1, 2, 3, 4, 6], @@ -338,11 +359,19 @@ def test_get_wal_full_path(self, tmpdir): (None, 2, "current"), # AND a target_time of 44 44, + # AND no target_xid + None, + # AND no target_lsn + None, + # AND target_immediate is False + False, # WHEN get_required_xlog_files runs for a backup on tli 2 - # the first two WALs on tli 2 are returned along with all history - # files. The WAL on tli 2 which starts after the target_time is - # not returned. - [1, 2, 3, 5, 7], + # all WALs on tli 2 are returned along with all history files. + # All WALs on tli 2 are returned because there is no reliable + # way of determining the required WAL files based on target_time + # other than inspecting pg_waldump, which would put a lot of + # overhead + [1, 2, 3, 4, 5, 7], ), ( # Verify both WALs on timeline 2 are returned plus all history files @@ -360,15 +389,119 @@ def test_get_wal_full_path(self, tmpdir): (10, "latest"), # AND no target_time None, + # AND no target_xid + None, + # AND no target_lsn + None, + # AND target_immediate is False + False, # WHEN get_required_xlog_files runs for a backup on tli 2 # all WALs on timelines 2 and 10 are returned along with all history # files. [1, 2, 3, 4, 5, 6], ), + ( + # GIVEN The following WALs + [ + create_fake_info_file("000000010000000000000002", 42, 43), + create_fake_info_file("00000001.history", 42, 43), + create_fake_info_file("000000020000000000000003", 42, 44), + create_fake_info_file("000000020000000000000005", 42, 45), + create_fake_info_file("000000020000000000000007", 42, 45), + create_fake_info_file("000000020000000000000009", 42, 45), + create_fake_info_file("000000020000000000000010", 42, 46), + create_fake_info_file("00000002.history", 42, 44), + create_fake_info_file("0000000A0000000000000005", 42, 47), + create_fake_info_file("0000000A.history", 42, 47), + ], + # AND target_tli values None, 2 and current + (None, 2, "current"), + # AND no target_time + None, + # AND target_xid of 100 + "100", + # AND no target_lsn + None, + # AND target_immediate is False + False, + # WHEN get_required_xlog_files runs for a backup on tli 2 + # all WALs on tli 2 are returned along with all history files. + # All WALs on tli 2 are returned because there is no reliable + # way of determining the required WAL files based on target_xid + # other than inspecting pg_waldump, which would put a lot of + # overhead + [1, 2, 3, 4, 5, 6, 7, 9], + ), + ( + # GIVEN The following WALs + [ + create_fake_info_file("000000010000000000000002", 42, 43), + create_fake_info_file("00000001.history", 42, 43), + create_fake_info_file("000000020000000000000003", 42, 44), + create_fake_info_file("000000020000000000000005", 42, 45), + create_fake_info_file("000000020000000000000007", 42, 45), + create_fake_info_file("000000020000000000000009", 42, 45), + create_fake_info_file("000000020000000000000010", 42, 46), + create_fake_info_file("00000002.history", 42, 44), + create_fake_info_file("0000000A0000000000000005", 42, 47), + create_fake_info_file("0000000A.history", 42, 47), + ], + # AND target_tli values None, 2 and current + (None, 2, "current"), + # AND no target_time + None, + # AND no target_xid + None, + # AND a target_lsn of '0/07000000' + "0/07000000", + # AND target_immediate is False + False, + # WHEN get_required_xlog_files runs for a backup on tli 2 + # all WALs on tli 2 up to the requested LSN are returned along + # with all history files. + [1, 2, 3, 4, 7, 9], + ), + ( + # GIVEN The following WALs + [ + create_fake_info_file("000000010000000000000002", 42, 43), + create_fake_info_file("00000001.history", 42, 43), + create_fake_info_file("000000020000000000000003", 42, 44), + create_fake_info_file("000000020000000000000005", 42, 45), + create_fake_info_file("000000020000000000000007", 42, 45), + create_fake_info_file("000000020000000000000009", 42, 45), + create_fake_info_file("000000020000000000000010", 42, 46), + create_fake_info_file("00000002.history", 42, 44), + create_fake_info_file("0000000A0000000000000005", 42, 47), + create_fake_info_file("0000000A.history", 42, 47), + ], + # AND target_tli values None, 2 and current + (None, 2, "current"), + # AND no target_time + None, + # AND no target_xid + None, + # AND no target_lsn + None, + # AND target_immediate is True + True, + # WHEN get_required_xlog_files runs for a backup on tli 2 + # all WALs on tli 2 up to the end_xlog from the backup are + # returned along with all history files. + [1, 2, 7, 9], + ), ], ) def test_get_required_xlog_files( - self, wal_info_files, target_tlis, target_time, expected_indices, tmpdir + self, + wal_info_files, + target_tlis, + target_time, + target_xid, + target_lsn, + target_immediate, + expected_indices, + tmpdir, ): """ Tests get_required_xlog_files function. @@ -414,7 +547,12 @@ def test_get_required_xlog_files( for target_tli in target_tlis: wals = [] for wal_file in server.get_required_xlog_files( - backup, target_tli, target_time + backup, + target_tli, + target_time, + target_xid, + target_lsn, + target_immediate=target_immediate, ): # get the result of the xlogdb read wals.append(wal_file.name) @@ -1148,58 +1286,137 @@ def test_backup( out, err = capsys.readouterr() assert "Permission denied, unable to access" in err - @patch("barman.server.Server.get_first_backup_id") + @patch("barman.backup.BackupManager.should_keep_backup") + def test_cannot_delete_keep_backup(self, mock_should_keep_backup, caplog, tmpdir): + """Verify that we cannot delete backups directly if they have a keep""" + server = build_real_server({"barman_home": tmpdir.strpath}) + backup_info = build_test_backup_info( + backup_id="fake_backup_id", + status=BackupInfo.DONE, + server=server, + ) + backup_info.save() + + mock_should_keep_backup.return_value = True + + assert server.delete_backup(backup_info) is False + assert ( + "Skipping delete of backup %s for server %s as it has a current " + "keep request. If you really want to delete this backup please remove " + "the keep and try again." % (backup_info.backup_id, server.config.name) + in caplog.text + ) + + @patch("barman.backup.BackupManager.get_available_backups") + def test_cannot_delete_backup_due_to_minimum_redundancy( + self, mock_get_available_backups, caplog, tmpdir + ): + """ + Verify that we cannot delete a backup if it does not satisfy the server's + minimum redundancy policy + """ + server = build_real_server({"barman_home": tmpdir.strpath}) + server.config.minimum_redundancy = 2 + backup_info = build_test_backup_info( + backup_id="fake_backup_id", + status=BackupInfo.DONE, + server=server, + ) + backup_info.save() + + mock_get_available_backups.return_value = {backup_info.backup_id: backup_info} + + assert server.delete_backup(backup_info) is False + assert ( + "Skipping delete of backup %s for server %s due to minimum redundancy " + "requirements (minimum redundancy = 2, current redundancy = 1)" + % (backup_info.backup_id, server.config.name) + in caplog.text + ) + @patch("barman.server.BackupManager.delete_backup") + @patch("barman.backup.BackupManager.get_available_backups") def test_delete_running_backup( - self, delete_mock, get_first_backup_mock, tmpdir, capsys + self, get_available_backups, delete_mock, tmpdir, capsys ): """ Simple test for the deletion of a running backup. We want to test the behaviour of the server.delete_backup method when invoked on a running backup """ - # Test the removal of a running backup. status STARTED + # Test the removal of a running backup server = build_real_server({"barman_home": tmpdir.strpath}) - backup_info_started = build_test_backup_info( - status=BackupInfo.STARTED, server_name=server.config.name - ) - get_first_backup_mock.return_value = backup_info_started.backup_id - with ServerBackupLock(tmpdir.strpath, server.config.name): - server.delete_backup(backup_info_started) - out, err = capsys.readouterr() - assert ( - "Another action is in progress for the backup %s" - " of server %s. Impossible to delete the backup." - % (backup_info_started.backup_id, server.config.name) - in err - ) - - # Test the removal of a running backup. status EMPTY - backup_info_empty = build_test_backup_info( - status=BackupInfo.EMPTY, server_name=server.config.name + backup_info = build_test_backup_info( + status=BackupInfo.DONE, server_name=server.config.name ) - get_first_backup_mock.return_value = backup_info_empty.backup_id + get_available_backups.return_value = {backup_info.backup_id: backup_info} with ServerBackupLock(tmpdir.strpath, server.config.name): - server.delete_backup(backup_info_empty) + server.delete_backup(backup_info) out, err = capsys.readouterr() assert ( - "Another action is in progress for the backup %s" - " of server %s. Impossible to delete the backup." - % (backup_info_started.backup_id, server.config.name) + "Another process in running on server %s. Impossible to delete the backup." + % server.config.name in err ) - # Test the removal of a running backup. status DONE - backup_info_done = build_test_backup_info( - status=BackupInfo.DONE, server_name=server.config.name - ) - with ServerBackupLock(tmpdir.strpath, server.config.name): - server.delete_backup(backup_info_done) - delete_mock.assert_called_with(backup_info_done) + # Test the removal of a backup not running + server.delete_backup(backup_info) + delete_mock.assert_called_with(backup_info) - # Test the removal of a backup not running. status STARTED - server.delete_backup(backup_info_started) - delete_mock.assert_called_with(backup_info_started) + def test_delete_backup_with_children(self, tmpdir): + """ + Test that a parent backup is deleted along with its descendants + """ + server = build_real_server({"barman_home": tmpdir.strpath}) + server.backup_manager.delete_backup = Mock() + + # This test works with the following backup tree structure: + # root + # | + # ----------------- + # | | + # child1 child2 + # | | + # | child2.1 + # child1.1 child1.2 + + # Mounts the tree. key = backup_id, value = tuple(parent_id, children_ids) + backup_tree = { + "root": (None, ["child1", "child2"]), + "child1": ("root", ["child1.1", "child1.2"]), + "child1.1": ("child1", None), + "child1.2": ("child1", None), + "child2": ("root", ["child2.1"]), + "child2.1": ("child2", None), + } + for backup_id, attributes in backup_tree.items(): + backup_info_object = build_test_backup_info( + backup_id=backup_id, + server=server, + parent_backup_id=attributes[0], + children_backup_ids=attributes[1], + ) + backup_info_object.save() + backup_tree[backup_id] = backup_info_object + + # Test 1: deleting the root backup should also delete all its children + root_backup = backup_tree["root"] + server.delete_backup(root_backup) + # assert that the backup manager mock received the expected backups for deletion + manager_delete_calls = server.backup_manager.delete_backup.call_args_list + to_delete = ["child1.1", "child1.2", "child1", "child2.1", "child2", "root"] + for n_call, call_obj in enumerate(manager_delete_calls): + assert call_obj.args[0].backup_id == to_delete[n_call] + + # Test 2: deleting a leaf backup should only delete that one + server.backup_manager.delete_backup.reset_mock() + leaf_backup = backup_tree["child2.1"] + server.delete_backup(leaf_backup) + server.backup_manager.delete_backup.assert_called_once_with(leaf_backup) + + # We could have additional tests with other backups in different positions in the tree + # but then we would essentially be testing the tree-walk algorithm instead. + # This test only ensures that children are being deleted along with the parent when they exist @patch("subprocess.Popen") def test_archive_wal_lock_acquisition(self, subprocess_mock, tmpdir, capsys): @@ -1842,7 +2059,7 @@ def test_receive_wal_checks( @patch("barman.infofile.BackupInfo.save") @patch("os.path.exists") def test_check_backup( - self, mock_exists, backup_info_save, tmpdir, orig_exists=os.path.exists + self, mock_exists, backup_info_save, tmpdir, capsys, orig_exists=os.path.exists ): """ Test the check_backup method @@ -1918,6 +2135,8 @@ def mock_os_path_exist(file_name): "The first missing WAL file is " "000000010000000000000003" ) + _, err = capsys.readouterr() + assert backup_info.error in err backup_info_save.reset_mock() # Case 4: the more recent WAL archived is more recent than the end @@ -1957,6 +2176,8 @@ def mock_os_path_exist(file_name): "The first missing WAL file is " "000000010000000000000004" ) + _, err = capsys.readouterr() + assert backup_info.error in err backup_info_save.reset_mock() # Case 4.3: we have all the files, but the backup is marked as @@ -2210,7 +2431,7 @@ def test_put_wal_fsync(self, fd_mock, ff_mock, tmpdir, capsys, caplog): assert ( "Received file '00000001000000EF000000AB' " "with checksum '34743e1e454e967eb76a16c66372b0ef' " - "by put-wal for server 'main'\n" in caplog.text + "by put-wal for server 'main'" in caplog.text ) # Verify fsync calls @@ -2604,6 +2825,101 @@ def test_check_backup_validity_under_minimum_size(self, server, capsys): in out ) + @patch("barman.infofile.LocalBackupInfo.walk_to_root") + @patch("barman.server.Server.get_children_timelines") + @patch("barman.server.Server.get_wal_info") + @patch("barman.backup.BackupManager.get_next_backup") + @patch("barman.backup.BackupManager.get_previous_backup") + def test_get_backup_ext_info( + self, + prev_backup_mock, + next_backup_mock, + wal_info_mock, + children_timeline_mock, + walk_to_root_mock, + ): + """ + Unit test for the get_backup_ext_info method that creates a dict + to be used as an input to render outputs. + + This unit tests checks if all fields read or created are present + in the final dict. + + :param prev_backup_mock: get_previous_backup mock parameter + :param next_backup_mock: get_next_backup mock parameter + :param wal_info_mock: get_wal_info mock parameter + :param children_timeline_mock: get_children_timelines mock parameter + :param walk_to_root_mock: walk_to_root mock parameter + """ + prev_backup_id = prev_backup_mock.return_value.backup_id = "12345" + next_backup_id = next_backup_mock.return_value.backup_id = "12347" + wal_info_mock.return_value = dict( + wal_num=1, + wal_size=1024, + wal_until_next_num=12, + wal_until_next_size=1024, + wal_until_next_compression_ratio=0.5, + wal_compression_ratio=0.5, + ) + children_tlis = children_timeline_mock.return_value = [ + mock.Mock(tli="1"), + mock.Mock(tli="2"), + ] + + server = build_real_server(main_conf={"backup_options": "concurrent_backup"}) + + wtr_list = walk_to_root_mock.return_value = [ + build_test_backup_info( + backup_id="b%s" % i, + server=server, + parent_backup_id=(None if i == 0 else "b" + str(i - 1)), + ) + for i in range(2) + ] + root_backup_id = wtr_list[-1].backup_id + chain_size = len(wtr_list) + + backup_info = build_test_backup_info( + server=server, + backup_id="b2", + summarize_wal="on", + cluster_size=2048, + deduplicated_size=1234, + systemid="systemid", + data_checksums="on", + copy_stats={"analysis_time": 2, "copy_time": 1, "number_of_workers": 2}, + parent_backup_id="b1", + ) + analysis_time = backup_info.copy_stats["analysis_time"] + copy_time = backup_info.copy_stats["copy_time"] + number_of_workers = backup_info.copy_stats["number_of_workers"] + est_throughput = backup_info.deduplicated_size * copy_time + ext_info = server.get_backup_ext_info(backup_info) + key_pairs_check = [ + ("previous_backup_id", prev_backup_id), + ("next_backup_id", next_backup_id), + ("retention_policy_status", None), + ("children_timelines", children_tlis), + ("copy_time", copy_time), + ("analysis_time", analysis_time), + ("estimated_throughput", est_throughput), + ("number_of_workers", number_of_workers), + ("mode", backup_info.mode), + ("parent_backup_id", backup_info.parent_backup_id), + ("children_backup_ids", backup_info.children_backup_ids), + ("cluster_size", backup_info.cluster_size), + ("root_backup_id", root_backup_id), + ("chain_size", chain_size), + ("deduplication_ratio", backup_info.deduplication_ratio), + ( + "est_dedup_size", + backup_info.cluster_size * backup_info.deduplication_ratio, + ), + ("backup_type", backup_info.backup_type), + ] + for field in key_pairs_check: + assert field[0] in ext_info and field[1] == ext_info[field[0]] + class TestCheckStrategy(object): """ diff --git a/tests/test_sync.py b/tests/test_sync.py index 6403e3541..9c6406723 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -47,7 +47,7 @@ "size": 12345, "server_name": "main", "begin_xlog": "0/2000028", - "deduplicated_size": None, + "deduplicated_size": 1024, "version": 90302, "ident_file": "/pgdata/location/pg_ident.conf", "end_time": "Wed Jul 23 12:00:43 2014", @@ -74,6 +74,11 @@ "xlog_segment_size": 16777216, "systemid": None, "compression": None, + "data_checksums": None, + "summarize_wal": None, + "parent_backup_id": None, + "children_backup_ids": None, + "cluster_size": 2048, } }, "config": {}, @@ -747,7 +752,10 @@ def test_passive_node_cron( # Add a backup to the remote response primary_info = dict(EXPECTED_MINIMAL) - backup_info_dict = LocalBackupInfo(server, backup_id="1234567891").to_json() + backup_info_dict = LocalBackupInfo( + server, + backup_id="1234567891", + ).to_json() primary_info["backups"]["1234567891"] = backup_info_dict command_mock.return_value.out = json.dumps(primary_info) server.cron() diff --git a/tests/testing_helpers.py b/tests/testing_helpers.py index db7d8866d..a97bf0214 100644 --- a/tests/testing_helpers.py +++ b/tests/testing_helpers.py @@ -44,8 +44,9 @@ def build_test_backup_info( begin_time=None, begin_wal="000000010000000000000002", begin_xlog="0/2000028", + compression=None, config_file="/pgdata/location/postgresql.conf", - deduplicated_size=None, + deduplicated_size=1024, end_offset=184, end_time=None, end_wal="000000010000000000000002", @@ -69,6 +70,11 @@ def build_test_backup_info( server=None, systemid=None, copy_stats=None, + data_checksums=None, + summarize_wal=None, + parent_backup_id=None, + children_backup_ids=None, + cluster_size=2048, ): """ Create an 'Ad Hoc' BackupInfo object for testing purposes. @@ -101,6 +107,11 @@ def build_test_backup_info( :param int version: postgres version of the backup :param barman.server.Server|None server: Server object for the backup :param dict|None: Copy stats dictionary + :param str|None data_checksums: The checksum state (on/off) + :param str|None: summarize_wal status flag + :param str|None: parent_backup_id status flag + :param list|None: children_backup_ids status flag + :param int|None: cluster_size status flag :rtype: barman.infofile.LocalBackupInfo """ if begin_time is None: @@ -143,6 +154,15 @@ def mock_backup_ext_info( wal_until_next_compression_ratio=0.0, children_timelines=[], copy_stats={}, + root_backup_id=None, + chain_size=None, + est_dedup_size=None, + deduplication_ratio=None, + backup_type=None, + copy_time=None, + analysis_time=None, + number_of_workers=None, + estimated_throughput=None, **kwargs ): # make a dictionary with all the arguments @@ -262,6 +282,7 @@ def build_config_dictionary(config_keys=None): "archiver": True, "archiver_batch_size": 0, "autogenerate_manifest": False, + "aws_await_snapshots_timeout": 3600, "aws_profile": None, "aws_region": None, "azure_credential": None, @@ -293,6 +314,8 @@ def build_config_dictionary(config_keys=None): "gcp_zone": None, "immediate_checkpoint": False, "incoming_wals_directory": "/some/barman/home/main/incoming", + "keepalive_interval": 60, + "local_staging_path": None, "max_incoming_wals_queue": None, "minimum_redundancy": "0", "name": "main", @@ -405,7 +428,9 @@ def build_real_server(global_conf=None, main_conf=None): ) -def build_mocked_server(name=None, config=None, global_conf=None, main_conf=None): +def build_mocked_server( + name=None, config=None, global_conf=None, main_conf=None, pg_version=None +): """ Build a mock server object :param str name: server name, defaults to 'main' @@ -415,6 +440,7 @@ def build_mocked_server(name=None, config=None, global_conf=None, main_conf=None it is possible to override or add new values to the [barman] section :param dict[str,str|None]|None main_conf: using this dictionary it is possible to override/add new values to the [main] section + :param int pg_version: postgres server version, default to ``None`` :rtype: barman.server.Server """ # instantiate a retention policy object using mocked parameters @@ -433,11 +459,17 @@ def build_mocked_server(name=None, config=None, global_conf=None, main_conf=None server.postgres.xlog_segment_size = DEFAULT_XLOG_SEG_SIZE server.path = "/test/bin" server.systemid = "6721602258895701769" + server.postgres.server_version = pg_version return server def build_backup_manager( - server=None, name=None, config=None, global_conf=None, main_conf=None + server=None, + name=None, + config=None, + global_conf=None, + main_conf=None, + pg_version=None, ): """ Instantiate a BackupManager object using mocked parameters @@ -448,7 +480,7 @@ def build_backup_manager( :rtype: barman.backup.BackupManager """ if server is None: - server = build_mocked_server(name, config, global_conf, main_conf) + server = build_mocked_server(name, config, global_conf, main_conf, pg_version) with mock.patch("barman.backup.CompressionManager"): manager = BackupManager(server=server) manager.compression_manager.unidentified_compression = None