docker-compose.testing.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

# Basic Airflow cluster configuration for CeleryExecutor with Redis and PostgreSQL.
#
# WARNING: This configuration is for local development. Do not use it in a production deployment.
#
# This configuration supports basic configuration using environment variables or an .env file
# See README.md > Environment variable
#
# Feel free to modify this file to suit your needs.
---
version: '3'

x-airflow-common:
  # In order to add custom dependencies or upgrade provider packages you can use your extended image.
  # Comment the image line, place your Dockerfile in the directory where you placed the docker-compose.yaml
  # and uncomment the "build" line below, Then run `docker-compose build` to build the images.
  &airflow-common
  image: importer/airflow
  build:
    context: ./
    dockerfile: ./docker/Dockerfile
    args:
      CUSTOM_UID: ${AIRFLOW_UID:-50000}

  environment:
    &airflow-common-env
    AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${POSTGRES_USER:-airflow}:${POSTGRES_PASSWORD:-airflow}@testing_postgres/airflow
    AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://${POSTGRES_USER:-airflow}:${POSTGRES_PASSWORD:-airflow}@testing_postgres/airflow
    _PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-}
    IMPORTER_MYSQL_HOST: ${IMPORTER_MYSQL_HOST:-testing_importer-mysql}
    IMPORTER_MYSQL_PORT: ${IMPORTER_MYSQL_PORT:-3306}
    IMPORTER_MYSQL_LOGIN: ${IMPORTER_MYSQL_LOGIN:-importer}
    IMPORTER_MYSQL_PASSWORD: ${IMPORTER_MYSQL_PASSWORD:-importer}
    IMPORTER_MYSQL_SCHEMA: importer
  volumes:
    - ${INPUT_DIR:-./importer/var/input}:/var/input
    - ${WORK_DIR:-./importer/var/work}:/var/work
    - ${OUTPUT_DIR:-./importer/var/output}:/var/output
  user: "${AIRFLOW_UID:-50000}:0"
  depends_on:
    &airflow-common-depends-on
    testing_redis:
      condition: service_healthy
    testing_postgres:
      condition: service_healthy
    testing_importer-mysql:
      condition: service_healthy

services:
  testing_postgres:
    image: postgres:13
    environment:
      POSTGRES_USER: ${POSTGRES_USER:-airflow}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-airflow}
      POSTGRES_DB: airflow
    expose:
      - 5432
    healthcheck:
      test: [ "CMD", "pg_isready", "-U", "$${POSTGRES_USER}" ]
      interval: 5s
      retries: 5
    restart: unless-stopped

  testing_importer-mysql:
    image: mariadb:10.5.9-focal
    volumes:
      - ./importer-mysql/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
    expose:
      - 3306
    environment:
      MYSQL_DATABASE: importer
      MYSQL_ROOT_PASSWORD: ${IMPORTER_MYSQL_ROOT_PASSWORD:-importer}
      MYSQL_PASSWORD: ${IMPORTER_MYSQL_PASSWORD:-importer}
      MYSQL_USER: ${IMPORTER_MYSQL_LOGIN:-importer}
    restart: unless-stopped
    healthcheck:
      test:
        [
          "CMD",
          "mysqladmin",
          "ping",
          "--user=$IMPORTER_MYSQL_LOGIN",
          "--password=$IMPORTER_MYSQL_LOGIN",
          "-h",
          "localhost"
        ]
      interval: 5s
      retries: 5

  testing_redis:
    image: redis:latest
    expose:
      - 6379
    healthcheck:
      test: [ "CMD", "redis-cli", "ping" ]
      interval: 5s
      timeout: 30s
      retries: 50
    restart: unless-stopped

  testing_airflow-init:
    <<: *airflow-common
    entrypoint: /bin/bash
    # yamllint disable rule:line-length
    command:
      - ./entrypoints/init.sh
    # yamllint enable rule:line-length
    environment:
      <<: *airflow-common-env
      _AIRFLOW_DB_UPGRADE: 'true'
      _AIRFLOW_WWW_USER_CREATE: 'true'
      _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME:-airflow}
      _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-airflow}
    user: "0:0"
    volumes:
      - ${WORK_DIR:-./importer/var/work}:/var/work
      - ${OUTPUT_DIR:-./importer/var/output}:/var/output

  tests:
    <<: *airflow-common
    entrypoint: /bin/bash
    user: airflow
    command:
      - ./entrypoints/test.sh
      - ${TESTS_PARAM:-}
    volumes:
      # for test results
      - ./testResults:/opt/airflow/testResults
      - ./importer:/opt/airflow/
      - ./docker/entrypoints:/opt/airflow/entrypoints
      - ./pyproject.toml:/opt/airflow/pyproject.toml
      - ./poetry.lock:/opt/airflow/poetry.lock
    environment:
      <<: *airflow-common-env
      TEST_FILES: ${TEST_FILES:-}
      TESTS_PARAM: ${TESTS_PARAM:-}
    depends_on:
      <<: *airflow-common-depends-on
      testing_airflow-init:
        condition: service_completed_successfully