justfile

set dotenv-load := false

COLOR := "\\033[0;34m"
NO_COLOR := "\\033[0m"

# Show all available recipes
@_default:
    printf "\n{{ COLOR }}# Catalog (path: \`catalog/\`)\n"
    printf "============================{{ NO_COLOR }}\n"
    just --list --unsorted


SERVICE := env_var_or_default("SERVICE", "scheduler")

###########
# Version #
###########

export CATALOG_PY_VERSION := `grep '# PYTHON' requirements-prod.txt | awk -F= '{print $2}'`
export CATALOG_AIRFLOW_VERSION := `grep '^apache-airflow' requirements-prod.txt | awk -F= '{print $3}'`

# Print the required Python version
@py-version:
    echo $CATALOG_PY_VERSION

# Print the current Airflow version
@airflow-version:
    echo $CATALOG_AIRFLOW_VERSION

###########
# Install #
###########

# Create a virtual environment using the project Python version
venv:
    # Invokes `python<version>`, like `python3.10`
    bash -c "python$CATALOG_PY_VERSION -m venv .venv/"

# Check that the active Python version matches the required Python version
check-py-version:
    #!/usr/bin/env python
    import os
    import sys
    current = f"{sys.version_info[0]}.{sys.version_info[1]}"
    required = os.environ.get("CATALOG_PY_VERSION")
    if current != required:
        print(f"Detected Python version {current} but required {required}", file=sys.stderr)
        sys.exit(1)

# Install dependencies
install *args: check-py-version
    python -m pip install -r requirements-dev.txt

######
# Up #
######

# Bring up services specific to the catalog profile
[positional-arguments]
up *flags:
    env COMPOSE_PROFILES="catalog" just ../up "$@"

# Bring up services specific to the catalog profile, except Airflow
[positional-arguments]
up-deps *flags:
    env COMPOSE_PROFILES="catalog_dependencies" just ../up "$@"

# Load sample data into upstream DB
init: up
    cd .. && ./load_sample_data.sh -c

# Tear down all the services and recreate the ones of the catalog profile
recreate:
    just ../down -v
    just up --force-recreate --build
    just init

##################
# Administration #
##################

# Launch a Bash shell in an existing container under `SERVICE`
shell:
    env DC_USER="airflow" just ../exec {{ SERVICE }} /bin/bash

# Run an Airflow CLI command and then exit
[positional-arguments]
cli *args:
    env DC_USER="airflow" just ../exec {{ SERVICE }} "$@"

# Launch an IPython shell in a new container under `SERVICE`
[positional-arguments]
ipython *args: up-deps
    env DC_USER="airflow" just ../run \
        --workdir /opt/airflow/catalog/dags \
        {{ SERVICE }} \
        bash -c "ipython ${@:2}"

# Launch a `pgcli` shell in the PostgreSQL container
pgcli db_user_pass="deploy" db_name="openledger": up
    just ../_pgcli upstream_db {{ db_user_pass }} {{ db_name }} upstream_db

#########
# Tests #
#########

# Run a command in a test container under `SERVICE`
[positional-arguments]
_mount-test *command: up-deps
    env DC_USER="airflow" just ../run \
        -e AIRFLOW_VAR_INGESTION_LIMIT=1000000 \
        -w /opt/airflow/catalog \
        --volume {{ justfile_directory() }}/../docker:/opt/airflow/docker/ \
        {{ SERVICE }} \
        "$@"

# Launch a Bash shell in a test container under `SERVICE`
# Run pytest with `--pdb` to workaround xdist breaking pdb.set_trace()
test-session:
    just _mount-test bash

# Run pytest in a test container under `SERVICE`
[positional-arguments]
test *args:
    just _mount-test bash -c "pytest $@"

#############
# Utilities #
#############

# Generate the documentation (either "dag" or "media-props")
generate-docs doc="dag" fail_on_diff="false":
    #! /usr/bin/env bash
    set -e
    if [ "{{ doc }}" == "dag" ]; then
        SCRIPT_PATH="catalog/utilities/dag_doc_gen/dag_doc_generation.py"
        GENERATED_REL_PATH="utilities/dag_doc_gen/DAGs.md"
        FINAL_FILE="documentation/catalog/reference/DAGs.md"
        NAME="DAG"
    elif [ "{{ doc }}" == "media-props" ]; then
        SCRIPT_PATH="catalog/utilities/media_props_gen/generate_media_properties.py"
        GENERATED_REL_PATH="utilities/media_props_gen/media_properties.md"
        FINAL_FILE="documentation/meta/media_properties/catalog.md"
        NAME="Media properties"
    else
        echo "Invalid documentation type specified, use \`dag\` or \`media-props\`. Exiting."
        exit 1
    fi

    GENERATED_ABS_PATH="/opt/airflow/catalog/${GENERATED_REL_PATH}"
    just ../run \
      --volume {{ justfile_directory() }}/../docker:/opt/airflow/docker/ \
      -e PYTHONPATH=/opt/airflow/catalog:/opt/airflow/catalog/dags \
      {{ SERVICE }} \
      bash -c "python $SCRIPT_PATH && chmod 666 $GENERATED_ABS_PATH"

    TEMP="documentation/meta/temp"
    mv ../catalog/"$GENERATED_REL_PATH" ../"$TEMP".md
    echo "Moved the generated file to ../$TEMP.md"
    echo -n "Running linting..."
    # Linting step afterwards is necessary since the generated output differs greatly from what prettier expects
    just ../lint prettier "$TEMP".md &>/dev/null || true
    echo "Linting done!"

    echo -n "Replacing linted md <hr> '---' with '----' required by sphinx..."
    sed 's/^---$/----/' "../$TEMP.md" > "../$TEMP.tmp"
    mv "../$TEMP.tmp" "../$TEMP.md"
    echo "Replacements done!"

    mv ../$TEMP.md ../$FINAL_FILE
    echo "Moved the generated file to ../$FINAL_FILE"
    if {{ fail_on_diff }}; then
      set +e
      git diff --exit-code -- ../$FINAL_FILE
      if [ $? -ne 0 ]; then
          printf "\n\n\e[31m!! Changes found in $NAME documentation, please run 'just catalog/generate-docs {{ doc }}' locally and commit difference !!\n\n"
          exit 1
      fi
    fi

# Generate files for a new provider
add-provider provider_name endpoint +media_types="image":
    python3 templates/create_provider_ingester.py "{{ provider_name }}" "{{ endpoint }}" -m {{ media_types }}

# Run bash in the container set in the SERVICE env-var
run:
    just ../run {{ SERVICE }} bash