.gitlab-ci.yml

image: aistorage/ci:latest

stages:
  - build
  - test-short
  - test-long

variables:
  MODE: debug # run tests with debug asserts
  NUM_TARGET:
    value: "5"
    description: "Number of targets deployed."
  NUM_PROXY:
    value: "5"
    description: "Number of proxies deployed."
  FS_CNT:
    value: "6"
    description: "Number of disks defined for each target."
  AWS_REGION:
    value: "us-east-2"
    description: "Region of AWS bucket."
  CLOUD_BCKS:
    value: "aws://ais-blossom gs://ais-ci-kube" # NOTE: additionally used: aws://ais-cicd and aws://ais-ci-python
    description: "Set of buckets used when running cloud tests."
  S3_COMPAT_RUN_ALL:
    value: "False"
    description: "Whether to run all s3 compatibility tests or only verified passing ones"
  S3_COMPAT_STRICT:
    value: "True"
    description: "Whether to fail pipeline stage immediately on s3 compatibility test failure"

# Templates

.gather_logs_template: &gather_logs_def
  after_script:
    - make kill # To make sure that nodes flushed the logs.
    - mkdir $CI_PROJECT_DIR/logs
    - find /tmp/ais -type f -name "*\.INFO\.*" -exec cp {} $CI_PROJECT_DIR/logs/ \;
  artifacts:
    when: on_failure
    paths: [ logs/ ]
    expire_in: 1 days

.k8s_long_exit_template: &k8s_long_exit_def
  after_script:
    - make kill
    - mkdir $CI_PROJECT_DIR/logs
    - find /tmp/ais -type f -name "*\.INFO\.*" -exec cp {} $CI_PROJECT_DIR/logs/ \;
  artifacts:
    when: on_failure
    paths: [ logs/ ]
    expire_in: 1 days

.default_only_template: &default_only_def
  only:
    - master
    - merge_requests
    - schedules
    - webs

.test_short_template: &test_short_def
  stage: test-short
  tags:
    - ais
  timeout: 25m
  <<: *default_only_def
  except:
    variables:
      - $CI_MERGE_REQUEST_LABELS =~ /.*skip-ci.*/
      - $CI_MERGE_REQUEST_LABELS =~ /.*python-tests-only.*/
  <<: *gather_logs_def

.test_short_optional_template: &test_short_optional_def
  stage: test-short
  tags:
    - ais
  timeout: 25m
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH == "master" || $CI_PIPELINE_SOURCE == "web"'
      when: manual
      allow_failure: true
  <<: *gather_logs_def

.test_long_template: &test_long_def
  stage: test-long
  tags:
    - ais
  timeout: 3h
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH == "master" || $CI_PIPELINE_SOURCE == "web"'
      when: manual
      allow_failure: true
  <<: *gather_logs_def

# Stages

build:linux:
  stage: build
  tags:
    - ais
  timeout: 10m
  <<: *default_only_def
  script:
    - MODE="" make node # Build node without backends in production mode.
    - MODE="debug" make node # Build node without backends in debug mode.
    - AIS_BACKEND_PROVIDERS="ais aws azure gcp hdfs" MODE="" make node # Build with all backends (production mode).
    - AIS_BACKEND_PROVIDERS="ais aws azure gcp hdfs" MODE="debug" make node # Build with all backends (debug mode).
    - MEM_PROFILE="/tmp/mem" CPU_PROFILE="/tmp/cpu" make node # Build with profile.
    - TAGS="nethttp" make node # Build with net/http transport support (fasthttp is used by default).
    - TAGS="s3rproxy" make node # Build with reverse proxy support (redirect is used by default).
    - make authn
    - make aisfs
    - make cli
    - make aisloader

lint:linux:
  stage: build
  tags:
    - ais
  timeout: 10m
  <<: *default_only_def
  script:
    - make lint
    - make fmt-check
    - make spell-check

# Runs cluster with 5 proxies and 5 targets (each with 6 mountpaths).
test:short:
  <<: *test_short_def
  variables:
    BUCKET: "ais://test"
  script:
    - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --deployment all
    - make test-short
    - FLAGS="--duration=10s" make test-aisloader

# Runs cluster with 5 proxies and 5 targets (each with 6 mountpaths).
test:short:python:
  <<: *test_short_def
  variables:
    BUCKET: "aws://ais-ci-python"
  script:
    - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --deployment all --aws
    - cd python
    - make python_sdk_tests
    - make python_botocore_tests
  except:
    variables:
      - $CI_MERGE_REQUEST_LABELS =~ /.*skip-ci.*/

test:short:python-etl:
  <<: *test_short_def
  tags:
    - ais-k8s
  script:
    - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --deployment all
    - cd python
    - make python_etl_tests
  except:
    variables:
      - $CI_MERGE_REQUEST_LABELS =~ /.*skip-ci.*/
      - $CI_MERGE_REQUEST_LABELS !~ /.*python.*/

# Runs cluster with 1 proxy and 1 target (with 6 mountpaths).
test:short:minimal:
  <<: *test_short_def
  variables:
    BUCKET: "ais://test"
  script:
    - deploy/scripts/clean_deploy.sh --target-cnt 1 --proxy-cnt 1 --mountpath-cnt $FS_CNT --deployment all
    - NUM_PROXY=1 NUM_TARGET=1 make test-short

test:short:s3-compat:
  <<: *test_short_def
  tags:
    - ais
  script:
    - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --deployment all
    - ais config cluster features Provide-S3-API-via-Root
    - ais config cluster checksum.type=md5
    - cd python/tests/s3compat
    - ./compatibility_test.sh
  except:
    variables:
      - $CI_MERGE_REQUEST_LABELS !~ /.*s3-compat.*/

test:short:authn:
  <<: *test_short_optional_def
  variables:
    AIS_AUTHN_ENABLED: "true"
    AIS_AUTHN_SU_NAME: "admin"
    AIS_AUTHN_SU_PASS: "admin"
    AIS_AUTHN_URL: "http://localhost:52001"
    BUCKET: "ais://test"
    RE: "TestAuth"
  script:
    - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT
    - ais auth login $AIS_AUTHN_SU_NAME -p $AIS_AUTHN_SU_PASS
    - make test-run
    - ais auth logout

test:short:https:
  <<: *test_short_optional_def
  variables:
    AIS_USE_HTTPS: "true"
    AIS_SKIP_VERIFY_CRT: "true"
    AIS_SERVER_CRT: "$CI_PROJECT_DIR/localhost.crt"
    AIS_SERVER_KEY: "$CI_PROJECT_DIR/localhost.key"
    AIS_ENDPOINT: "https://localhost:8080"
    BUCKET: "ais://ais-ci"
  script:
    - openssl req -x509 -out $AIS_SERVER_CRT -keyout $AIS_SERVER_KEY -newkey rsa:2048 -nodes -sha256 -subj '/CN=localhost' -extensions EXT -config <( printf "[dn]\nCN=localhost\n[req]\ndistinguished_name = dn\n[EXT]\nsubjectAltName=DNS:localhost\nkeyUsage=digitalSignature\nextendedKeyUsage=serverAuth")
    - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --deployment all --https
    - make test-short

test:short:s3rproxy:
  <<: *test_short_optional_def
  variables:
    BUCKET: "ais://ais-ci"
    TAGS: "s3rproxy"
    RE: "S3"
  script:
    - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT
    - make test-short

test:long:
  <<: *test_long_def
  variables:
    NUM_PROXY: 6
    BUCKET: "ais://ais-ci"
  script:
    - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT
    - make test-long

test:long:aws:
  <<: *test_long_def
  variables:
    NUM_PROXY: 6
    BUCKET: "aws://ais-cicd"
  script:
    - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --aws
    - make test-long

test:long:gcp:
  <<: *test_long_def
  variables:
    NUM_PROXY: 6
    GOOGLE_APPLICATION_CREDENTIALS: "/tmp/gcs.json"
    BUCKET: "gs://ais-ci"
  script:
    - echo "${GOOGLE_APPLICATION_CREDENTIALS_JSON}" > "${GOOGLE_APPLICATION_CREDENTIALS}"
    - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --gcp
    - make test-long

test:long:hdfs:
  image: aistorage/hdfs-ci
  <<: *test_long_def
  variables:
    NUM_PROXY: 6
    BUCKET: "hdfs://ais-ci"
  script:
    - bash deploy/ci/setup_hdfs.sh
    - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --hdfs
    - ais bucket create "${BUCKET}" --props="extra.hdfs.ref_directory=/"
    - make test-long

test:long:aisloader:
  stage: test-long
  tags:
    - ais
  timeout: 10m
  only:
    - schedules
  script:
    - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT
    - sleep 10 # make sure that cluster properly starts
    - FLAGS="--duration=5m" make test-aisloader

#
# Kubernetes stages
#

.test_k8s_short_template: &test_k8s_short_def
  stage: test-short
  tags:
    - ais-k8s
  variables:
    NUM_TARGET: 1
    NUM_PROXY: 1
    GOOGLE_APPLICATION_CREDENTIALS: "/tmp/gcs.json"
    BUCKET: "gs://ais-ci-kube"
    RE: "TestETL|TestConfig|TestMountpath"
    TESTS_DIR: "ais/test"
  before_script:
    - kubectl delete pods,services -l nvidia.com/ais-etl-name # TODO: this can be removed once the lifecycle of transformers is implemented.

.test_k8s_long_template: &test_k8s_long_def
  extends: .test_k8s_short_template
  stage: test-long

test:short:k8s:
  stage: test-short
  <<: *test_k8s_short_def
  timeout: 30m
  only:
    - merge_requests
  except:
    variables:
      - $CI_MERGE_REQUEST_LABELS =~ /.*skip-ci.*/
      - $CI_MERGE_REQUEST_LABELS =~ /.*python-tests-only.*/
  script:
    - export NUM_TARGET=3
    - export NUM_PROXY=3
    - echo "${GOOGLE_APPLICATION_CREDENTIALS_JSON}" > "${GOOGLE_APPLICATION_CREDENTIALS}"
    - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --gcp
    - make test-short
    - status=$? make kill && exit $status

#  e.g. RE: "ETLBucket|ETLConnectionError|ETLInitCode" (or any other regex to select tests)
test:short:assorted:k8s:
  stage: test-short
  tags:
    - ais-k8s
  timeout: 30m
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH == "master" || $CI_PIPELINE_SOURCE == "web"'
      when: manual
      allow_failure: true
  variables:
    GOOGLE_APPLICATION_CREDENTIALS: "/tmp/gcs.json"
    BUCKET: "gs://ais-ci-kube"
    TESTS_DIR: "ais/test"
    RE: "ETLAnyToAny|ETLMultiObj"
  script:
    - export NUM_TARGET=3
    - export NUM_PROXY=3
    - echo "${GOOGLE_APPLICATION_CREDENTIALS_JSON}" > "${GOOGLE_APPLICATION_CREDENTIALS}"
    - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --gcp
    - make test-run
    - status=$? make kill && exit $status

test:long:k8s:
  stage: test-long
  <<: *test_k8s_long_def
  timeout: 3h
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
    - if: '$CI_MERGE_REQUEST_LABELS =~ /.*k8s-ci.*/'
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH == "master" || $CI_PIPELINE_SOURCE == "web"'
      when: manual
      allow_failure: true
  script:
    - export NUM_TARGET=5
    - echo "${GOOGLE_APPLICATION_CREDENTIALS_JSON}" > "${GOOGLE_APPLICATION_CREDENTIALS}"
    - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --gcp
    - make test-run
    - status=$? make kill && exit $status
  <<: *k8s_long_exit_def

test:long:k8s:single-target:
  stage: test-long
  <<: *test_k8s_long_def
  timeout: 3h
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
    - if: '$CI_MERGE_REQUEST_LABELS =~ /.*k8s-ci.*/'
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH == "master" || $CI_PIPELINE_SOURCE == "web"'
      when: manual
      allow_failure: true
  script:
    - echo "${GOOGLE_APPLICATION_CREDENTIALS_JSON}" > "${GOOGLE_APPLICATION_CREDENTIALS}"
    - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --gcp
    - status=0
    - make test-run
    - status=$? make kill && exit $status
  <<: *k8s_long_exit_def

test:long:k8s:aisloader:
  stage: test-long
  <<: *test_k8s_long_def
  timeout: 15m
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
    - if: '$CI_MERGE_REQUEST_LABELS =~ /.*k8s-ci.*/'
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH == "master" || $CI_PIPELINE_SOURCE == "web"'
      when: manual
      allow_failure: true
  script:
    - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT
    - sleep 10 # Give some time for the cluster to stabilize.
    - BUCKET="ais://test" FLAGS="--duration=2m --etl" make test-aisloader
    - status=$? make kill && exit $status

test:long:k8s:all:
  stage: test-long
  tags:
    - ais-k8s
  variables:
    GOOGLE_APPLICATION_CREDENTIALS: "/tmp/gcs.json"
  timeout: 5h
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
    - if: '$CI_MERGE_REQUEST_LABELS =~ /.*k8s-ci.*/'
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH == "master" || $CI_PIPELINE_SOURCE == "web"'
      when: manual
      allow_failure: true
  before_script:
    - kubectl delete pods,services -l nvidia.com/ais-etl-name # TODO: this can be removed once the lifecycle of transformers is implemented.
    # Make sure that metrics collection is enabled.
    - git clone https://github.com/prometheus-operator/kube-prometheus.git
    - kubectl apply -f kube-prometheus/manifests/setup && kubectl apply -f kube-prometheus/manifests && rm -rf kube-prometheus
  script:
    - echo "${GOOGLE_APPLICATION_CREDENTIALS_JSON}" > "${GOOGLE_APPLICATION_CREDENTIALS}"
    - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --gcp --aws
    - |
      echo "----- RUNNING K8S TESTS -----"
      BUCKET="aws://ais-blossom" RE="TestETL|TestConfig|TestMountpath" make test-run
      exit_code=$?
      result=$((result + exit_code))
      echo "----- K8S TESTS FINISHED WITH: ${exit_code} -----"
    - |
      for bucket in ${CLOUD_BCKS}; do
        echo "----- RUNNING LONG TESTS WITH: ${bucket} -----"
        BUCKET=${bucket} make test-long && make test-aisloader
        exit_code=$?
        result=$((result + exit_code))
        echo "----- LONG TESTS FINISHED WITH: ${exit_code} -----"
      done
    - make kill && exit $result
  <<: *k8s_long_exit_def