Skip to content
CI

Update root readme adding key features, repo structure details, user uide reference, jira reference, details for repo clarity. #2615

Sign in to view logs

Update root readme adding key features, repo structure details, user uide reference, jira reference, details for repo clarity.

Update root readme adding key features, repo structure details, user uide reference, jira reference, details for repo clarity. #2615

Workflow file for this run

.github/workflows/CI.yml at 564ea9c
name: CI
on:
push:
pull_request:
env:
python-version: '3.11'
java-version: '11'
gradle-version: '8.0.2'
node-version: '18.x'
jobs:
style-check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-java@v4
with:
java-version: ${{ env.java-version }}
distribution: "corretto"
- uses: gradle/actions/setup-gradle@v3
with:
gradle-version: ${{ env.gradle-version }}
- name: Run Gradle Build
run: ./gradlew spotlessCheck
env:
OS_MIGRATIONS_GRADLE_SCAN_TOS_AGREE_AND_ENABLED: ''
python-lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ env.python-version }}
- name: Install dependencies
run: |
python3 -m pip install --upgrade pip
python3 -m pip install flake8
- name: Analysing the code with flake8
run: |
flake8 $(git ls-files '*.py')
python-tests:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
py-project:
- ./TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link
defaults:
run:
working-directory: ${{ matrix.py-project }}
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ env.python-version }}
- run: |
python3 -m pip install --upgrade pipenv
pipenv install --deploy --dev
pipenv run coverage run -m pytest
pipenv run coverage xml
- name: Upload to Codecov
uses: codecov/codecov-action@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: ./coverage.xml
flags: python-test
fail_ci_if_error: true
console-api-tests:
runs-on: ubuntu-latest
defaults:
run:
working-directory: ./TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_api
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ env.python-version }}
- run: |
python3 -m pip install --upgrade pipenv
pipenv install --deploy --dev
pipenv run coverage run --source='.' manage.py test console_api
pipenv run coverage xml
- name: Upload to Codecov
uses: codecov/codecov-action@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: ./coverage.xml
fail_ci_if_error: true
gradle-tests:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-java@v4
with:
java-version: ${{ env.java-version }}
distribution: 'corretto'
- uses: gradle/actions/setup-gradle@v3
with:
gradle-version: ${{ env.gradle-version }}
gradle-home-cache-cleanup: true
- name: Run Gradle Build
run: ./gradlew build -x test -x TrafficCapture:dockerSolution:build -x spotlessCheck --stacktrace
env:
OS_MIGRATIONS_GRADLE_SCAN_TOS_AGREE_AND_ENABLED: ''
- name: Run Tests with Coverage
run: ./gradlew mergeJacocoReports -x TrafficCapture:dockerSolution:build -x spotlessCheck --stacktrace
env:
OS_MIGRATIONS_GRADLE_SCAN_TOS_AGREE_AND_ENABLED: ''
- uses: actions/upload-artifact@v4
if: always()
with:
name: test-reports
path: |
**/build/reports/tests/
**/reports/jacoco/mergedReport/
- name: Upload to Codecov
uses: codecov/codecov-action@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: ${{ github.workspace }}/build/reports/jacoco/mergedReport/jacocoMergedReport.xml
flags: gradle-test
fail_ci_if_error: true
disable_search: true
python-e2e-tests:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-java@v4
with:
java-version: ${{ env.java-version }}
distribution: 'corretto'
- uses: gradle/actions/setup-gradle@v3
with:
gradle-version: ${{ env.gradle-version }}
- uses: actions/setup-python@v5
with:
python-version: ${{ env.python-version }}
- name: Generate Cache Key from Dockerfiles
id: generate_cache_key
run: |
files=$(find . -type f \( -name 'docker-compose.yml' -o -name 'Dockerfile' \))
file_contents=$(cat $files)
key=$(echo "${file_contents}" | sha1sum | awk '{print $1}')
echo "key=${key}" >> "$GITHUB_OUTPUT"
- name: Cache Docker Images
uses: ScribeMD/[email protected]
with:
key: docker-${{ runner.os }}-${{ steps.generate_cache_key.outputs.key }}
- name: Start Docker Solution
run: ./gradlew -p TrafficCapture dockerSolution:ComposeUp -x test -x spotlessCheck --info --stacktrace
env:
OS_MIGRATIONS_GRADLE_SCAN_TOS_AGREE_AND_ENABLED: ''
- name: Run E2E test script
working-directory: TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test
run: |
docker exec $(docker ps --filter "name=migration-console" -q) pipenv run pytest /root/lib/integ_test/integ_test/replayer_tests.py --unique_id="testindex" -s
- name: Collect Docker, OpenSearch Benchmark, and Shared Logs
if: always()
run: |
mkdir -p logs/docker logs/opensearch_benchmark_logs logs/shared_logs_output
for container in $(docker ps -aq); do
container_name=$(docker inspect --format '{{.Name}}' $container | sed 's/\///')
docker logs $container > logs/docker/${container_name}_logs.txt 2>&1
done
docker cp $(docker ps --filter "name=migration-console" -q):/root/.benchmark/logs logs/opensearch_benchmark_logs
docker cp $(docker ps --filter "name=migration-console" -q):/shared-logs-output logs/shared_logs_output
- name: Upload Logs
if: always()
uses: actions/upload-artifact@v4
with:
name: e2e-test-logs
path: |
logs/docker
logs/opensearch_benchmark_logs
logs/shared_logs_output
- name: Clean up migrations docker images before caching
run: |
docker stop $(docker ps -q) && docker system prune --volumes -f
docker image ls --format '{{.Repository}}:{{.Tag}}' | grep '^migrations/' | xargs -I {} docker image rm {}
cdk-tests:
runs-on: ubuntu-latest
defaults:
run:
working-directory: ./deployment/cdk/opensearch-service-migration
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: ${{ env.node-version }}
- name: Install NPM dependencies
run: npm ci
- name: Run CDK Jest Tests
run: npm test
link-checker:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: lychee Link Checker
id: lychee
uses: lycheeverse/lychee-action@v1
with:
args: --verbose --accept=200,403,429 "**/*.html" "**/*.md" "**/*.txt" "**/*.json"
--offline
--exclude "file:///github/workspace/*"
--exclude "http://localhost*"
--exclude "https://localhost*"
--exclude "http://capture-proxy*"
--exclude "https://capture-proxy*"
--exclude-mail
fail: true
env:
GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
all-ci-checks-pass:
needs:
- style-check
- cdk-tests
- gradle-tests
- link-checker
- python-e2e-tests
- python-lint
- python-tests
if: always()
runs-on: ubuntu-latest
steps:
- if: ${{ contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'failure') || contains(needs.*.result, 'skipped') }}
run: |
echo "One or more job cancelled, failed, or skipped" && exit 1
- run: |
echo '## :heavy_check_mark: All continous integration checks pass' >> $GITHUB_STEP_SUMMARY