From e5393a9d74aced67766edb564ea7c2755cf8c80b Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 30 Oct 2024 09:35:59 +0100 Subject: [PATCH 01/12] Add DE and FR testing configurations --- build/config/rmi_pacta_2023q4_pa2024ch.json | 25 +++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/build/config/rmi_pacta_2023q4_pa2024ch.json b/build/config/rmi_pacta_2023q4_pa2024ch.json index 95af8093..bc093a93 100644 --- a/build/config/rmi_pacta_2023q4_pa2024ch.json +++ b/build/config/rmi_pacta_2023q4_pa2024ch.json @@ -10,6 +10,20 @@ "peer_group": ["assetmanager", "bank", "insurance", "other", "pensionfund"], "user_results": ["https://pactadatadev.blob.core.windows.net/pa2024ch/outputs/user_data"], "include": [ + { + "user_id": 128, + "peer_group": "bank" + }, + { + "user_id": 128, + "peer_group": "bank", + "language": "DE" + }, + { + "user_id": 128, + "peer_group": "bank", + "language": "FR" + }, { "user_id": 128, "peer_group": "bank" @@ -29,7 +43,18 @@ { "user_id": 135, "peer_group": "other" + }, + { + "user_id": 135, + "peer_group": "other", + "language": "DE" + }, + { + "user_id": 135, + "peer_group": "other", + "language": "FR" } + ] }, "resultsDestinationURL": "https://pactadatadev.blob.core.windows.net/ghactions-workflow-transition-monitor-results-full", From 8ee461055a4e907fcae7281d7b5e97edb9dfae04 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 30 Oct 2024 10:16:50 +0100 Subject: [PATCH 02/12] Make all language/peer group combinations explicit --- build/config/rmi_pacta_2023q4_pa2024ch.json | 22 ++++++++++----------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/build/config/rmi_pacta_2023q4_pa2024ch.json b/build/config/rmi_pacta_2023q4_pa2024ch.json index bc093a93..e446c52a 100644 --- a/build/config/rmi_pacta_2023q4_pa2024ch.json +++ b/build/config/rmi_pacta_2023q4_pa2024ch.json @@ -6,13 +6,12 @@ "peer_results": "https://pactadatadev.blob.core.windows.net/project-files/pa2024ch/peer_files", "net_zero_targets": "https://pactadatadev.blob.core.windows.net/project-files/pa2024ch/fin_data_net_zero_targets.csv", "test_matrix": { - "language": ["EN"], - "peer_group": ["assetmanager", "bank", "insurance", "other", "pensionfund"], "user_results": ["https://pactadatadev.blob.core.windows.net/pa2024ch/outputs/user_data"], "include": [ { "user_id": 128, - "peer_group": "bank" + "peer_group": "bank", + "language": "EN" }, { "user_id": 128, @@ -24,25 +23,25 @@ "peer_group": "bank", "language": "FR" }, - { - "user_id": 128, - "peer_group": "bank" - }, { "user_id": 130, - "peer_group": "assetmanager" + "peer_group": "assetmanager", + "language": "EN" }, { "user_id": 131, - "peer_group": "insurance" + "peer_group": "insurance", + "language": "EN" }, { "user_id": 134, - "peer_group": "pensionfund" + "peer_group": "pensionfund", + "language": "EN" }, { "user_id": 135, - "peer_group": "other" + "peer_group": "other", + "language": "EN" }, { "user_id": 135, @@ -54,7 +53,6 @@ "peer_group": "other", "language": "FR" } - ] }, "resultsDestinationURL": "https://pactadatadev.blob.core.windows.net/ghactions-workflow-transition-monitor-results-full", From 8792f1da939b4d054d0e0e5abc9c68fdbe71725e Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 30 Oct 2024 11:13:52 +0100 Subject: [PATCH 03/12] Minimize pipeline for testing matrix --- .github/workflows/build-push-private.yml | 806 +++++++++++------------ 1 file changed, 403 insertions(+), 403 deletions(-) diff --git a/.github/workflows/build-push-private.yml b/.github/workflows/build-push-private.yml index 5c2beb24..6c7d0915 100644 --- a/.github/workflows/build-push-private.yml +++ b/.github/workflows/build-push-private.yml @@ -103,220 +103,220 @@ jobs: echo "net-zero-targets=$NET_ZERO_TARGETS" echo "net-zero-targets=$NET_ZERO_TARGETS" >> "$GITHUB_OUTPUT" - prepare-indices: - name: Run Index Preparation - needs: read-config - secrets: inherit - uses: RMI-PACTA/workflow.prepare.pacta.indices/.github/workflows/run-index-preparation.yml@main - with: - image-tag: | - ${{ github.event_name == 'pull_request' && format('{0}{1}','pr', github.event.pull_request.number) || - github.event_name == 'schedule' && 'main' || - github.event_name == 'push' && github.ref_name || - github.event_name == 'workflow_dispatch' && github.ref_name || - github.sha }} - data-share-path: ${{ needs.read-config.outputs.pacta-data-share-path }} - config_active: ${{ needs.read-config.outputs.pacta-data-quarter }} - - docker-build: - runs-on: ubuntu-latest - needs: [read-config, prepare-indices] - permissions: - contents: read - id-token: write - timeout-minutes: 45 - outputs: - full-image-name: ${{ steps.export-outputs.outputs.full-image-name }} - - steps: - - name: Checkout workflow.transition.monitor - uses: actions/checkout@v4 - with: - path: workflow.transition.monitor - - - name: Checkout templates.transition.monitor - uses: actions/checkout@v4 - with: - repository: RMI-PACTA/templates.transition.monitor - path: templates.transition.monitor - token: ${{ secrets.REPO_PAT }} - ref: ${{ needs.read-config.outputs.templates-ref }} - - # https://github.com/Azure/login?tab=readme-ov-file#login-with-openid-connect-oidc-recommended - - name: Azure Login - uses: azure/login@v2 - with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - # https://github.com/marketplace/actions/azure-cli-action#workflow-to-execute-an-azure-cli-script-of-a-specific-cli-version - - name: Download pacta-data - uses: azure/CLI@v2 - env: - index_share_path: ${{ needs.prepare-indices.outputs.results-dir }} - pacta_data_quarter: ${{ needs.read-config.outputs.pacta-data-quarter }} - pacta_data_share_path: ${{ needs.read-config.outputs.pacta-data-share-path }} - peer_results: ${{ needs.read-config.outputs.peer-results }} - net_zero_targets: ${{ needs.read-config.outputs.net-zero-targets }} - with: - # azcliversion: 2.30.0 - inlineScript: | - pacta_data_share_url="https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs" - az storage copy \ - --source "$pacta_data_share_url/$pacta_data_share_path" \ - --destination "pacta-data" \ - --recursive \ - --exclude-pattern "*.sqlite" - # download indices - index_share_url="https://pactadatadev.file.core.windows.net/workflow-prepare-pacta-indices-outputs" - az storage copy \ - --source "$index_share_url/$index_share_path" \ - --destination "index-data" \ - --recursive \ - --exclude-pattern "*.sqlite" - - # Download peer results if necessary - if [ -n "$peer_results" ]; then - az storage copy \ - --source "$peer_results/*" \ - --destination "peer_results" \ - --recursive - else - echo "No Peer Results defined in config" - fi - - # Download net zero targets if necessary - if [ -n "$net_zero_targets" ]; then - az storage copy \ - --source "$net_zero_targets" \ - --destination "peer_results" \ - --recursive - else - echo "No Peer Results defined in config" - fi - - # move data - mv "pacta-data/$pacta_data_share_path" "pacta-data/$pacta_data_quarter" - mv "index-data/$index_share_path"/* "pacta-data/$pacta_data_quarter" - if [ -n "$peer_results" ]; then - mv peer_results/* "pacta-data/$pacta_data_quarter" - fi - ls pacta-data/$pacta_data_quarter - - - name: Identify LABELs in dockerfile - id: custom-labels - run: | - DOCKERFILE_LABELS="$(grep "^LABEL" workflow.transition.monitor/Dockerfile | sed 's/^LABEL[[:space:]]*//')" - echo "$DOCKERFILE_LABELS" - echo 'dockerfile-labels<> $GITHUB_OUTPUT - echo "$DOCKERFILE_LABELS" >> $GITHUB_OUTPUT - echo 'EOF' >> $GITHUB_OUTPUT - - # Setup docker metadata, including tags and labels (and annotations) - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - env: - additional_image_tags: ${{ inputs.additional-image-tags }} - now: ${{ needs.read-config.outputs.now }} - with: - images: ${{ needs.read-config.outputs.registry-image }} - annotations: - ${{ steps.custom-labels.outputs.DOCKERFILE_LABELS }} - labels: - ${{ steps.custom-labels.outputs.DOCKERFILE_LABELS }} - tags: | - type=schedule,enable=true,pattern={{date 'YYYYMMDD[T]HHmmss[Z]' tz='UTC'}} - type=schedule,enable=true,pattern=nightly,priority=950 - type=raw,value=latest,enable={{is_default_branch}} - type=ref,event=branch - type=ref,event=tag - type=ref,event=pr - ${{ inputs.additional-image-tags }} - ${{ needs.read-config.outputs.now }},priority=1100 - - # Set up Dockerignore file - - name: Copy dockerignore - run: | - cp ./workflow.transition.monitor/.dockerignore.CIbuild .dockerignore - - # set up our build environment - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#example-masking-a-generated-output-within-a-single-job - - name: Get credentials for docker registry - id: docker-credentials - uses: azure/CLI@v2 - env: - registry: ${{ inputs.registry }} - with: - # azcliversion: 2.30.0 - inlineScript: | - # Username is not a placeholder. See https://stackoverflow.com/a/73250630 - DOCKER_USER="00000000-0000-0000-0000-000000000000" >> "$GITHUB_OUTPUT" - DOCKER_TOKEN="$(az acr login -n $registry --expose-token --query 'accessToken' --output tsv)" >> "$GITHUB_OUTPUT" - echo "::add-mask::$DOCKER_TOKEN" - echo "docker-user=$DOCKER_USER" >> "$GITHUB_OUTPUT" - echo "docker-token=$DOCKER_TOKEN" >> "$GITHUB_OUTPUT" - - - name: Login to Registry - env: - REGISTRY: ${{ inputs.registry }} - DOCKER_TOKEN: ${{ steps.docker-credentials.outputs.docker-token }} - DOCKER_USER: ${{ steps.docker-credentials.outputs.docker-user }} - run: | - echo "$DOCKER_TOKEN" | docker login $REGISTRY --username "$DOCKER_USER" --password-stdin - - # Actually build the image (for a single architecture)! - - name: Build - id: build - uses: docker/build-push-action@v6 - with: - context: . - file: ./workflow.transition.monitor/Dockerfile - # include the labels from the meta step - labels: ${{ steps.meta.outputs.labels }} - # the same for annotations - annotations: ${{ steps.meta.outputs.annotations }} - push: true - tags: ${{ steps.meta.outputs.tags }} - # Use the GitHub actions cache to speed up repeated builds - cache-from: type=gha - cache-to: type=gha,mode=min - # but don't cache the install pacta step. - no-cache-filters: install-pacta - - - name: Export Outputs - id: export-outputs - run: | - TAGGED_IMAGE="$(jq -rc '.tags[0]' <<< "$DOCKER_METADATA_OUTPUT_JSON")" - echo "full-image-name=$TAGGED_IMAGE" - echo "full-image-name=$TAGGED_IMAGE" >> "$GITHUB_OUTPUT" - - - name: Prepare inputs archive - id: prepare-inputs-archive - env: - full_image_name: ${{ steps.export-outputs.outputs.full-image-name }} - run: | - tarfile="$(echo $full_image_name | sed 's:.*/::' | tr ":" "-").tar.gz" - tar -cvz -f "$tarfile" pacta-data templates.transition.monitor - echo "$tarfile" - echo "input-tar=$tarfile" - echo "input-tar=$tarfile" >> "$GITHUB_OUTPUT" - - - - name: Export inputs archive - id: upload-report - uses: RMI-PACTA/actions/actions/azure/blob-copy@main - with: - source: ${{ steps.prepare-inputs-archive.outputs.input-tar }} - destination: https://pactadatadev.blob.core.windows.net/ghactions-workflow-transition-monitor-input-pacta-data - overwrite: false + # prepare-indices: + # name: Run Index Preparation + # needs: read-config + # secrets: inherit + # uses: RMI-PACTA/workflow.prepare.pacta.indices/.github/workflows/run-index-preparation.yml@main + # with: + # image-tag: | + # ${{ github.event_name == 'pull_request' && format('{0}{1}','pr', github.event.pull_request.number) || + # github.event_name == 'schedule' && 'main' || + # github.event_name == 'push' && github.ref_name || + # github.event_name == 'workflow_dispatch' && github.ref_name || + # github.sha }} + # data-share-path: ${{ needs.read-config.outputs.pacta-data-share-path }} + # config_active: ${{ needs.read-config.outputs.pacta-data-quarter }} + + # docker-build: + # runs-on: ubuntu-latest + # needs: [read-config, prepare-indices] + # permissions: + # contents: read + # id-token: write + # timeout-minutes: 45 + # outputs: + # full-image-name: ${{ steps.export-outputs.outputs.full-image-name }} + + # steps: + # - name: Checkout workflow.transition.monitor + # uses: actions/checkout@v4 + # with: + # path: workflow.transition.monitor + + # - name: Checkout templates.transition.monitor + # uses: actions/checkout@v4 + # with: + # repository: RMI-PACTA/templates.transition.monitor + # path: templates.transition.monitor + # token: ${{ secrets.REPO_PAT }} + # ref: ${{ needs.read-config.outputs.templates-ref }} + + # # https://github.com/Azure/login?tab=readme-ov-file#login-with-openid-connect-oidc-recommended + # - name: Azure Login + # uses: azure/login@v2 + # with: + # client-id: ${{ secrets.AZURE_CLIENT_ID }} + # tenant-id: ${{ secrets.AZURE_TENANT_ID }} + # subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + # # https://github.com/marketplace/actions/azure-cli-action#workflow-to-execute-an-azure-cli-script-of-a-specific-cli-version + # - name: Download pacta-data + # uses: azure/CLI@v2 + # env: + # index_share_path: ${{ needs.prepare-indices.outputs.results-dir }} + # pacta_data_quarter: ${{ needs.read-config.outputs.pacta-data-quarter }} + # pacta_data_share_path: ${{ needs.read-config.outputs.pacta-data-share-path }} + # peer_results: ${{ needs.read-config.outputs.peer-results }} + # net_zero_targets: ${{ needs.read-config.outputs.net-zero-targets }} + # with: + # # azcliversion: 2.30.0 + # inlineScript: | + # pacta_data_share_url="https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs" + # az storage copy \ + # --source "$pacta_data_share_url/$pacta_data_share_path" \ + # --destination "pacta-data" \ + # --recursive \ + # --exclude-pattern "*.sqlite" + # # download indices + # index_share_url="https://pactadatadev.file.core.windows.net/workflow-prepare-pacta-indices-outputs" + # az storage copy \ + # --source "$index_share_url/$index_share_path" \ + # --destination "index-data" \ + # --recursive \ + # --exclude-pattern "*.sqlite" + + # # Download peer results if necessary + # if [ -n "$peer_results" ]; then + # az storage copy \ + # --source "$peer_results/*" \ + # --destination "peer_results" \ + # --recursive + # else + # echo "No Peer Results defined in config" + # fi + + # # Download net zero targets if necessary + # if [ -n "$net_zero_targets" ]; then + # az storage copy \ + # --source "$net_zero_targets" \ + # --destination "peer_results" \ + # --recursive + # else + # echo "No Peer Results defined in config" + # fi + + # # move data + # mv "pacta-data/$pacta_data_share_path" "pacta-data/$pacta_data_quarter" + # mv "index-data/$index_share_path"/* "pacta-data/$pacta_data_quarter" + # if [ -n "$peer_results" ]; then + # mv peer_results/* "pacta-data/$pacta_data_quarter" + # fi + # ls pacta-data/$pacta_data_quarter + + # - name: Identify LABELs in dockerfile + # id: custom-labels + # run: | + # DOCKERFILE_LABELS="$(grep "^LABEL" workflow.transition.monitor/Dockerfile | sed 's/^LABEL[[:space:]]*//')" + # echo "$DOCKERFILE_LABELS" + # echo 'dockerfile-labels<> $GITHUB_OUTPUT + # echo "$DOCKERFILE_LABELS" >> $GITHUB_OUTPUT + # echo 'EOF' >> $GITHUB_OUTPUT + + # # Setup docker metadata, including tags and labels (and annotations) + # - name: Docker meta + # id: meta + # uses: docker/metadata-action@v5 + # env: + # additional_image_tags: ${{ inputs.additional-image-tags }} + # now: ${{ needs.read-config.outputs.now }} + # with: + # images: ${{ needs.read-config.outputs.registry-image }} + # annotations: + # ${{ steps.custom-labels.outputs.DOCKERFILE_LABELS }} + # labels: + # ${{ steps.custom-labels.outputs.DOCKERFILE_LABELS }} + # tags: | + # type=schedule,enable=true,pattern={{date 'YYYYMMDD[T]HHmmss[Z]' tz='UTC'}} + # type=schedule,enable=true,pattern=nightly,priority=950 + # type=raw,value=latest,enable={{is_default_branch}} + # type=ref,event=branch + # type=ref,event=tag + # type=ref,event=pr + # ${{ inputs.additional-image-tags }} + # ${{ needs.read-config.outputs.now }},priority=1100 + + # # Set up Dockerignore file + # - name: Copy dockerignore + # run: | + # cp ./workflow.transition.monitor/.dockerignore.CIbuild .dockerignore + + # # set up our build environment + # - name: Set up QEMU + # uses: docker/setup-qemu-action@v3 + + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v3 + + # # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#example-masking-a-generated-output-within-a-single-job + # - name: Get credentials for docker registry + # id: docker-credentials + # uses: azure/CLI@v2 + # env: + # registry: ${{ inputs.registry }} + # with: + # # azcliversion: 2.30.0 + # inlineScript: | + # # Username is not a placeholder. See https://stackoverflow.com/a/73250630 + # DOCKER_USER="00000000-0000-0000-0000-000000000000" >> "$GITHUB_OUTPUT" + # DOCKER_TOKEN="$(az acr login -n $registry --expose-token --query 'accessToken' --output tsv)" >> "$GITHUB_OUTPUT" + # echo "::add-mask::$DOCKER_TOKEN" + # echo "docker-user=$DOCKER_USER" >> "$GITHUB_OUTPUT" + # echo "docker-token=$DOCKER_TOKEN" >> "$GITHUB_OUTPUT" + + # - name: Login to Registry + # env: + # REGISTRY: ${{ inputs.registry }} + # DOCKER_TOKEN: ${{ steps.docker-credentials.outputs.docker-token }} + # DOCKER_USER: ${{ steps.docker-credentials.outputs.docker-user }} + # run: | + # echo "$DOCKER_TOKEN" | docker login $REGISTRY --username "$DOCKER_USER" --password-stdin + + # # Actually build the image (for a single architecture)! + # - name: Build + # id: build + # uses: docker/build-push-action@v6 + # with: + # context: . + # file: ./workflow.transition.monitor/Dockerfile + # # include the labels from the meta step + # labels: ${{ steps.meta.outputs.labels }} + # # the same for annotations + # annotations: ${{ steps.meta.outputs.annotations }} + # push: true + # tags: ${{ steps.meta.outputs.tags }} + # # Use the GitHub actions cache to speed up repeated builds + # cache-from: type=gha + # cache-to: type=gha,mode=min + # # but don't cache the install pacta step. + # no-cache-filters: install-pacta + + # - name: Export Outputs + # id: export-outputs + # run: | + # TAGGED_IMAGE="$(jq -rc '.tags[0]' <<< "$DOCKER_METADATA_OUTPUT_JSON")" + # echo "full-image-name=$TAGGED_IMAGE" + # echo "full-image-name=$TAGGED_IMAGE" >> "$GITHUB_OUTPUT" + + # - name: Prepare inputs archive + # id: prepare-inputs-archive + # env: + # full_image_name: ${{ steps.export-outputs.outputs.full-image-name }} + # run: | + # tarfile="$(echo $full_image_name | sed 's:.*/::' | tr ":" "-").tar.gz" + # tar -cvz -f "$tarfile" pacta-data templates.transition.monitor + # echo "$tarfile" + # echo "input-tar=$tarfile" + # echo "input-tar=$tarfile" >> "$GITHUB_OUTPUT" + + + # - name: Export inputs archive + # id: upload-report + # uses: RMI-PACTA/actions/actions/azure/blob-copy@main + # with: + # source: ${{ steps.prepare-inputs-archive.outputs.input-tar }} + # destination: https://pactadatadev.blob.core.windows.net/ghactions-workflow-transition-monitor-input-pacta-data + # overwrite: false test: runs-on: ubuntu-latest @@ -327,7 +327,7 @@ jobs: matrix: ${{ fromJson(needs.read-config.outputs.test-matrix) }} needs: - read-config - - docker-build + # - docker-build permissions: contents: read id-token: write @@ -348,51 +348,51 @@ jobs: - name: Checkout workflow.transition.monitor uses: actions/checkout@v4 - - name: Prepare User and results directories - id: prepare-results-dir - env: - full_image_name: ${{ needs.docker-build.outputs.full-image-name }} - image_name: ${{ inputs.image-name }} - language: ${{ matrix.language }} - pacta_data_quarter: ${{ needs.read-config.outputs.pacta-data-quarter }} - peer_group: ${{ matrix.peer_group }} - project_code: ${{ needs.read-config.outputs.project-code }} - run_attempt: ${{ github.run_attempt }} - user_id: ${{ matrix.user_id }} - run: | - test_dir_parent_name="$(echo $full_image_name | sed 's:.*/::' | tr ":" "-")" - TEST_DIR_PARENT="$test_dir_parent_name" - echo "TEST_DIR_PARENT=$TEST_DIR_PARENT" - echo "test-dir-parent=$TEST_DIR_PARENT" >> "$GITHUB_OUTPUT" - - TEST_DIR="$TEST_DIR_PARENT/$language/$peer_group/$run_attempt" - mkdir -p "$TEST_DIR" - chmod -R 777 "$TEST_DIR" - echo "TEST_DIR=$TEST_DIR" - echo "test-dir=$TEST_DIR" >> "$GITHUB_OUTPUT" - - mkdir -p $TEST_DIR/user_dir - cp -R working_dir $TEST_DIR - - params_file="$TEST_DIR/working_dir/10_Parameter_File/${image_name}_PortfolioParameters.yml" - echo $params_file - mkdir -p "$(dirname $params_file)" - sed \ - -e "s/{{portfolio_name}}/$image_name-PORTFOLIO/" \ - -e "s/{{investor_name}}/$image_name-$peer_group-INVESTOR/" \ - -e "s/{{peer_group}}/$peer_group/" \ - -e "s/{{language}}/$language/" \ - -e "s/{{project_code}}/$project_code/" \ - -e "s/{{holdings_date}}/$pacta_data_quarter/" \ - -e "s/{{user_id}}/$user_id/" \ - working_dir/10_Parameter_File/test_PortfolioParameters.yml \ - >> $params_file - cat "$params_file" - - portfolio_file="$TEST_DIR/working_dir/20_Raw_Inputs/${image_name}.csv" - mkdir -p "$(dirname $portfolio_file)" - mv working_dir/20_Raw_Inputs/1234.csv $portfolio_file - ls -lR $TEST_DIR + # - name: Prepare User and results directories + # id: prepare-results-dir + # env: + # full_image_name: ${{ needs.docker-build.outputs.full-image-name }} + # image_name: ${{ inputs.image-name }} + # language: ${{ matrix.language }} + # pacta_data_quarter: ${{ needs.read-config.outputs.pacta-data-quarter }} + # peer_group: ${{ matrix.peer_group }} + # project_code: ${{ needs.read-config.outputs.project-code }} + # run_attempt: ${{ github.run_attempt }} + # user_id: ${{ matrix.user_id }} + # run: | + # test_dir_parent_name="$(echo $full_image_name | sed 's:.*/::' | tr ":" "-")" + # TEST_DIR_PARENT="$test_dir_parent_name" + # echo "TEST_DIR_PARENT=$TEST_DIR_PARENT" + # echo "test-dir-parent=$TEST_DIR_PARENT" >> "$GITHUB_OUTPUT" + + # TEST_DIR="$TEST_DIR_PARENT/$language/$peer_group/$run_attempt" + # mkdir -p "$TEST_DIR" + # chmod -R 777 "$TEST_DIR" + # echo "TEST_DIR=$TEST_DIR" + # echo "test-dir=$TEST_DIR" >> "$GITHUB_OUTPUT" + + # mkdir -p $TEST_DIR/user_dir + # cp -R working_dir $TEST_DIR + + # params_file="$TEST_DIR/working_dir/10_Parameter_File/${image_name}_PortfolioParameters.yml" + # echo $params_file + # mkdir -p "$(dirname $params_file)" + # sed \ + # -e "s/{{portfolio_name}}/$image_name-PORTFOLIO/" \ + # -e "s/{{investor_name}}/$image_name-$peer_group-INVESTOR/" \ + # -e "s/{{peer_group}}/$peer_group/" \ + # -e "s/{{language}}/$language/" \ + # -e "s/{{project_code}}/$project_code/" \ + # -e "s/{{holdings_date}}/$pacta_data_quarter/" \ + # -e "s/{{user_id}}/$user_id/" \ + # working_dir/10_Parameter_File/test_PortfolioParameters.yml \ + # >> $params_file + # cat "$params_file" + + # portfolio_file="$TEST_DIR/working_dir/20_Raw_Inputs/${image_name}.csv" + # mkdir -p "$(dirname $portfolio_file)" + # mv working_dir/20_Raw_Inputs/1234.csv $portfolio_file + # ls -lR $TEST_DIR - name: Download User Results from Blob store id: download-user-results @@ -418,146 +418,146 @@ jobs: $USER_RESULTS_DIR fi - # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#example-masking-a-generated-output-within-a-single-job - - name: Get credentials for docker registry - id: docker-credentials - uses: azure/CLI@v2 - env: - registry: ${{ inputs.registry }} - with: - # azcliversion: 2.30.0 - inlineScript: | - # Username is not a placeholder. See https://stackoverflow.com/a/73250630 - DOCKER_USER="00000000-0000-0000-0000-000000000000" >> "$GITHUB_OUTPUT" - DOCKER_TOKEN="$(az acr login -n $registry --expose-token --query 'accessToken' --output tsv)" >> "$GITHUB_OUTPUT" - echo "::add-mask::$DOCKER_TOKEN" - echo "docker-user=$DOCKER_USER" >> "$GITHUB_OUTPUT" - echo "docker-token=$DOCKER_TOKEN" >> "$GITHUB_OUTPUT" - - - name: Login to Registry - env: - REGISTRY: ${{ inputs.registry }} - DOCKER_TOKEN: ${{ steps.docker-credentials.outputs.docker-token }} - DOCKER_USER: ${{ steps.docker-credentials.outputs.docker-user }} - run: | - echo "$DOCKER_TOKEN" | docker login $REGISTRY --username "$DOCKER_USER" --password-stdin - - - name: Test - env: - full_image_name: ${{ needs.docker-build.outputs.full-image-name }} - image_name: ${{ inputs.image-name }} - test_dir: ${{ steps.prepare-results-dir.outputs.test-dir }} - workspace: ${{ github.workspace }} - run: | - chmod -R 777 "$test_dir/working_dir" - docker run \ - --network none \ - --user root \ - --mount "type=bind,source=$workspace/$test_dir/working_dir,target=/bound/working_dir" \ - --mount "type=bind,readonly,source=$workspace/$test_dir/user_dir,target=/user_results" \ - $full_image_name \ - /bound/bin/run-r-scripts $image_name - - # https://github.com/Azure/login?tab=readme-ov-file#login-with-openid-connect-oidc-recommended - - name: Azure Login - uses: azure/login@v2 - with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - - name: Upload results to Blob store - id: upload-results - uses: RMI-PACTA/actions/actions/azure/blob-copy@main - with: - source: ${{ steps.prepare-results-dir.outputs.test-dir-parent }} - destination: https://pactadatadev.blob.core.windows.net/ghactions-workflow-transition-monitor-results-full - overwrite: false - - - name: Upload report to Blob store - id: upload-report - uses: RMI-PACTA/actions/actions/azure/blob-copy@main - with: - source: ${{ steps.prepare-results-dir.outputs.test-dir-parent }}/${{ matrix.language }}/${{ matrix.peer_group }}/${{ github.run_attempt }}/working_dir/50_Outputs - destination: https://pactadatadev.blob.core.windows.net/ghactions-workflow-transition-monitor-results-reports - overwrite: false - - - name: Export Outputs - id: export-outputs - env: - REPORT_UPLOADED_FILES: ${{ steps.upload-report.outputs.destination-files }} - image_name: ${{ inputs.image-name }} - test_dir: ${{ steps.prepare-results-dir.outputs.test-dir }} - run: | - - REPORT_URL="$( - echo "$REPORT_UPLOADED_FILES" | jq -rc '. [] | match(".*index.html$") | .string' - )" - echo "report-url=$REPORT_URL" - echo "report-url=$REPORT_URL" >> "$GITHUB_OUTPUT" - - SUMMARY_PATH="${test_dir}/working_dir/50_Outputs/$image_name/executive_summary/template.pdf" - if [ -f "$SUMMARY_PATH" ]; then - SUMMARY_URL="$( - echo "$REPORT_UPLOADED_FILES" | jq -rc '. [] | match(".*template.pdf$") | .string' - )" - echo "summary-url=$SUMMARY_URL" - echo "summary-url=$SUMMARY_URL" >> "$GITHUB_OUTPUT" - fi - - - name: Prepare comment artifact - id: prepare-artifact - env: - commit_time: ${{ github.event.pull_request.updated_at }} - full_image_name: ${{ needs.docker-build.outputs.full-image-name }} - git_sha: ${{ github.event.pull_request.head.sha }}, - image_name: ${{ inputs.image-name }} - language: ${{ matrix.language }} - pacta_data_quarter: ${{ needs.read-config.outputs.pacta-data-quarter }} - peer_group: ${{ matrix.peer_group }} - project_code: ${{ needs.read-config.outputs.project-code }} - registry: ${{ inputs.registry }} - report_url: ${{ steps.export-outputs.outputs.report-url }} - run_attempt: ${{ github.run_attempt }} - summary_url: ${{ steps.export-outputs.outputs.summary-url }} - run: | - mkdir -p /tmp/comment-json - unique_string= - json_filename=$( \ - echo "comment-json-merge-$image_name-$registry-$language-$peer_group.json" | \ - tr '/' '-' \ - ) - echo "json-filename=$json_filename" >> "$GITHUB_OUTPUT" - json_file="/tmp/comment-json/$json_filename" - jq \ - -n \ - --arg commit_time "$commit_time" \ - --arg git_sha "$git_sha" \ - --arg project_code "$project_code" \ - --arg pacta_data_quarter "$pacta_data_quarter" \ - --arg language "$language" \ - --arg peer_group "$peer_group" \ - --arg report_url "$report_url" \ - --arg summary_url "$summary_url" \ - --arg full_image_name "$full_image_name" \ - '{ - "commit_time": $commit_time, - "git_sha": $git_sha, - "project_code": $project_code, - "holdings_date": $pacta_data_quarter, - "language": $language, - "peer_group": $peer_group, - "report": (if (("" + $report_url) == "") then null else "[Report](" + $report_url + ")" end), - "summary": (if (("" + $summary_url) == "") then null else "[Summary](" + $summary_url + ")" end), - "image": ("`" + $full_image_name + "`") - }' \ - >> $json_file - cat "$json_file" - - - name: Upload comment JSON - uses: actions/upload-artifact@v4 - with: - name: ${{ steps.prepare-artifact.outputs.json-filename }} - path: /tmp/comment-json/* - if-no-files-found: error - retention-days: 1 + # # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#example-masking-a-generated-output-within-a-single-job + # - name: Get credentials for docker registry + # id: docker-credentials + # uses: azure/CLI@v2 + # env: + # registry: ${{ inputs.registry }} + # with: + # # azcliversion: 2.30.0 + # inlineScript: | + # # Username is not a placeholder. See https://stackoverflow.com/a/73250630 + # DOCKER_USER="00000000-0000-0000-0000-000000000000" >> "$GITHUB_OUTPUT" + # DOCKER_TOKEN="$(az acr login -n $registry --expose-token --query 'accessToken' --output tsv)" >> "$GITHUB_OUTPUT" + # echo "::add-mask::$DOCKER_TOKEN" + # echo "docker-user=$DOCKER_USER" >> "$GITHUB_OUTPUT" + # echo "docker-token=$DOCKER_TOKEN" >> "$GITHUB_OUTPUT" + + # - name: Login to Registry + # env: + # REGISTRY: ${{ inputs.registry }} + # DOCKER_TOKEN: ${{ steps.docker-credentials.outputs.docker-token }} + # DOCKER_USER: ${{ steps.docker-credentials.outputs.docker-user }} + # run: | + # echo "$DOCKER_TOKEN" | docker login $REGISTRY --username "$DOCKER_USER" --password-stdin + + # - name: Test + # env: + # full_image_name: ${{ needs.docker-build.outputs.full-image-name }} + # image_name: ${{ inputs.image-name }} + # test_dir: ${{ steps.prepare-results-dir.outputs.test-dir }} + # workspace: ${{ github.workspace }} + # run: | + # chmod -R 777 "$test_dir/working_dir" + # docker run \ + # --network none \ + # --user root \ + # --mount "type=bind,source=$workspace/$test_dir/working_dir,target=/bound/working_dir" \ + # --mount "type=bind,readonly,source=$workspace/$test_dir/user_dir,target=/user_results" \ + # $full_image_name \ + # /bound/bin/run-r-scripts $image_name + + # # https://github.com/Azure/login?tab=readme-ov-file#login-with-openid-connect-oidc-recommended + # - name: Azure Login + # uses: azure/login@v2 + # with: + # client-id: ${{ secrets.AZURE_CLIENT_ID }} + # tenant-id: ${{ secrets.AZURE_TENANT_ID }} + # subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + # - name: Upload results to Blob store + # id: upload-results + # uses: RMI-PACTA/actions/actions/azure/blob-copy@main + # with: + # source: ${{ steps.prepare-results-dir.outputs.test-dir-parent }} + # destination: https://pactadatadev.blob.core.windows.net/ghactions-workflow-transition-monitor-results-full + # overwrite: false + + # - name: Upload report to Blob store + # id: upload-report + # uses: RMI-PACTA/actions/actions/azure/blob-copy@main + # with: + # source: ${{ steps.prepare-results-dir.outputs.test-dir-parent }}/${{ matrix.language }}/${{ matrix.peer_group }}/${{ github.run_attempt }}/working_dir/50_Outputs + # destination: https://pactadatadev.blob.core.windows.net/ghactions-workflow-transition-monitor-results-reports + # overwrite: false + + # - name: Export Outputs + # id: export-outputs + # env: + # REPORT_UPLOADED_FILES: ${{ steps.upload-report.outputs.destination-files }} + # image_name: ${{ inputs.image-name }} + # test_dir: ${{ steps.prepare-results-dir.outputs.test-dir }} + # run: | + + # REPORT_URL="$( + # echo "$REPORT_UPLOADED_FILES" | jq -rc '. [] | match(".*index.html$") | .string' + # )" + # echo "report-url=$REPORT_URL" + # echo "report-url=$REPORT_URL" >> "$GITHUB_OUTPUT" + + # SUMMARY_PATH="${test_dir}/working_dir/50_Outputs/$image_name/executive_summary/template.pdf" + # if [ -f "$SUMMARY_PATH" ]; then + # SUMMARY_URL="$( + # echo "$REPORT_UPLOADED_FILES" | jq -rc '. [] | match(".*template.pdf$") | .string' + # )" + # echo "summary-url=$SUMMARY_URL" + # echo "summary-url=$SUMMARY_URL" >> "$GITHUB_OUTPUT" + # fi + + # - name: Prepare comment artifact + # id: prepare-artifact + # env: + # commit_time: ${{ github.event.pull_request.updated_at }} + # full_image_name: ${{ needs.docker-build.outputs.full-image-name }} + # git_sha: ${{ github.event.pull_request.head.sha }}, + # image_name: ${{ inputs.image-name }} + # language: ${{ matrix.language }} + # pacta_data_quarter: ${{ needs.read-config.outputs.pacta-data-quarter }} + # peer_group: ${{ matrix.peer_group }} + # project_code: ${{ needs.read-config.outputs.project-code }} + # registry: ${{ inputs.registry }} + # report_url: ${{ steps.export-outputs.outputs.report-url }} + # run_attempt: ${{ github.run_attempt }} + # summary_url: ${{ steps.export-outputs.outputs.summary-url }} + # run: | + # mkdir -p /tmp/comment-json + # unique_string= + # json_filename=$( \ + # echo "comment-json-merge-$image_name-$registry-$language-$peer_group.json" | \ + # tr '/' '-' \ + # ) + # echo "json-filename=$json_filename" >> "$GITHUB_OUTPUT" + # json_file="/tmp/comment-json/$json_filename" + # jq \ + # -n \ + # --arg commit_time "$commit_time" \ + # --arg git_sha "$git_sha" \ + # --arg project_code "$project_code" \ + # --arg pacta_data_quarter "$pacta_data_quarter" \ + # --arg language "$language" \ + # --arg peer_group "$peer_group" \ + # --arg report_url "$report_url" \ + # --arg summary_url "$summary_url" \ + # --arg full_image_name "$full_image_name" \ + # '{ + # "commit_time": $commit_time, + # "git_sha": $git_sha, + # "project_code": $project_code, + # "holdings_date": $pacta_data_quarter, + # "language": $language, + # "peer_group": $peer_group, + # "report": (if (("" + $report_url) == "") then null else "[Report](" + $report_url + ")" end), + # "summary": (if (("" + $summary_url) == "") then null else "[Summary](" + $summary_url + ")" end), + # "image": ("`" + $full_image_name + "`") + # }' \ + # >> $json_file + # cat "$json_file" + + # - name: Upload comment JSON + # uses: actions/upload-artifact@v4 + # with: + # name: ${{ steps.prepare-artifact.outputs.json-filename }} + # path: /tmp/comment-json/* + # if-no-files-found: error + # retention-days: 1 From a6de0107eaedff8c1aba1e81716fc64a2ab3d41b Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 30 Oct 2024 11:22:31 +0100 Subject: [PATCH 04/12] Exh Matrix --- .github/workflows/build-push-private.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/build-push-private.yml b/.github/workflows/build-push-private.yml index 6c7d0915..a3c6e00f 100644 --- a/.github/workflows/build-push-private.yml +++ b/.github/workflows/build-push-private.yml @@ -394,6 +394,12 @@ jobs: # mv working_dir/20_Raw_Inputs/1234.csv $portfolio_file # ls -lR $TEST_DIR + - name: Show Matrix + env: + MATRIX: ${{ matrix }} + run: | + echo "$MATRIX" | jq '.' + - name: Download User Results from Blob store id: download-user-results uses: RMI-PACTA/actions/actions/azure/blob-copy@main From bef23e7e6b14908b388179ebec4ef82a446b54cc Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 30 Oct 2024 11:29:14 +0100 Subject: [PATCH 05/12] Expanded test matrix --- build/config/rmi_pacta_2023q4_pa2024ch.json | 37 ++++++--------------- 1 file changed, 11 insertions(+), 26 deletions(-) diff --git a/build/config/rmi_pacta_2023q4_pa2024ch.json b/build/config/rmi_pacta_2023q4_pa2024ch.json index e446c52a..5c4d053e 100644 --- a/build/config/rmi_pacta_2023q4_pa2024ch.json +++ b/build/config/rmi_pacta_2023q4_pa2024ch.json @@ -6,52 +6,37 @@ "peer_results": "https://pactadatadev.blob.core.windows.net/project-files/pa2024ch/peer_files", "net_zero_targets": "https://pactadatadev.blob.core.windows.net/project-files/pa2024ch/fin_data_net_zero_targets.csv", "test_matrix": { + "peer_group": ["bank", "assetmanager", "insurance", "pensionfund", "other"], + "language": ["EN", "DE", "FR"], "user_results": ["https://pactadatadev.blob.core.windows.net/pa2024ch/outputs/user_data"], "include": [ { "user_id": 128, - "peer_group": "bank", - "language": "EN" - }, - { - "user_id": 128, - "peer_group": "bank", - "language": "DE" - }, - { - "user_id": 128, - "peer_group": "bank", - "language": "FR" + "peer_group": "bank" }, { "user_id": 130, - "peer_group": "assetmanager", - "language": "EN" + "peer_group": "assetmanager" }, { "user_id": 131, - "peer_group": "insurance", - "language": "EN" + "peer_group": "insurance" }, { "user_id": 134, - "peer_group": "pensionfund", - "language": "EN" + "peer_group": "pensionfund" }, { "user_id": 135, - "peer_group": "other", - "language": "EN" + "peer_group": "other" }, { - "user_id": 135, - "peer_group": "other", - "language": "DE" + "language": "FR", + "peer_group": "other" }, { - "user_id": 135, - "peer_group": "other", - "language": "FR" + "language": "DE", + "peer_group": "other" } ] }, From b17775d65a1cce5ade93ae587c95c38db4c691d6 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 30 Oct 2024 11:32:39 +0100 Subject: [PATCH 06/12] Further testing matrix configs --- .../workflows/build-Docker-image-triggers.yml | 36 +++++++++---------- .github/workflows/build-push-private.yml | 5 ++- build/config/rmi_pacta_2023q4_pa2024ch.json | 2 +- 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build-Docker-image-triggers.yml b/.github/workflows/build-Docker-image-triggers.yml index adeb17eb..1001fc2d 100644 --- a/.github/workflows/build-Docker-image-triggers.yml +++ b/.github/workflows/build-Docker-image-triggers.yml @@ -9,27 +9,27 @@ on: workflow_dispatch: jobs: - build_docker_image: - name: "Call build and push action" - uses: ./.github/workflows/build-and-push-Docker-image.yml - secrets: inherit - permissions: - packages: write - contents: read - with: - image-name: ${{ github.event.repository.name }} - image-tag: | - ${{ - github.event_name == 'pull_request' && format('{0}{1}','pr', github.event.pull_request.number) || - github.event_name == 'schedule' && 'nightly' || - github.event_name == 'push' && github.ref_name || - github.event_name == 'workflow_dispatch' && github.ref_name || - github.sha - }} + # build_docker_image: + # name: "Call build and push action" + # uses: ./.github/workflows/build-and-push-Docker-image.yml + # secrets: inherit + # permissions: + # packages: write + # contents: read + # with: + # image-name: ${{ github.event.repository.name }} + # image-tag: | + # ${{ + # github.event_name == 'pull_request' && format('{0}{1}','pr', github.event.pull_request.number) || + # github.event_name == 'schedule' && 'nightly' || + # github.event_name == 'push' && github.ref_name || + # github.event_name == 'workflow_dispatch' && github.ref_name || + # github.sha + # }} build_private_docker_image: name: "Build private docker image" - needs: build_docker_image + # needs: build_docker_image strategy: # if one build or test fails, permit the others to run fail-fast: false diff --git a/.github/workflows/build-push-private.yml b/.github/workflows/build-push-private.yml index a3c6e00f..53d00560 100644 --- a/.github/workflows/build-push-private.yml +++ b/.github/workflows/build-push-private.yml @@ -396,7 +396,10 @@ jobs: - name: Show Matrix env: - MATRIX: ${{ matrix }} + PEER_GROUP: ${{ matrix.peer_group }} + LANGUAGE: ${{ matrix.language}} + USER_RESULTS: ${{ matrix.user_results }} + USER_ID: ${{ matrix.user_id }} run: | echo "$MATRIX" | jq '.' diff --git a/build/config/rmi_pacta_2023q4_pa2024ch.json b/build/config/rmi_pacta_2023q4_pa2024ch.json index 5c4d053e..f836be4e 100644 --- a/build/config/rmi_pacta_2023q4_pa2024ch.json +++ b/build/config/rmi_pacta_2023q4_pa2024ch.json @@ -7,7 +7,7 @@ "net_zero_targets": "https://pactadatadev.blob.core.windows.net/project-files/pa2024ch/fin_data_net_zero_targets.csv", "test_matrix": { "peer_group": ["bank", "assetmanager", "insurance", "pensionfund", "other"], - "language": ["EN", "DE", "FR"], + "language": ["EN"], "user_results": ["https://pactadatadev.blob.core.windows.net/pa2024ch/outputs/user_data"], "include": [ { From c13c9d3e8ec4e9b55dc6daaf0aaddeb4449790f7 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 30 Oct 2024 11:35:34 +0100 Subject: [PATCH 07/12] disable comment step --- .github/workflows/build-Docker-image-triggers.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-Docker-image-triggers.yml b/.github/workflows/build-Docker-image-triggers.yml index 1001fc2d..1b9bd0c5 100644 --- a/.github/workflows/build-Docker-image-triggers.yml +++ b/.github/workflows/build-Docker-image-triggers.yml @@ -44,10 +44,10 @@ jobs: with: image-name: ${{ matrix.image-name }} - add-comment: - if: ${{ github.event_name == 'pull_request' }} - uses: RMI-PACTA/actions/.github/workflows/add-comment-table.yml@main - needs: [build_docker_image, build_private_docker_image] - with: - header: "Docker build status" + # add-comment: + # if: ${{ github.event_name == 'pull_request' }} + # uses: RMI-PACTA/actions/.github/workflows/add-comment-table.yml@main + # needs: [build_docker_image, build_private_docker_image] + # with: + # header: "Docker build status" From 084c24607bc8bdf9d249e3a03f14b9b2bc142f4f Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 30 Oct 2024 11:38:45 +0100 Subject: [PATCH 08/12] aactuall echo matrix values --- .github/workflows/build-push-private.yml | 39 +++++++++++++----------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build-push-private.yml b/.github/workflows/build-push-private.yml index 53d00560..ee5188c2 100644 --- a/.github/workflows/build-push-private.yml +++ b/.github/workflows/build-push-private.yml @@ -337,16 +337,16 @@ jobs: steps: - # https://github.com/Azure/login?tab=readme-ov-file#login-with-openid-connect-oidc-recommended - - name: Azure Login - uses: azure/login@v2 - with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + # # https://github.com/Azure/login?tab=readme-ov-file#login-with-openid-connect-oidc-recommended + # - name: Azure Login + # uses: azure/login@v2 + # with: + # client-id: ${{ secrets.AZURE_CLIENT_ID }} + # tenant-id: ${{ secrets.AZURE_TENANT_ID }} + # subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - name: Checkout workflow.transition.monitor - uses: actions/checkout@v4 + # - name: Checkout workflow.transition.monitor + # uses: actions/checkout@v4 # - name: Prepare User and results directories # id: prepare-results-dir @@ -401,16 +401,19 @@ jobs: USER_RESULTS: ${{ matrix.user_results }} USER_ID: ${{ matrix.user_id }} run: | - echo "$MATRIX" | jq '.' + echo "PEER_GROUP: $PEER_GROUP" + echo "LANGUAGE: $LANGUAGE" + echo "USER_RESULTS: $USER_RESULTS" + echo "USER_ID: $USER_ID" - - name: Download User Results from Blob store - id: download-user-results - uses: RMI-PACTA/actions/actions/azure/blob-copy@main - if: ${{ matrix.user_results }} - with: - source: ${{ matrix.user_results }}/${{ matrix.user_id }} - destination: ${{ steps.prepare-results-dir.outputs.test-dir }}/user_dir - overwrite: false + # - name: Download User Results from Blob store + # id: download-user-results + # uses: RMI-PACTA/actions/actions/azure/blob-copy@main + # if: ${{ matrix.user_results }} + # with: + # source: ${{ matrix.user_results }}/${{ matrix.user_id }} + # destination: ${{ steps.prepare-results-dir.outputs.test-dir }}/user_dir + # overwrite: false - name: Move user results to correct location shell: sh From 989340252b952972aad8745927ed6aaa34727995 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 30 Oct 2024 11:40:48 +0100 Subject: [PATCH 09/12] Use exclusions to reduce testing --- build/config/rmi_pacta_2023q4_pa2024ch.json | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/build/config/rmi_pacta_2023q4_pa2024ch.json b/build/config/rmi_pacta_2023q4_pa2024ch.json index f836be4e..f093f5ea 100644 --- a/build/config/rmi_pacta_2023q4_pa2024ch.json +++ b/build/config/rmi_pacta_2023q4_pa2024ch.json @@ -7,8 +7,18 @@ "net_zero_targets": "https://pactadatadev.blob.core.windows.net/project-files/pa2024ch/fin_data_net_zero_targets.csv", "test_matrix": { "peer_group": ["bank", "assetmanager", "insurance", "pensionfund", "other"], - "language": ["EN"], + "language": ["EN", "DE", "FR"], "user_results": ["https://pactadatadev.blob.core.windows.net/pa2024ch/outputs/user_data"], + "exclude": [ + { + "language": "DE", + "peer_group": "assetmanager" + }, + { + "language": "FR", + "peer_group": "pensionfund" + } + ] "include": [ { "user_id": 128, @@ -29,14 +39,6 @@ { "user_id": 135, "peer_group": "other" - }, - { - "language": "FR", - "peer_group": "other" - }, - { - "language": "DE", - "peer_group": "other" } ] }, From e3709bf55e189531fd19cbf4ef4a905be1778cbe Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 30 Oct 2024 11:51:15 +0100 Subject: [PATCH 10/12] Fix JSON Syntax --- build/config/rmi_pacta_2023q4_pa2024ch.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/config/rmi_pacta_2023q4_pa2024ch.json b/build/config/rmi_pacta_2023q4_pa2024ch.json index f093f5ea..e2819f8a 100644 --- a/build/config/rmi_pacta_2023q4_pa2024ch.json +++ b/build/config/rmi_pacta_2023q4_pa2024ch.json @@ -18,7 +18,7 @@ "language": "FR", "peer_group": "pensionfund" } - ] + ], "include": [ { "user_id": 128, From 5aaaae232855fd9bf14a28b9172da933b7a8a59d Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 30 Oct 2024 11:53:25 +0100 Subject: [PATCH 11/12] Finish test exclusion list --- build/config/rmi_pacta_2023q4_pa2024ch.json | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/build/config/rmi_pacta_2023q4_pa2024ch.json b/build/config/rmi_pacta_2023q4_pa2024ch.json index e2819f8a..7f898f5d 100644 --- a/build/config/rmi_pacta_2023q4_pa2024ch.json +++ b/build/config/rmi_pacta_2023q4_pa2024ch.json @@ -14,6 +14,22 @@ "language": "DE", "peer_group": "assetmanager" }, + { + "language": "FR", + "peer_group": "assetmanager" + }, + { + "language": "DE", + "peer_group": "insurance" + }, + { + "language": "FR", + "peer_group": "insurance" + }, + { + "language": "DE", + "peer_group": "pensionfund" + }, { "language": "FR", "peer_group": "pensionfund" From b4ae41d588becc321df407d2be0ae4ae38d51452 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 30 Oct 2024 11:54:48 +0100 Subject: [PATCH 12/12] Restore original test behavior --- .../workflows/build-Docker-image-triggers.yml | 48 +- .github/workflows/build-push-private.yml | 854 +++++++++--------- 2 files changed, 445 insertions(+), 457 deletions(-) diff --git a/.github/workflows/build-Docker-image-triggers.yml b/.github/workflows/build-Docker-image-triggers.yml index 1b9bd0c5..adeb17eb 100644 --- a/.github/workflows/build-Docker-image-triggers.yml +++ b/.github/workflows/build-Docker-image-triggers.yml @@ -9,27 +9,27 @@ on: workflow_dispatch: jobs: - # build_docker_image: - # name: "Call build and push action" - # uses: ./.github/workflows/build-and-push-Docker-image.yml - # secrets: inherit - # permissions: - # packages: write - # contents: read - # with: - # image-name: ${{ github.event.repository.name }} - # image-tag: | - # ${{ - # github.event_name == 'pull_request' && format('{0}{1}','pr', github.event.pull_request.number) || - # github.event_name == 'schedule' && 'nightly' || - # github.event_name == 'push' && github.ref_name || - # github.event_name == 'workflow_dispatch' && github.ref_name || - # github.sha - # }} + build_docker_image: + name: "Call build and push action" + uses: ./.github/workflows/build-and-push-Docker-image.yml + secrets: inherit + permissions: + packages: write + contents: read + with: + image-name: ${{ github.event.repository.name }} + image-tag: | + ${{ + github.event_name == 'pull_request' && format('{0}{1}','pr', github.event.pull_request.number) || + github.event_name == 'schedule' && 'nightly' || + github.event_name == 'push' && github.ref_name || + github.event_name == 'workflow_dispatch' && github.ref_name || + github.sha + }} build_private_docker_image: name: "Build private docker image" - # needs: build_docker_image + needs: build_docker_image strategy: # if one build or test fails, permit the others to run fail-fast: false @@ -44,10 +44,10 @@ jobs: with: image-name: ${{ matrix.image-name }} - # add-comment: - # if: ${{ github.event_name == 'pull_request' }} - # uses: RMI-PACTA/actions/.github/workflows/add-comment-table.yml@main - # needs: [build_docker_image, build_private_docker_image] - # with: - # header: "Docker build status" + add-comment: + if: ${{ github.event_name == 'pull_request' }} + uses: RMI-PACTA/actions/.github/workflows/add-comment-table.yml@main + needs: [build_docker_image, build_private_docker_image] + with: + header: "Docker build status" diff --git a/.github/workflows/build-push-private.yml b/.github/workflows/build-push-private.yml index ee5188c2..5c2beb24 100644 --- a/.github/workflows/build-push-private.yml +++ b/.github/workflows/build-push-private.yml @@ -103,220 +103,220 @@ jobs: echo "net-zero-targets=$NET_ZERO_TARGETS" echo "net-zero-targets=$NET_ZERO_TARGETS" >> "$GITHUB_OUTPUT" - # prepare-indices: - # name: Run Index Preparation - # needs: read-config - # secrets: inherit - # uses: RMI-PACTA/workflow.prepare.pacta.indices/.github/workflows/run-index-preparation.yml@main - # with: - # image-tag: | - # ${{ github.event_name == 'pull_request' && format('{0}{1}','pr', github.event.pull_request.number) || - # github.event_name == 'schedule' && 'main' || - # github.event_name == 'push' && github.ref_name || - # github.event_name == 'workflow_dispatch' && github.ref_name || - # github.sha }} - # data-share-path: ${{ needs.read-config.outputs.pacta-data-share-path }} - # config_active: ${{ needs.read-config.outputs.pacta-data-quarter }} - - # docker-build: - # runs-on: ubuntu-latest - # needs: [read-config, prepare-indices] - # permissions: - # contents: read - # id-token: write - # timeout-minutes: 45 - # outputs: - # full-image-name: ${{ steps.export-outputs.outputs.full-image-name }} - - # steps: - # - name: Checkout workflow.transition.monitor - # uses: actions/checkout@v4 - # with: - # path: workflow.transition.monitor - - # - name: Checkout templates.transition.monitor - # uses: actions/checkout@v4 - # with: - # repository: RMI-PACTA/templates.transition.monitor - # path: templates.transition.monitor - # token: ${{ secrets.REPO_PAT }} - # ref: ${{ needs.read-config.outputs.templates-ref }} - - # # https://github.com/Azure/login?tab=readme-ov-file#login-with-openid-connect-oidc-recommended - # - name: Azure Login - # uses: azure/login@v2 - # with: - # client-id: ${{ secrets.AZURE_CLIENT_ID }} - # tenant-id: ${{ secrets.AZURE_TENANT_ID }} - # subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - # # https://github.com/marketplace/actions/azure-cli-action#workflow-to-execute-an-azure-cli-script-of-a-specific-cli-version - # - name: Download pacta-data - # uses: azure/CLI@v2 - # env: - # index_share_path: ${{ needs.prepare-indices.outputs.results-dir }} - # pacta_data_quarter: ${{ needs.read-config.outputs.pacta-data-quarter }} - # pacta_data_share_path: ${{ needs.read-config.outputs.pacta-data-share-path }} - # peer_results: ${{ needs.read-config.outputs.peer-results }} - # net_zero_targets: ${{ needs.read-config.outputs.net-zero-targets }} - # with: - # # azcliversion: 2.30.0 - # inlineScript: | - # pacta_data_share_url="https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs" - # az storage copy \ - # --source "$pacta_data_share_url/$pacta_data_share_path" \ - # --destination "pacta-data" \ - # --recursive \ - # --exclude-pattern "*.sqlite" - # # download indices - # index_share_url="https://pactadatadev.file.core.windows.net/workflow-prepare-pacta-indices-outputs" - # az storage copy \ - # --source "$index_share_url/$index_share_path" \ - # --destination "index-data" \ - # --recursive \ - # --exclude-pattern "*.sqlite" - - # # Download peer results if necessary - # if [ -n "$peer_results" ]; then - # az storage copy \ - # --source "$peer_results/*" \ - # --destination "peer_results" \ - # --recursive - # else - # echo "No Peer Results defined in config" - # fi - - # # Download net zero targets if necessary - # if [ -n "$net_zero_targets" ]; then - # az storage copy \ - # --source "$net_zero_targets" \ - # --destination "peer_results" \ - # --recursive - # else - # echo "No Peer Results defined in config" - # fi - - # # move data - # mv "pacta-data/$pacta_data_share_path" "pacta-data/$pacta_data_quarter" - # mv "index-data/$index_share_path"/* "pacta-data/$pacta_data_quarter" - # if [ -n "$peer_results" ]; then - # mv peer_results/* "pacta-data/$pacta_data_quarter" - # fi - # ls pacta-data/$pacta_data_quarter - - # - name: Identify LABELs in dockerfile - # id: custom-labels - # run: | - # DOCKERFILE_LABELS="$(grep "^LABEL" workflow.transition.monitor/Dockerfile | sed 's/^LABEL[[:space:]]*//')" - # echo "$DOCKERFILE_LABELS" - # echo 'dockerfile-labels<> $GITHUB_OUTPUT - # echo "$DOCKERFILE_LABELS" >> $GITHUB_OUTPUT - # echo 'EOF' >> $GITHUB_OUTPUT - - # # Setup docker metadata, including tags and labels (and annotations) - # - name: Docker meta - # id: meta - # uses: docker/metadata-action@v5 - # env: - # additional_image_tags: ${{ inputs.additional-image-tags }} - # now: ${{ needs.read-config.outputs.now }} - # with: - # images: ${{ needs.read-config.outputs.registry-image }} - # annotations: - # ${{ steps.custom-labels.outputs.DOCKERFILE_LABELS }} - # labels: - # ${{ steps.custom-labels.outputs.DOCKERFILE_LABELS }} - # tags: | - # type=schedule,enable=true,pattern={{date 'YYYYMMDD[T]HHmmss[Z]' tz='UTC'}} - # type=schedule,enable=true,pattern=nightly,priority=950 - # type=raw,value=latest,enable={{is_default_branch}} - # type=ref,event=branch - # type=ref,event=tag - # type=ref,event=pr - # ${{ inputs.additional-image-tags }} - # ${{ needs.read-config.outputs.now }},priority=1100 - - # # Set up Dockerignore file - # - name: Copy dockerignore - # run: | - # cp ./workflow.transition.monitor/.dockerignore.CIbuild .dockerignore - - # # set up our build environment - # - name: Set up QEMU - # uses: docker/setup-qemu-action@v3 - - # - name: Set up Docker Buildx - # uses: docker/setup-buildx-action@v3 - - # # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#example-masking-a-generated-output-within-a-single-job - # - name: Get credentials for docker registry - # id: docker-credentials - # uses: azure/CLI@v2 - # env: - # registry: ${{ inputs.registry }} - # with: - # # azcliversion: 2.30.0 - # inlineScript: | - # # Username is not a placeholder. See https://stackoverflow.com/a/73250630 - # DOCKER_USER="00000000-0000-0000-0000-000000000000" >> "$GITHUB_OUTPUT" - # DOCKER_TOKEN="$(az acr login -n $registry --expose-token --query 'accessToken' --output tsv)" >> "$GITHUB_OUTPUT" - # echo "::add-mask::$DOCKER_TOKEN" - # echo "docker-user=$DOCKER_USER" >> "$GITHUB_OUTPUT" - # echo "docker-token=$DOCKER_TOKEN" >> "$GITHUB_OUTPUT" - - # - name: Login to Registry - # env: - # REGISTRY: ${{ inputs.registry }} - # DOCKER_TOKEN: ${{ steps.docker-credentials.outputs.docker-token }} - # DOCKER_USER: ${{ steps.docker-credentials.outputs.docker-user }} - # run: | - # echo "$DOCKER_TOKEN" | docker login $REGISTRY --username "$DOCKER_USER" --password-stdin - - # # Actually build the image (for a single architecture)! - # - name: Build - # id: build - # uses: docker/build-push-action@v6 - # with: - # context: . - # file: ./workflow.transition.monitor/Dockerfile - # # include the labels from the meta step - # labels: ${{ steps.meta.outputs.labels }} - # # the same for annotations - # annotations: ${{ steps.meta.outputs.annotations }} - # push: true - # tags: ${{ steps.meta.outputs.tags }} - # # Use the GitHub actions cache to speed up repeated builds - # cache-from: type=gha - # cache-to: type=gha,mode=min - # # but don't cache the install pacta step. - # no-cache-filters: install-pacta - - # - name: Export Outputs - # id: export-outputs - # run: | - # TAGGED_IMAGE="$(jq -rc '.tags[0]' <<< "$DOCKER_METADATA_OUTPUT_JSON")" - # echo "full-image-name=$TAGGED_IMAGE" - # echo "full-image-name=$TAGGED_IMAGE" >> "$GITHUB_OUTPUT" - - # - name: Prepare inputs archive - # id: prepare-inputs-archive - # env: - # full_image_name: ${{ steps.export-outputs.outputs.full-image-name }} - # run: | - # tarfile="$(echo $full_image_name | sed 's:.*/::' | tr ":" "-").tar.gz" - # tar -cvz -f "$tarfile" pacta-data templates.transition.monitor - # echo "$tarfile" - # echo "input-tar=$tarfile" - # echo "input-tar=$tarfile" >> "$GITHUB_OUTPUT" - - - # - name: Export inputs archive - # id: upload-report - # uses: RMI-PACTA/actions/actions/azure/blob-copy@main - # with: - # source: ${{ steps.prepare-inputs-archive.outputs.input-tar }} - # destination: https://pactadatadev.blob.core.windows.net/ghactions-workflow-transition-monitor-input-pacta-data - # overwrite: false + prepare-indices: + name: Run Index Preparation + needs: read-config + secrets: inherit + uses: RMI-PACTA/workflow.prepare.pacta.indices/.github/workflows/run-index-preparation.yml@main + with: + image-tag: | + ${{ github.event_name == 'pull_request' && format('{0}{1}','pr', github.event.pull_request.number) || + github.event_name == 'schedule' && 'main' || + github.event_name == 'push' && github.ref_name || + github.event_name == 'workflow_dispatch' && github.ref_name || + github.sha }} + data-share-path: ${{ needs.read-config.outputs.pacta-data-share-path }} + config_active: ${{ needs.read-config.outputs.pacta-data-quarter }} + + docker-build: + runs-on: ubuntu-latest + needs: [read-config, prepare-indices] + permissions: + contents: read + id-token: write + timeout-minutes: 45 + outputs: + full-image-name: ${{ steps.export-outputs.outputs.full-image-name }} + + steps: + - name: Checkout workflow.transition.monitor + uses: actions/checkout@v4 + with: + path: workflow.transition.monitor + + - name: Checkout templates.transition.monitor + uses: actions/checkout@v4 + with: + repository: RMI-PACTA/templates.transition.monitor + path: templates.transition.monitor + token: ${{ secrets.REPO_PAT }} + ref: ${{ needs.read-config.outputs.templates-ref }} + + # https://github.com/Azure/login?tab=readme-ov-file#login-with-openid-connect-oidc-recommended + - name: Azure Login + uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + # https://github.com/marketplace/actions/azure-cli-action#workflow-to-execute-an-azure-cli-script-of-a-specific-cli-version + - name: Download pacta-data + uses: azure/CLI@v2 + env: + index_share_path: ${{ needs.prepare-indices.outputs.results-dir }} + pacta_data_quarter: ${{ needs.read-config.outputs.pacta-data-quarter }} + pacta_data_share_path: ${{ needs.read-config.outputs.pacta-data-share-path }} + peer_results: ${{ needs.read-config.outputs.peer-results }} + net_zero_targets: ${{ needs.read-config.outputs.net-zero-targets }} + with: + # azcliversion: 2.30.0 + inlineScript: | + pacta_data_share_url="https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs" + az storage copy \ + --source "$pacta_data_share_url/$pacta_data_share_path" \ + --destination "pacta-data" \ + --recursive \ + --exclude-pattern "*.sqlite" + # download indices + index_share_url="https://pactadatadev.file.core.windows.net/workflow-prepare-pacta-indices-outputs" + az storage copy \ + --source "$index_share_url/$index_share_path" \ + --destination "index-data" \ + --recursive \ + --exclude-pattern "*.sqlite" + + # Download peer results if necessary + if [ -n "$peer_results" ]; then + az storage copy \ + --source "$peer_results/*" \ + --destination "peer_results" \ + --recursive + else + echo "No Peer Results defined in config" + fi + + # Download net zero targets if necessary + if [ -n "$net_zero_targets" ]; then + az storage copy \ + --source "$net_zero_targets" \ + --destination "peer_results" \ + --recursive + else + echo "No Peer Results defined in config" + fi + + # move data + mv "pacta-data/$pacta_data_share_path" "pacta-data/$pacta_data_quarter" + mv "index-data/$index_share_path"/* "pacta-data/$pacta_data_quarter" + if [ -n "$peer_results" ]; then + mv peer_results/* "pacta-data/$pacta_data_quarter" + fi + ls pacta-data/$pacta_data_quarter + + - name: Identify LABELs in dockerfile + id: custom-labels + run: | + DOCKERFILE_LABELS="$(grep "^LABEL" workflow.transition.monitor/Dockerfile | sed 's/^LABEL[[:space:]]*//')" + echo "$DOCKERFILE_LABELS" + echo 'dockerfile-labels<> $GITHUB_OUTPUT + echo "$DOCKERFILE_LABELS" >> $GITHUB_OUTPUT + echo 'EOF' >> $GITHUB_OUTPUT + + # Setup docker metadata, including tags and labels (and annotations) + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + env: + additional_image_tags: ${{ inputs.additional-image-tags }} + now: ${{ needs.read-config.outputs.now }} + with: + images: ${{ needs.read-config.outputs.registry-image }} + annotations: + ${{ steps.custom-labels.outputs.DOCKERFILE_LABELS }} + labels: + ${{ steps.custom-labels.outputs.DOCKERFILE_LABELS }} + tags: | + type=schedule,enable=true,pattern={{date 'YYYYMMDD[T]HHmmss[Z]' tz='UTC'}} + type=schedule,enable=true,pattern=nightly,priority=950 + type=raw,value=latest,enable={{is_default_branch}} + type=ref,event=branch + type=ref,event=tag + type=ref,event=pr + ${{ inputs.additional-image-tags }} + ${{ needs.read-config.outputs.now }},priority=1100 + + # Set up Dockerignore file + - name: Copy dockerignore + run: | + cp ./workflow.transition.monitor/.dockerignore.CIbuild .dockerignore + + # set up our build environment + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#example-masking-a-generated-output-within-a-single-job + - name: Get credentials for docker registry + id: docker-credentials + uses: azure/CLI@v2 + env: + registry: ${{ inputs.registry }} + with: + # azcliversion: 2.30.0 + inlineScript: | + # Username is not a placeholder. See https://stackoverflow.com/a/73250630 + DOCKER_USER="00000000-0000-0000-0000-000000000000" >> "$GITHUB_OUTPUT" + DOCKER_TOKEN="$(az acr login -n $registry --expose-token --query 'accessToken' --output tsv)" >> "$GITHUB_OUTPUT" + echo "::add-mask::$DOCKER_TOKEN" + echo "docker-user=$DOCKER_USER" >> "$GITHUB_OUTPUT" + echo "docker-token=$DOCKER_TOKEN" >> "$GITHUB_OUTPUT" + + - name: Login to Registry + env: + REGISTRY: ${{ inputs.registry }} + DOCKER_TOKEN: ${{ steps.docker-credentials.outputs.docker-token }} + DOCKER_USER: ${{ steps.docker-credentials.outputs.docker-user }} + run: | + echo "$DOCKER_TOKEN" | docker login $REGISTRY --username "$DOCKER_USER" --password-stdin + + # Actually build the image (for a single architecture)! + - name: Build + id: build + uses: docker/build-push-action@v6 + with: + context: . + file: ./workflow.transition.monitor/Dockerfile + # include the labels from the meta step + labels: ${{ steps.meta.outputs.labels }} + # the same for annotations + annotations: ${{ steps.meta.outputs.annotations }} + push: true + tags: ${{ steps.meta.outputs.tags }} + # Use the GitHub actions cache to speed up repeated builds + cache-from: type=gha + cache-to: type=gha,mode=min + # but don't cache the install pacta step. + no-cache-filters: install-pacta + + - name: Export Outputs + id: export-outputs + run: | + TAGGED_IMAGE="$(jq -rc '.tags[0]' <<< "$DOCKER_METADATA_OUTPUT_JSON")" + echo "full-image-name=$TAGGED_IMAGE" + echo "full-image-name=$TAGGED_IMAGE" >> "$GITHUB_OUTPUT" + + - name: Prepare inputs archive + id: prepare-inputs-archive + env: + full_image_name: ${{ steps.export-outputs.outputs.full-image-name }} + run: | + tarfile="$(echo $full_image_name | sed 's:.*/::' | tr ":" "-").tar.gz" + tar -cvz -f "$tarfile" pacta-data templates.transition.monitor + echo "$tarfile" + echo "input-tar=$tarfile" + echo "input-tar=$tarfile" >> "$GITHUB_OUTPUT" + + + - name: Export inputs archive + id: upload-report + uses: RMI-PACTA/actions/actions/azure/blob-copy@main + with: + source: ${{ steps.prepare-inputs-archive.outputs.input-tar }} + destination: https://pactadatadev.blob.core.windows.net/ghactions-workflow-transition-monitor-input-pacta-data + overwrite: false test: runs-on: ubuntu-latest @@ -327,7 +327,7 @@ jobs: matrix: ${{ fromJson(needs.read-config.outputs.test-matrix) }} needs: - read-config - # - docker-build + - docker-build permissions: contents: read id-token: write @@ -337,83 +337,71 @@ jobs: steps: - # # https://github.com/Azure/login?tab=readme-ov-file#login-with-openid-connect-oidc-recommended - # - name: Azure Login - # uses: azure/login@v2 - # with: - # client-id: ${{ secrets.AZURE_CLIENT_ID }} - # tenant-id: ${{ secrets.AZURE_TENANT_ID }} - # subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - # - name: Checkout workflow.transition.monitor - # uses: actions/checkout@v4 - - # - name: Prepare User and results directories - # id: prepare-results-dir - # env: - # full_image_name: ${{ needs.docker-build.outputs.full-image-name }} - # image_name: ${{ inputs.image-name }} - # language: ${{ matrix.language }} - # pacta_data_quarter: ${{ needs.read-config.outputs.pacta-data-quarter }} - # peer_group: ${{ matrix.peer_group }} - # project_code: ${{ needs.read-config.outputs.project-code }} - # run_attempt: ${{ github.run_attempt }} - # user_id: ${{ matrix.user_id }} - # run: | - # test_dir_parent_name="$(echo $full_image_name | sed 's:.*/::' | tr ":" "-")" - # TEST_DIR_PARENT="$test_dir_parent_name" - # echo "TEST_DIR_PARENT=$TEST_DIR_PARENT" - # echo "test-dir-parent=$TEST_DIR_PARENT" >> "$GITHUB_OUTPUT" - - # TEST_DIR="$TEST_DIR_PARENT/$language/$peer_group/$run_attempt" - # mkdir -p "$TEST_DIR" - # chmod -R 777 "$TEST_DIR" - # echo "TEST_DIR=$TEST_DIR" - # echo "test-dir=$TEST_DIR" >> "$GITHUB_OUTPUT" - - # mkdir -p $TEST_DIR/user_dir - # cp -R working_dir $TEST_DIR - - # params_file="$TEST_DIR/working_dir/10_Parameter_File/${image_name}_PortfolioParameters.yml" - # echo $params_file - # mkdir -p "$(dirname $params_file)" - # sed \ - # -e "s/{{portfolio_name}}/$image_name-PORTFOLIO/" \ - # -e "s/{{investor_name}}/$image_name-$peer_group-INVESTOR/" \ - # -e "s/{{peer_group}}/$peer_group/" \ - # -e "s/{{language}}/$language/" \ - # -e "s/{{project_code}}/$project_code/" \ - # -e "s/{{holdings_date}}/$pacta_data_quarter/" \ - # -e "s/{{user_id}}/$user_id/" \ - # working_dir/10_Parameter_File/test_PortfolioParameters.yml \ - # >> $params_file - # cat "$params_file" - - # portfolio_file="$TEST_DIR/working_dir/20_Raw_Inputs/${image_name}.csv" - # mkdir -p "$(dirname $portfolio_file)" - # mv working_dir/20_Raw_Inputs/1234.csv $portfolio_file - # ls -lR $TEST_DIR - - - name: Show Matrix + # https://github.com/Azure/login?tab=readme-ov-file#login-with-openid-connect-oidc-recommended + - name: Azure Login + uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: Checkout workflow.transition.monitor + uses: actions/checkout@v4 + + - name: Prepare User and results directories + id: prepare-results-dir env: - PEER_GROUP: ${{ matrix.peer_group }} - LANGUAGE: ${{ matrix.language}} - USER_RESULTS: ${{ matrix.user_results }} - USER_ID: ${{ matrix.user_id }} + full_image_name: ${{ needs.docker-build.outputs.full-image-name }} + image_name: ${{ inputs.image-name }} + language: ${{ matrix.language }} + pacta_data_quarter: ${{ needs.read-config.outputs.pacta-data-quarter }} + peer_group: ${{ matrix.peer_group }} + project_code: ${{ needs.read-config.outputs.project-code }} + run_attempt: ${{ github.run_attempt }} + user_id: ${{ matrix.user_id }} run: | - echo "PEER_GROUP: $PEER_GROUP" - echo "LANGUAGE: $LANGUAGE" - echo "USER_RESULTS: $USER_RESULTS" - echo "USER_ID: $USER_ID" - - # - name: Download User Results from Blob store - # id: download-user-results - # uses: RMI-PACTA/actions/actions/azure/blob-copy@main - # if: ${{ matrix.user_results }} - # with: - # source: ${{ matrix.user_results }}/${{ matrix.user_id }} - # destination: ${{ steps.prepare-results-dir.outputs.test-dir }}/user_dir - # overwrite: false + test_dir_parent_name="$(echo $full_image_name | sed 's:.*/::' | tr ":" "-")" + TEST_DIR_PARENT="$test_dir_parent_name" + echo "TEST_DIR_PARENT=$TEST_DIR_PARENT" + echo "test-dir-parent=$TEST_DIR_PARENT" >> "$GITHUB_OUTPUT" + + TEST_DIR="$TEST_DIR_PARENT/$language/$peer_group/$run_attempt" + mkdir -p "$TEST_DIR" + chmod -R 777 "$TEST_DIR" + echo "TEST_DIR=$TEST_DIR" + echo "test-dir=$TEST_DIR" >> "$GITHUB_OUTPUT" + + mkdir -p $TEST_DIR/user_dir + cp -R working_dir $TEST_DIR + + params_file="$TEST_DIR/working_dir/10_Parameter_File/${image_name}_PortfolioParameters.yml" + echo $params_file + mkdir -p "$(dirname $params_file)" + sed \ + -e "s/{{portfolio_name}}/$image_name-PORTFOLIO/" \ + -e "s/{{investor_name}}/$image_name-$peer_group-INVESTOR/" \ + -e "s/{{peer_group}}/$peer_group/" \ + -e "s/{{language}}/$language/" \ + -e "s/{{project_code}}/$project_code/" \ + -e "s/{{holdings_date}}/$pacta_data_quarter/" \ + -e "s/{{user_id}}/$user_id/" \ + working_dir/10_Parameter_File/test_PortfolioParameters.yml \ + >> $params_file + cat "$params_file" + + portfolio_file="$TEST_DIR/working_dir/20_Raw_Inputs/${image_name}.csv" + mkdir -p "$(dirname $portfolio_file)" + mv working_dir/20_Raw_Inputs/1234.csv $portfolio_file + ls -lR $TEST_DIR + + - name: Download User Results from Blob store + id: download-user-results + uses: RMI-PACTA/actions/actions/azure/blob-copy@main + if: ${{ matrix.user_results }} + with: + source: ${{ matrix.user_results }}/${{ matrix.user_id }} + destination: ${{ steps.prepare-results-dir.outputs.test-dir }}/user_dir + overwrite: false - name: Move user results to correct location shell: sh @@ -430,146 +418,146 @@ jobs: $USER_RESULTS_DIR fi - # # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#example-masking-a-generated-output-within-a-single-job - # - name: Get credentials for docker registry - # id: docker-credentials - # uses: azure/CLI@v2 - # env: - # registry: ${{ inputs.registry }} - # with: - # # azcliversion: 2.30.0 - # inlineScript: | - # # Username is not a placeholder. See https://stackoverflow.com/a/73250630 - # DOCKER_USER="00000000-0000-0000-0000-000000000000" >> "$GITHUB_OUTPUT" - # DOCKER_TOKEN="$(az acr login -n $registry --expose-token --query 'accessToken' --output tsv)" >> "$GITHUB_OUTPUT" - # echo "::add-mask::$DOCKER_TOKEN" - # echo "docker-user=$DOCKER_USER" >> "$GITHUB_OUTPUT" - # echo "docker-token=$DOCKER_TOKEN" >> "$GITHUB_OUTPUT" - - # - name: Login to Registry - # env: - # REGISTRY: ${{ inputs.registry }} - # DOCKER_TOKEN: ${{ steps.docker-credentials.outputs.docker-token }} - # DOCKER_USER: ${{ steps.docker-credentials.outputs.docker-user }} - # run: | - # echo "$DOCKER_TOKEN" | docker login $REGISTRY --username "$DOCKER_USER" --password-stdin - - # - name: Test - # env: - # full_image_name: ${{ needs.docker-build.outputs.full-image-name }} - # image_name: ${{ inputs.image-name }} - # test_dir: ${{ steps.prepare-results-dir.outputs.test-dir }} - # workspace: ${{ github.workspace }} - # run: | - # chmod -R 777 "$test_dir/working_dir" - # docker run \ - # --network none \ - # --user root \ - # --mount "type=bind,source=$workspace/$test_dir/working_dir,target=/bound/working_dir" \ - # --mount "type=bind,readonly,source=$workspace/$test_dir/user_dir,target=/user_results" \ - # $full_image_name \ - # /bound/bin/run-r-scripts $image_name - - # # https://github.com/Azure/login?tab=readme-ov-file#login-with-openid-connect-oidc-recommended - # - name: Azure Login - # uses: azure/login@v2 - # with: - # client-id: ${{ secrets.AZURE_CLIENT_ID }} - # tenant-id: ${{ secrets.AZURE_TENANT_ID }} - # subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - # - name: Upload results to Blob store - # id: upload-results - # uses: RMI-PACTA/actions/actions/azure/blob-copy@main - # with: - # source: ${{ steps.prepare-results-dir.outputs.test-dir-parent }} - # destination: https://pactadatadev.blob.core.windows.net/ghactions-workflow-transition-monitor-results-full - # overwrite: false - - # - name: Upload report to Blob store - # id: upload-report - # uses: RMI-PACTA/actions/actions/azure/blob-copy@main - # with: - # source: ${{ steps.prepare-results-dir.outputs.test-dir-parent }}/${{ matrix.language }}/${{ matrix.peer_group }}/${{ github.run_attempt }}/working_dir/50_Outputs - # destination: https://pactadatadev.blob.core.windows.net/ghactions-workflow-transition-monitor-results-reports - # overwrite: false - - # - name: Export Outputs - # id: export-outputs - # env: - # REPORT_UPLOADED_FILES: ${{ steps.upload-report.outputs.destination-files }} - # image_name: ${{ inputs.image-name }} - # test_dir: ${{ steps.prepare-results-dir.outputs.test-dir }} - # run: | - - # REPORT_URL="$( - # echo "$REPORT_UPLOADED_FILES" | jq -rc '. [] | match(".*index.html$") | .string' - # )" - # echo "report-url=$REPORT_URL" - # echo "report-url=$REPORT_URL" >> "$GITHUB_OUTPUT" - - # SUMMARY_PATH="${test_dir}/working_dir/50_Outputs/$image_name/executive_summary/template.pdf" - # if [ -f "$SUMMARY_PATH" ]; then - # SUMMARY_URL="$( - # echo "$REPORT_UPLOADED_FILES" | jq -rc '. [] | match(".*template.pdf$") | .string' - # )" - # echo "summary-url=$SUMMARY_URL" - # echo "summary-url=$SUMMARY_URL" >> "$GITHUB_OUTPUT" - # fi - - # - name: Prepare comment artifact - # id: prepare-artifact - # env: - # commit_time: ${{ github.event.pull_request.updated_at }} - # full_image_name: ${{ needs.docker-build.outputs.full-image-name }} - # git_sha: ${{ github.event.pull_request.head.sha }}, - # image_name: ${{ inputs.image-name }} - # language: ${{ matrix.language }} - # pacta_data_quarter: ${{ needs.read-config.outputs.pacta-data-quarter }} - # peer_group: ${{ matrix.peer_group }} - # project_code: ${{ needs.read-config.outputs.project-code }} - # registry: ${{ inputs.registry }} - # report_url: ${{ steps.export-outputs.outputs.report-url }} - # run_attempt: ${{ github.run_attempt }} - # summary_url: ${{ steps.export-outputs.outputs.summary-url }} - # run: | - # mkdir -p /tmp/comment-json - # unique_string= - # json_filename=$( \ - # echo "comment-json-merge-$image_name-$registry-$language-$peer_group.json" | \ - # tr '/' '-' \ - # ) - # echo "json-filename=$json_filename" >> "$GITHUB_OUTPUT" - # json_file="/tmp/comment-json/$json_filename" - # jq \ - # -n \ - # --arg commit_time "$commit_time" \ - # --arg git_sha "$git_sha" \ - # --arg project_code "$project_code" \ - # --arg pacta_data_quarter "$pacta_data_quarter" \ - # --arg language "$language" \ - # --arg peer_group "$peer_group" \ - # --arg report_url "$report_url" \ - # --arg summary_url "$summary_url" \ - # --arg full_image_name "$full_image_name" \ - # '{ - # "commit_time": $commit_time, - # "git_sha": $git_sha, - # "project_code": $project_code, - # "holdings_date": $pacta_data_quarter, - # "language": $language, - # "peer_group": $peer_group, - # "report": (if (("" + $report_url) == "") then null else "[Report](" + $report_url + ")" end), - # "summary": (if (("" + $summary_url) == "") then null else "[Summary](" + $summary_url + ")" end), - # "image": ("`" + $full_image_name + "`") - # }' \ - # >> $json_file - # cat "$json_file" - - # - name: Upload comment JSON - # uses: actions/upload-artifact@v4 - # with: - # name: ${{ steps.prepare-artifact.outputs.json-filename }} - # path: /tmp/comment-json/* - # if-no-files-found: error - # retention-days: 1 + # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#example-masking-a-generated-output-within-a-single-job + - name: Get credentials for docker registry + id: docker-credentials + uses: azure/CLI@v2 + env: + registry: ${{ inputs.registry }} + with: + # azcliversion: 2.30.0 + inlineScript: | + # Username is not a placeholder. See https://stackoverflow.com/a/73250630 + DOCKER_USER="00000000-0000-0000-0000-000000000000" >> "$GITHUB_OUTPUT" + DOCKER_TOKEN="$(az acr login -n $registry --expose-token --query 'accessToken' --output tsv)" >> "$GITHUB_OUTPUT" + echo "::add-mask::$DOCKER_TOKEN" + echo "docker-user=$DOCKER_USER" >> "$GITHUB_OUTPUT" + echo "docker-token=$DOCKER_TOKEN" >> "$GITHUB_OUTPUT" + + - name: Login to Registry + env: + REGISTRY: ${{ inputs.registry }} + DOCKER_TOKEN: ${{ steps.docker-credentials.outputs.docker-token }} + DOCKER_USER: ${{ steps.docker-credentials.outputs.docker-user }} + run: | + echo "$DOCKER_TOKEN" | docker login $REGISTRY --username "$DOCKER_USER" --password-stdin + + - name: Test + env: + full_image_name: ${{ needs.docker-build.outputs.full-image-name }} + image_name: ${{ inputs.image-name }} + test_dir: ${{ steps.prepare-results-dir.outputs.test-dir }} + workspace: ${{ github.workspace }} + run: | + chmod -R 777 "$test_dir/working_dir" + docker run \ + --network none \ + --user root \ + --mount "type=bind,source=$workspace/$test_dir/working_dir,target=/bound/working_dir" \ + --mount "type=bind,readonly,source=$workspace/$test_dir/user_dir,target=/user_results" \ + $full_image_name \ + /bound/bin/run-r-scripts $image_name + + # https://github.com/Azure/login?tab=readme-ov-file#login-with-openid-connect-oidc-recommended + - name: Azure Login + uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: Upload results to Blob store + id: upload-results + uses: RMI-PACTA/actions/actions/azure/blob-copy@main + with: + source: ${{ steps.prepare-results-dir.outputs.test-dir-parent }} + destination: https://pactadatadev.blob.core.windows.net/ghactions-workflow-transition-monitor-results-full + overwrite: false + + - name: Upload report to Blob store + id: upload-report + uses: RMI-PACTA/actions/actions/azure/blob-copy@main + with: + source: ${{ steps.prepare-results-dir.outputs.test-dir-parent }}/${{ matrix.language }}/${{ matrix.peer_group }}/${{ github.run_attempt }}/working_dir/50_Outputs + destination: https://pactadatadev.blob.core.windows.net/ghactions-workflow-transition-monitor-results-reports + overwrite: false + + - name: Export Outputs + id: export-outputs + env: + REPORT_UPLOADED_FILES: ${{ steps.upload-report.outputs.destination-files }} + image_name: ${{ inputs.image-name }} + test_dir: ${{ steps.prepare-results-dir.outputs.test-dir }} + run: | + + REPORT_URL="$( + echo "$REPORT_UPLOADED_FILES" | jq -rc '. [] | match(".*index.html$") | .string' + )" + echo "report-url=$REPORT_URL" + echo "report-url=$REPORT_URL" >> "$GITHUB_OUTPUT" + + SUMMARY_PATH="${test_dir}/working_dir/50_Outputs/$image_name/executive_summary/template.pdf" + if [ -f "$SUMMARY_PATH" ]; then + SUMMARY_URL="$( + echo "$REPORT_UPLOADED_FILES" | jq -rc '. [] | match(".*template.pdf$") | .string' + )" + echo "summary-url=$SUMMARY_URL" + echo "summary-url=$SUMMARY_URL" >> "$GITHUB_OUTPUT" + fi + + - name: Prepare comment artifact + id: prepare-artifact + env: + commit_time: ${{ github.event.pull_request.updated_at }} + full_image_name: ${{ needs.docker-build.outputs.full-image-name }} + git_sha: ${{ github.event.pull_request.head.sha }}, + image_name: ${{ inputs.image-name }} + language: ${{ matrix.language }} + pacta_data_quarter: ${{ needs.read-config.outputs.pacta-data-quarter }} + peer_group: ${{ matrix.peer_group }} + project_code: ${{ needs.read-config.outputs.project-code }} + registry: ${{ inputs.registry }} + report_url: ${{ steps.export-outputs.outputs.report-url }} + run_attempt: ${{ github.run_attempt }} + summary_url: ${{ steps.export-outputs.outputs.summary-url }} + run: | + mkdir -p /tmp/comment-json + unique_string= + json_filename=$( \ + echo "comment-json-merge-$image_name-$registry-$language-$peer_group.json" | \ + tr '/' '-' \ + ) + echo "json-filename=$json_filename" >> "$GITHUB_OUTPUT" + json_file="/tmp/comment-json/$json_filename" + jq \ + -n \ + --arg commit_time "$commit_time" \ + --arg git_sha "$git_sha" \ + --arg project_code "$project_code" \ + --arg pacta_data_quarter "$pacta_data_quarter" \ + --arg language "$language" \ + --arg peer_group "$peer_group" \ + --arg report_url "$report_url" \ + --arg summary_url "$summary_url" \ + --arg full_image_name "$full_image_name" \ + '{ + "commit_time": $commit_time, + "git_sha": $git_sha, + "project_code": $project_code, + "holdings_date": $pacta_data_quarter, + "language": $language, + "peer_group": $peer_group, + "report": (if (("" + $report_url) == "") then null else "[Report](" + $report_url + ")" end), + "summary": (if (("" + $summary_url) == "") then null else "[Summary](" + $summary_url + ")" end), + "image": ("`" + $full_image_name + "`") + }' \ + >> $json_file + cat "$json_file" + + - name: Upload comment JSON + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.prepare-artifact.outputs.json-filename }} + path: /tmp/comment-json/* + if-no-files-found: error + retention-days: 1