build-push-private.yml

on:
  workflow_call:
    inputs:
      image-name:
        required: true
        type: string
      additional-image-tags:
        required: false
        type: string
      registry:
        required: false
        type: string
        default: transitionmonitordockerregistry.azurecr.io

    outputs:
      full-image-name:
        description: "Full pushed image name including host/registry, name, and tag"
        value: ${{ jobs.docker-build.outputs.full-image-name }}
      report-url:
        description: "URL to report generated in testing step"
        value: ${{ jobs.test.outputs.report-url }}

jobs:
  read-config:
    runs-on: ubuntu-latest
    permissions:
      contents: read
    outputs:
      net-zero-targets: ${{ steps.prepare.outputs.net-zero-targets }}
      now: ${{ steps.prepare.outputs.now }}
      pacta-data-quarter: ${{ steps.prepare.outputs.pacta-data-quarter }}
      pacta-data-share-path: ${{ steps.prepare.outputs.pacta-data-share-path }}
      peer-results: ${{ steps.prepare.outputs.peer-results }}
      project-code: ${{ steps.prepare.outputs.project-code }}
      registry-image: ${{ steps.prepare.outputs.registry-image }}
      templates-ref: ${{ steps.prepare.outputs.templates-ref }}
      test-matrix: ${{ steps.prepare.outputs.test-matrix }}
      user-results: ${{ steps.prepare.outputs.user-results }}
      results-destination-url: ${{ steps.prepare.outputs.results-destination-url }}
      reports-destination-url: ${{ steps.prepare.outputs.reports-destination-url }}
    steps:

      - name: Checkout workflow.transition.monitor
        uses: actions/checkout@v4
        with:
          path: workflow.transition.monitor

      - name: Prepare environment
        id: prepare
        env:
          image_name: ${{ inputs.image-name }}
          registry: ${{ inputs.registry }}
        run: |
          NOW="$(date -u +'%Y%m%dT%H%M%SZ')"
          echo "now=$NOW" >> $GITHUB_OUTPUT
          echo "$NOW"

          registry_image=$(
            echo "$registry/$image_name" | \
            tr '[:upper:]' '[:lower:]' \
          )
          REGISTRY_IMAGE=${registry_image}
          echo "registry-image=$REGISTRY_IMAGE"
          echo "registry-image=$REGISTRY_IMAGE" >> $GITHUB_OUTPUT

          config_file="workflow.transition.monitor/build/config/$image_name.json"

          PACTA_DATA_SHARE_PATH="$(jq -rc '.data_share_path' $config_file)"
          echo "pacta-data-share-path=$PACTA_DATA_SHARE_PATH"
          echo "pacta-data-share-path=$PACTA_DATA_SHARE_PATH" >> "$GITHUB_OUTPUT"

          PACTA_DATA_QUARTER="$(jq -rc '.pacta_data_quarter' $config_file)"
          echo "pacta-data-quarter=$PACTA_DATA_QUARTER"
          echo "pacta-data-quarter=$PACTA_DATA_QUARTER" >> "$GITHUB_OUTPUT"

          TEMPLATES_REF="$(jq -rc '.templates_ref' $config_file)"
          echo "templates-ref=$TEMPLATES_REF"
          echo "templates-ref=$TEMPLATES_REF" >> "$GITHUB_OUTPUT"

          TEST_MATRIX="$(jq -c '.test_matrix' $config_file)"
          echo "test-matrix=$TEST_MATRIX"
          echo "test-matrix=$TEST_MATRIX" >> "$GITHUB_OUTPUT"

          PROJECT_CODE="$(jq -r '.project_code' $config_file)"
          echo "project-code=$PROJECT_CODE"
          echo "project-code=$PROJECT_CODE" >> "$GITHUB_OUTPUT"

          RESULTS_DESTINATION_URL="$(jq -r '.resultsDestinationURL' $config_file)"
          echo "results-destination-url=$RESULTS_DESTINATION_URL"
          echo "results-destination-url=$RESULTS_DESTINATION_URL" >> "$GITHUB_OUTPUT"

          REPORTS_DESTINATION_URL="$(jq -r '.reportsDestinationURL' $config_file)"
          echo "reports-destination-url=$REPORTS_DESTINATION_URL"
          echo "reports-destination-url=$REPORTS_DESTINATION_URL" >> "$GITHUB_OUTPUT"

          # includes handling for null/missing keys
          PEER_RESULTS="$(jq -r '.peer_results | select( . != null )' $config_file)"
          echo "peer-results=$PEER_RESULTS"
          echo "peer-results=$PEER_RESULTS" >> "$GITHUB_OUTPUT"

          # includes handling for null/missing keys
          NET_ZERO_TARGETS="$(jq -r '.net_zero_targets | select( . != null )' $config_file)"
          echo "net-zero-targets=$NET_ZERO_TARGETS"
          echo "net-zero-targets=$NET_ZERO_TARGETS" >> "$GITHUB_OUTPUT"

  prepare-indices:
    name: Run Index Preparation
    needs: read-config
    secrets: inherit
    uses: RMI-PACTA/workflow.prepare.pacta.indices/.github/workflows/run-index-preparation.yml@main
    with:
      image-tag: |
        ${{ github.event_name == 'pull_request' && format('{0}{1}','pr', github.event.pull_request.number) ||
          github.event_name == 'schedule' && 'main' ||
          github.event_name == 'push' && github.ref_name ||
          github.event_name == 'workflow_dispatch' && github.ref_name ||
          github.sha }}
      data-share-path: ${{ needs.read-config.outputs.pacta-data-share-path }}
      config_active: ${{ needs.read-config.outputs.pacta-data-quarter }}

  docker-build:
    runs-on: ubuntu-latest
    needs: [read-config, prepare-indices]
    permissions:
      contents: read
      id-token: write
    timeout-minutes: 45
    outputs:
      full-image-name: ${{ steps.export-outputs.outputs.full-image-name }}

    steps:
      - name: Checkout workflow.transition.monitor
        uses: actions/checkout@v4
        with:
          path: workflow.transition.monitor

      - name: Checkout templates.transition.monitor
        uses: actions/checkout@v4
        with:
          repository: RMI-PACTA/templates.transition.monitor
          path: templates.transition.monitor
          token: ${{ secrets.REPO_PAT }}
          ref: ${{ needs.read-config.outputs.templates-ref }}

      # https://github.com/Azure/login?tab=readme-ov-file#login-with-openid-connect-oidc-recommended
      - name: Azure Login
        uses: azure/login@v2
        with:
          client-id: ${{ secrets.AZURE_CLIENT_ID }}
          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}

      # https://github.com/marketplace/actions/azure-cli-action#workflow-to-execute-an-azure-cli-script-of-a-specific-cli-version
      - name: Download pacta-data
        uses: azure/CLI@v2
        env:
          index_share_path: ${{ needs.prepare-indices.outputs.results-dir }}
          pacta_data_quarter: ${{ needs.read-config.outputs.pacta-data-quarter }}
          pacta_data_share_path: ${{ needs.read-config.outputs.pacta-data-share-path }}
          peer_results: ${{ needs.read-config.outputs.peer-results }}
          net_zero_targets: ${{ needs.read-config.outputs.net-zero-targets }}
        with:
          # azcliversion: 2.30.0
          inlineScript: |
            pacta_data_share_url="https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs"
            az storage copy \
              --source "$pacta_data_share_url/$pacta_data_share_path" \
              --destination "pacta-data" \
              --recursive \
              --exclude-pattern "*.sqlite"
            # download indices
            index_share_url="https://pactadatadev.file.core.windows.net/workflow-prepare-pacta-indices-outputs"
            az storage copy \
              --source "$index_share_url/$index_share_path" \
              --destination "index-data" \
              --recursive \
              --exclude-pattern "*.sqlite"

            # Download peer results if necessary
            if [ -n "$peer_results" ]; then
              az storage copy \
                --source "$peer_results/*" \
                --destination "peer_results" \
                --recursive
            else
              echo "No Peer Results defined in config"
            fi

            # Download net zero targets if necessary
            if [ -n "$net_zero_targets" ]; then
              az storage copy \
                --source "$net_zero_targets" \
                --destination "peer_results" \
                --recursive
            else
              echo "No Peer Results defined in config"
            fi

            # move data
            mv "pacta-data/$pacta_data_share_path" "pacta-data/$pacta_data_quarter"
            mv "index-data/$index_share_path"/* "pacta-data/$pacta_data_quarter"
            if [ -n "$peer_results" ]; then
              mv peer_results/* "pacta-data/$pacta_data_quarter"
            fi
            ls pacta-data/$pacta_data_quarter

      - name: Identify LABELs in dockerfile
        id: custom-labels
        run: |
          DOCKERFILE_LABELS="$(grep "^LABEL" workflow.transition.monitor/Dockerfile | sed 's/^LABEL[[:space:]]*//')"
          echo "$DOCKERFILE_LABELS"
          echo 'dockerfile-labels<<EOF' >> $GITHUB_OUTPUT
          echo "$DOCKERFILE_LABELS" >> $GITHUB_OUTPUT
          echo 'EOF' >> $GITHUB_OUTPUT

      # Setup docker metadata, including tags and labels (and annotations)
      - name: Docker meta
        id: meta
        uses: docker/metadata-action@v5
        env:
          additional_image_tags: ${{ inputs.additional-image-tags }}
          now: ${{ needs.read-config.outputs.now }}
        with:
          images: ${{ needs.read-config.outputs.registry-image }}
          annotations:
            ${{ steps.custom-labels.outputs.DOCKERFILE_LABELS }}
          labels:
            ${{ steps.custom-labels.outputs.DOCKERFILE_LABELS }}
          tags: |
            type=schedule,enable=true,pattern={{date 'YYYYMMDD[T]HHmmss[Z]' tz='UTC'}}
            type=schedule,enable=true,pattern=nightly,priority=950
            type=raw,value=latest,enable={{is_default_branch}}
            type=ref,event=branch
            type=ref,event=tag
            type=ref,event=pr
            ${{ inputs.additional-image-tags }}
            ${{ needs.read-config.outputs.now }},priority=1100

      # Set up Dockerignore file
      - name: Copy dockerignore
        run: |
          cp ./workflow.transition.monitor/.dockerignore.CIbuild .dockerignore

      # set up our build environment
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#example-masking-a-generated-output-within-a-single-job
      - name: Get credentials for docker registry
        id: docker-credentials
        uses: azure/CLI@v2
        env:
          registry: ${{ inputs.registry }}
        with:
          # azcliversion: 2.30.0
          inlineScript: |
            # Username is not a placeholder. See https://stackoverflow.com/a/73250630
            DOCKER_USER="00000000-0000-0000-0000-000000000000" >> "$GITHUB_OUTPUT"
            DOCKER_TOKEN="$(az acr login -n $registry --expose-token --query 'accessToken' --output tsv)" >> "$GITHUB_OUTPUT"
            echo "::add-mask::$DOCKER_TOKEN"
            echo "docker-user=$DOCKER_USER" >> "$GITHUB_OUTPUT"
            echo "docker-token=$DOCKER_TOKEN" >> "$GITHUB_OUTPUT"

      - name: Login to Registry
        env:
          REGISTRY: ${{ inputs.registry }}
          DOCKER_TOKEN: ${{ steps.docker-credentials.outputs.docker-token }}
          DOCKER_USER: ${{ steps.docker-credentials.outputs.docker-user }}
        run: |
          echo "$DOCKER_TOKEN" | docker login $REGISTRY --username "$DOCKER_USER" --password-stdin

      # Actually build the image (for a single architecture)!
      - name: Build
        id: build
        uses: docker/build-push-action@v6
        with:
          context: .
          file: ./workflow.transition.monitor/Dockerfile
          # include the labels from the meta step
          labels: ${{ steps.meta.outputs.labels }}
          # the same for annotations
          annotations: ${{ steps.meta.outputs.annotations }}
          push: true
          tags: ${{ steps.meta.outputs.tags }}
          # Use the GitHub actions cache to speed up repeated builds
          cache-from: type=gha
          cache-to: type=gha,mode=min
          # but don't cache the install pacta step.
          no-cache-filters: install-pacta

      - name: Export Outputs
        id: export-outputs
        run: |
          TAGGED_IMAGE="$(jq -rc '.tags[0]' <<< "$DOCKER_METADATA_OUTPUT_JSON")"
          echo "full-image-name=$TAGGED_IMAGE"
          echo "full-image-name=$TAGGED_IMAGE" >> "$GITHUB_OUTPUT"

      - name: Prepare inputs archive
        id: prepare-inputs-archive
        env:
          full_image_name: ${{ steps.export-outputs.outputs.full-image-name }}
        run: |
          tarfile="$(echo $full_image_name | sed 's:.*/::' |  tr ":" "-").tar.gz"
          tar -cvz -f "$tarfile" pacta-data templates.transition.monitor
          echo "$tarfile"
          echo "input-tar=$tarfile"
          echo "input-tar=$tarfile" >> "$GITHUB_OUTPUT"


      - name: Export inputs archive
        id: upload-report
        uses: RMI-PACTA/actions/actions/azure/blob-copy@main
        with:
          source: ${{ steps.prepare-inputs-archive.outputs.input-tar }}
          destination: https://pactadatadev.blob.core.windows.net/ghactions-workflow-transition-monitor-input-pacta-data
          overwrite: false

  test:
    runs-on: ubuntu-latest
    strategy:
      # if one test fails, permit the others to run
      fail-fast: false
      # build images in parallel
      matrix: ${{ fromJson(needs.read-config.outputs.test-matrix) }}
    needs:
      - read-config
      - docker-build
    permissions:
      contents: read
      id-token: write
    timeout-minutes: 30
    outputs:
      report-url: ${{ steps.export-outputs.outputs.report-url }}

    steps:

      # https://github.com/Azure/login?tab=readme-ov-file#login-with-openid-connect-oidc-recommended
      - name: Azure Login
        uses: azure/login@v2
        with:
          client-id: ${{ secrets.AZURE_CLIENT_ID }}
          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}

      - name: Checkout workflow.transition.monitor
        uses: actions/checkout@v4

      - name: Prepare User and results directories
        id: prepare-results-dir
        env:
          full_image_name: ${{ needs.docker-build.outputs.full-image-name }}
          image_name: ${{ inputs.image-name }}
          language: ${{ matrix.language }}
          pacta_data_quarter: ${{ needs.read-config.outputs.pacta-data-quarter }}
          peer_group: ${{ matrix.peer_group }}
          project_code: ${{ needs.read-config.outputs.project-code }}
          run_attempt: ${{ github.run_attempt }}
          user_id: ${{ matrix.user_id }}
        run: |
          test_dir_parent_name="$(echo $full_image_name | sed 's:.*/::' |  tr ":" "-")"
          TEST_DIR_PARENT="$test_dir_parent_name"
          echo "TEST_DIR_PARENT=$TEST_DIR_PARENT"
          echo "test-dir-parent=$TEST_DIR_PARENT" >> "$GITHUB_OUTPUT"

          TEST_DIR="$TEST_DIR_PARENT/$language/$peer_group/$run_attempt"
          mkdir -p "$TEST_DIR"
          chmod -R 777 "$TEST_DIR"
          echo "TEST_DIR=$TEST_DIR"
          echo "test-dir=$TEST_DIR" >> "$GITHUB_OUTPUT"

          mkdir -p $TEST_DIR/user_dir
          cp -R working_dir $TEST_DIR

          params_file="$TEST_DIR/working_dir/10_Parameter_File/${image_name}_PortfolioParameters.yml"
          echo $params_file
          mkdir -p "$(dirname $params_file)"
          sed \
            -e "s/{{portfolio_name}}/$image_name-PORTFOLIO/" \
            -e "s/{{investor_name}}/$image_name-$peer_group-INVESTOR/" \
            -e "s/{{peer_group}}/$peer_group/" \
            -e "s/{{language}}/$language/" \
            -e "s/{{project_code}}/$project_code/" \
            -e "s/{{holdings_date}}/$pacta_data_quarter/" \
            -e "s/{{user_id}}/$user_id/" \
            working_dir/10_Parameter_File/test_PortfolioParameters.yml \
            >> $params_file
          cat "$params_file"

          portfolio_file="$TEST_DIR/working_dir/20_Raw_Inputs/${image_name}.csv"
          mkdir -p "$(dirname $portfolio_file)"
          mv working_dir/20_Raw_Inputs/1234.csv $portfolio_file
          ls -lR $TEST_DIR

      - name: Download User Results from Blob store
        id: download-user-results
        uses: RMI-PACTA/actions/actions/azure/blob-copy@main
        if: ${{ matrix.user_results }}
        with:
          source: ${{ matrix.user_results }}/${{ matrix.user_id }}
          destination: ${{ steps.prepare-results-dir.outputs.test-dir }}/user_dir
          overwrite: false

      - name: Move user results to correct location
        shell: sh
        if: ${{ matrix.user_results }}
        env:
          USER_RESULTS_DIR: ${{ steps.prepare-results-dir.outputs.test-dir }}/user_dir
          BLOB_SOURCE: ${{ matrix.user_results }}/${{ matrix.user_id }}
          USER_ID: ${{ matrix.user_id }}
        run: |
          BLOB_STUB="$(echo $BLOB_SOURCE | sed -e 's/.*windows.net\/[^\/]*\///')"
          echo "Blob stub: $BLOB_STUB"
          TEMP_USER_DIR="$USER_RESULTS_DIR/$BLOB_STUB"
          echo "Temp Dir user: $TEMP_USER_DIR"
          if [ -d $TEMP_USER_DIR ]; then
            mv \
              --verbose \
              $TEMP_USER_DIR/* \
              $USER_RESULTS_DIR
          fi

      # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#example-masking-a-generated-output-within-a-single-job
      - name: Get credentials for docker registry
        id: docker-credentials
        uses: azure/CLI@v2
        env:
          registry: ${{ inputs.registry }}
        with:
          # azcliversion: 2.30.0
          inlineScript: |
            # Username is not a placeholder. See https://stackoverflow.com/a/73250630
            DOCKER_USER="00000000-0000-0000-0000-000000000000" >> "$GITHUB_OUTPUT"
            DOCKER_TOKEN="$(az acr login -n $registry --expose-token --query 'accessToken' --output tsv)" >> "$GITHUB_OUTPUT"
            echo "::add-mask::$DOCKER_TOKEN"
            echo "docker-user=$DOCKER_USER" >> "$GITHUB_OUTPUT"
            echo "docker-token=$DOCKER_TOKEN" >> "$GITHUB_OUTPUT"

      - name: Login to Registry
        env:
          REGISTRY: ${{ inputs.registry }}
          DOCKER_TOKEN: ${{ steps.docker-credentials.outputs.docker-token }}
          DOCKER_USER: ${{ steps.docker-credentials.outputs.docker-user }}
        run: |
          echo "$DOCKER_TOKEN" | docker login $REGISTRY --username "$DOCKER_USER" --password-stdin

      - name: Test
        env:
          full_image_name: ${{ needs.docker-build.outputs.full-image-name }}
          image_name: ${{ inputs.image-name }}
          test_dir: ${{ steps.prepare-results-dir.outputs.test-dir }}
          workspace: ${{ github.workspace }}
        run: |
          chmod -R 777 "$test_dir/working_dir"
          docker run \
            --network none \
            --user root \
            --mount "type=bind,source=$workspace/$test_dir/working_dir,target=/bound/working_dir" \
            --mount "type=bind,readonly,source=$workspace/$test_dir/user_dir,target=/user_results" \
            $full_image_name \
            /bound/bin/run-r-scripts $image_name

      # https://github.com/Azure/login?tab=readme-ov-file#login-with-openid-connect-oidc-recommended
      - name: Azure Login
        uses: azure/login@v2
        with:
          client-id: ${{ secrets.AZURE_CLIENT_ID }}
          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}

      - name: Upload results to Blob store
        id: upload-results
        uses: RMI-PACTA/actions/actions/azure/blob-copy@main
        with:
          source: ${{ steps.prepare-results-dir.outputs.test-dir }}
          destination: https://pactadatadev.blob.core.windows.net/ghactions-workflow-transition-monitor-results-full
          overwrite: false

      - name: Upload report to Blob store
        id: upload-report
        uses: RMI-PACTA/actions/actions/azure/blob-copy@main
        with:
          source: ${{ steps.prepare-results-dir.outputs.test-dir }}/working_dir/50_Outputs
          destination: https://pactadatadev.blob.core.windows.net/ghactions-workflow-transition-monitor-results-reports
          overwrite: false

      - name: Export Outputs
        id: export-outputs
        env:
          REPORT_UPLOADED_FILES: ${{ steps.upload-report.outputs.destination-files }}
          image_name: ${{ inputs.image-name }}
          test_dir: ${{ steps.prepare-results-dir.outputs.test-dir }}
        run: |

          REPORT_URL="$(
            echo "$REPORT_UPLOADED_FILES" | jq -rc '. [] | match(".*index.html$") | .string'
          )"
          echo "report-url=$REPORT_URL"
          echo "report-url=$REPORT_URL" >> "$GITHUB_OUTPUT"

          SUMMARY_PATH="${test_dir}/working_dir/50_Outputs/$image_name/executive_summary/template.pdf"
          if [ -f "$SUMMARY_PATH" ]; then
            SUMMARY_URL="$(
              echo "$REPORT_UPLOADED_FILES" | jq -rc '. [] | match(".*template.pdf$") | .string'
            )"
            echo "summary-url=$SUMMARY_URL"
            echo "summary-url=$SUMMARY_URL" >> "$GITHUB_OUTPUT"
          fi

      - name: Prepare comment artifact
        id: prepare-artifact
        env:
          commit_time: ${{ github.event.pull_request.updated_at }}
          full_image_name: ${{ needs.docker-build.outputs.full-image-name }}
          git_sha: ${{ github.event.pull_request.head.sha }},
          image_name: ${{ inputs.image-name }}
          language: ${{ matrix.language }}
          pacta_data_quarter: ${{ needs.read-config.outputs.pacta-data-quarter }}
          peer_group: ${{ matrix.peer_group }}
          project_code: ${{ needs.read-config.outputs.project-code }}
          registry: ${{ inputs.registry }}
          report_url: ${{ steps.export-outputs.outputs.report-url }}
          run_attempt: ${{ github.run_attempt }}
          summary_url: ${{ steps.export-outputs.outputs.summary-url }}
        run: |
          mkdir -p /tmp/comment-json
          unique_string=
          json_filename=$( \
            echo "comment-json-merge-$image_name-$registry-$language-$peer_group.json" | \
            tr '/' '-' \
            )
          echo "json-filename=$json_filename" >> "$GITHUB_OUTPUT"
          json_file="/tmp/comment-json/$json_filename"
          jq \
            -n \
            --arg commit_time "$commit_time" \
            --arg git_sha "$git_sha" \
            --arg project_code "$project_code" \
            --arg pacta_data_quarter "$pacta_data_quarter" \
            --arg language "$language" \
            --arg peer_group "$peer_group" \
            --arg report_url "$report_url" \
            --arg summary_url "$summary_url" \
            --arg full_image_name "$full_image_name" \
            '{
              "commit_time": $commit_time,
              "git_sha": $git_sha,
              "project_code": $project_code,
              "holdings_date": $pacta_data_quarter,
              "language": $language,
              "peer_group": $peer_group,
              "report": (if (("" + $report_url) == "") then null else "[Report](" + $report_url + ")" end),
              "summary": (if (("" + $summary_url) == "") then null else "[Summary](" + $summary_url + ")" end),
              "image": ("`" + $full_image_name + "`")
            }' \
          >> $json_file
          cat "$json_file"

      - name: Upload comment JSON
        uses: actions/upload-artifact@v4
        with:
          name: ${{ steps.prepare-artifact.outputs.json-filename }}
          path: /tmp/comment-json/*
          if-no-files-found: error
          retention-days: 1