feat(ingest/fivetran): protect against high sync volume (#11589) #147
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Publish Datahub Java Jars (Client, Spark Lineage, Protobuf, Auth API) | |
| on: | |
| push: | |
| branches: | |
| - master | |
| paths-ignore: | |
| - "docs/**" | |
| - "**.md" | |
| release: | |
| types: [published] | |
| # Allows you to run this workflow manually from the Actions tab | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| check-secret: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| publish-enabled: ${{ steps.publish-enabled.outputs.publish }} | |
| steps: | |
| - id: publish-enabled | |
| env: | |
| SIGNING_KEY: ${{ secrets.SIGNING_KEY }} | |
| run: | | |
| echo "Enable publish: ${{ env.SIGNING_KEY != '' }}" | |
| echo "publish=${{ env.SIGNING_KEY != '' }}" >> $GITHUB_OUTPUT | |
| setup: | |
| if: startsWith(github.ref, 'refs/tags/v') | |
| runs-on: ubuntu-latest | |
| outputs: | |
| tag: ${{ steps.tag.outputs.tag }} | |
| steps: | |
| - name: Checkout | |
| uses: acryldata/sane-checkout-action@v3 | |
| - name: Compute Tag | |
| id: tag | |
| run: | | |
| source .github/scripts/docker_helpers.sh | |
| TAG=$(echo ${GITHUB_REF} | sed -e 's,refs/tags/v,,g') | |
| echo "tag=$TAG" >> $GITHUB_OUTPUT | |
| publish: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| contents: read | |
| needs: ["check-secret", "setup"] | |
| if: ${{ needs.check-secret.outputs.publish-enabled == 'true' }} | |
| steps: | |
| - uses: acryldata/sane-checkout-action@v3 | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: "zulu" | |
| java-version: 17 | |
| - uses: gradle/actions/setup-gradle@v3 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| cache: "pip" | |
| - name: checkout upstream repo | |
| run: | | |
| git remote add upstream https://github.com/datahub-project/datahub.git | |
| git fetch upstream --tags --force --filter=tree:0 | |
| - name: publish datahub-client jar snapshot | |
| if: ${{ github.event_name != 'release' }} | |
| env: | |
| RELEASE_USERNAME: ${{ secrets.RELEASE_USERNAME }} | |
| RELEASE_PASSWORD: ${{ secrets.RELEASE_PASSWORD }} | |
| SIGNING_PASSWORD: ${{ secrets.SIGNING_PASSWORD }} | |
| SIGNING_KEY: ${{ secrets.SIGNING_KEY }} | |
| NEXUS_USERNAME: ${{ secrets.NEXUS_USERNAME }} | |
| NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} | |
| run: | | |
| echo signingKey=$SIGNING_KEY >> gradle.properties | |
| ./gradlew :metadata-integration:java:datahub-client:printVersion | |
| ./gradlew :metadata-integration:java:datahub-client:publish | |
| - name: release datahub-client jar | |
| if: ${{ github.event_name == 'release' }} | |
| env: | |
| RELEASE_USERNAME: ${{ secrets.RELEASE_USERNAME }} | |
| RELEASE_PASSWORD: ${{ secrets.RELEASE_PASSWORD }} | |
| SIGNING_PASSWORD: ${{ secrets.SIGNING_PASSWORD }} | |
| SIGNING_KEY: ${{ secrets.SIGNING_KEY }} | |
| NEXUS_USERNAME: ${{ secrets.NEXUS_USERNAME }} | |
| NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} | |
| run: | | |
| echo signingKey=$SIGNING_KEY >> gradle.properties | |
| ./gradlew -PreleaseVersion=${{ needs.setup.outputs.tag }} :metadata-integration:java:datahub-client:publish | |
| ./gradlew :metadata-integration:java:datahub-client:closeAndReleaseRepository --info | |
| - name: publish datahub-spark-lineage snapshot jar | |
| if: ${{ github.event_name != 'release' }} | |
| env: | |
| RELEASE_USERNAME: ${{ secrets.RELEASE_USERNAME }} | |
| RELEASE_PASSWORD: ${{ secrets.RELEASE_PASSWORD }} | |
| SIGNING_PASSWORD: ${{ secrets.SIGNING_PASSWORD }} | |
| SIGNING_KEY: ${{ secrets.SIGNING_KEY }} | |
| NEXUS_USERNAME: ${{ secrets.NEXUS_USERNAME }} | |
| NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} | |
| run: | | |
| echo signingKey=$SIGNING_KEY >> gradle.properties | |
| ./gradlew :metadata-integration:java:spark-lineage:printVersion | |
| ./gradlew :metadata-integration:java:spark-lineage:publish | |
| - name: release datahub-spark-lineage jar | |
| if: ${{ github.event_name == 'release' }} | |
| env: | |
| RELEASE_USERNAME: ${{ secrets.RELEASE_USERNAME }} | |
| RELEASE_PASSWORD: ${{ secrets.RELEASE_PASSWORD }} | |
| SIGNING_PASSWORD: ${{ secrets.SIGNING_PASSWORD }} | |
| SIGNING_KEY: ${{ secrets.SIGNING_KEY }} | |
| NEXUS_USERNAME: ${{ secrets.NEXUS_USERNAME }} | |
| NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} | |
| run: | | |
| echo signingKey=$SIGNING_KEY >> gradle.properties | |
| ./gradlew -PreleaseVersion=${{ needs.setup.outputs.tag }} :metadata-integration:java:spark-lineage:publish | |
| ./gradlew :metadata-integration:java:spark-lineage:closeAndReleaseRepository --info | |
| - name: publish datahub-protobuf snapshot jar | |
| if: ${{ github.event_name != 'release' }} | |
| env: | |
| RELEASE_USERNAME: ${{ secrets.RELEASE_USERNAME }} | |
| RELEASE_PASSWORD: ${{ secrets.RELEASE_PASSWORD }} | |
| SIGNING_PASSWORD: ${{ secrets.SIGNING_PASSWORD }} | |
| SIGNING_KEY: ${{ secrets.SIGNING_KEY }} | |
| NEXUS_USERNAME: ${{ secrets.NEXUS_USERNAME }} | |
| NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} | |
| run: | | |
| echo signingKey=$SIGNING_KEY >> gradle.properties | |
| ./gradlew :metadata-integration:java:datahub-protobuf:printVersion | |
| ./gradlew :metadata-integration:java:datahub-protobuf:publish | |
| - name: release datahub-protobuf jar | |
| if: ${{ github.event_name == 'release' }} | |
| env: | |
| RELEASE_USERNAME: ${{ secrets.RELEASE_USERNAME }} | |
| RELEASE_PASSWORD: ${{ secrets.RELEASE_PASSWORD }} | |
| SIGNING_PASSWORD: ${{ secrets.SIGNING_PASSWORD }} | |
| SIGNING_KEY: ${{ secrets.SIGNING_KEY }} | |
| NEXUS_USERNAME: ${{ secrets.NEXUS_USERNAME }} | |
| NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} | |
| run: | | |
| echo signingKey=$SIGNING_KEY >> gradle.properties | |
| ./gradlew -PreleaseVersion=${{ needs.setup.outputs.tag }} :metadata-integration:java:datahub-protobuf:publish | |
| ./gradlew :metadata-integration:java:datahub-protobuf:closeAndReleaseRepository --info | |
| - name: publish datahub-auth-api snapshot jar | |
| if: ${{ github.event_name != 'release' }} | |
| env: | |
| RELEASE_USERNAME: ${{ secrets.RELEASE_USERNAME }} | |
| RELEASE_PASSWORD: ${{ secrets.RELEASE_PASSWORD }} | |
| SIGNING_PASSWORD: ${{ secrets.SIGNING_PASSWORD }} | |
| SIGNING_KEY: ${{ secrets.SIGNING_KEY }} | |
| NEXUS_USERNAME: ${{ secrets.NEXUS_USERNAME }} | |
| NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} | |
| run: | | |
| echo signingKey=$SIGNING_KEY >> gradle.properties | |
| ./gradlew :metadata-auth:auth-api:printVersion | |
| ./gradlew :metadata-auth:auth-api:publish | |
| - name: release datahub-auth-api jar | |
| if: ${{ github.event_name == 'release' }} | |
| env: | |
| RELEASE_USERNAME: ${{ secrets.RELEASE_USERNAME }} | |
| RELEASE_PASSWORD: ${{ secrets.RELEASE_PASSWORD }} | |
| SIGNING_PASSWORD: ${{ secrets.SIGNING_PASSWORD }} | |
| SIGNING_KEY: ${{ secrets.SIGNING_KEY }} | |
| NEXUS_USERNAME: ${{ secrets.NEXUS_USERNAME }} | |
| NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} | |
| run: | | |
| echo signingKey=$SIGNING_KEY >> gradle.properties | |
| ./gradlew -PreleaseVersion=${{ needs.setup.outputs.tag }} :metadata-auth:auth-api:publish | |
| ./gradlew :metadata-auth:auth-api:closeAndReleaseRepository --info | |
| - name: publish datahub-custom-plugin-lib snapshot jar | |
| if: ${{ github.event_name != 'release' }} | |
| env: | |
| RELEASE_USERNAME: ${{ secrets.RELEASE_USERNAME }} | |
| RELEASE_PASSWORD: ${{ secrets.RELEASE_PASSWORD }} | |
| SIGNING_PASSWORD: ${{ secrets.SIGNING_PASSWORD }} | |
| SIGNING_KEY: ${{ secrets.SIGNING_KEY }} | |
| NEXUS_USERNAME: ${{ secrets.NEXUS_USERNAME }} | |
| NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} | |
| run: | | |
| echo signingKey=$SIGNING_KEY >> gradle.properties | |
| ./gradlew :metadata-integration:java:custom-plugin-lib:printVersion | |
| ./gradlew :metadata-integration:java:custom-plugin-lib:publish | |
| - name: release datahub-custom-plugin-lib jar | |
| if: ${{ github.event_name == 'release' }} | |
| env: | |
| RELEASE_USERNAME: ${{ secrets.RELEASE_USERNAME }} | |
| RELEASE_PASSWORD: ${{ secrets.RELEASE_PASSWORD }} | |
| SIGNING_PASSWORD: ${{ secrets.SIGNING_PASSWORD }} | |
| SIGNING_KEY: ${{ secrets.SIGNING_KEY }} | |
| NEXUS_USERNAME: ${{ secrets.NEXUS_USERNAME }} | |
| NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} | |
| run: | | |
| echo signingKey=$SIGNING_KEY >> gradle.properties | |
| ./gradlew -PreleaseVersion=${{ needs.setup.outputs.tag }} :metadata-integration:java:custom-plugin-lib:publish | |
| ./gradlew :metadata-integration:java:custom-plugin-lib:closeAndReleaseRepository --info |