From 624a98009db6eb1aba9bfe3d80adbae853d6866c Mon Sep 17 00:00:00 2001 From: Mathieu Westphal Date: Sun, 17 Sep 2023 16:31:23 +0200 Subject: [PATCH 01/14] Add first version of the action --- README.md | 96 +++++++++++++++++++++++++++++++++++++++++++++++++- action.yml | 82 ++++++++++++++++++++++++++++++++++++++++++ copy_lfs.cmake | 12 +++++++ 3 files changed, 189 insertions(+), 1 deletion(-) create mode 100644 action.yml create mode 100644 copy_lfs.cmake diff --git a/README.md b/README.md index 6578f12..6ef910d 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,96 @@ # lfs-data-cache-action -A github action to cache and recover LFS data + +A producer/consumer github action to cache and recover LFS data. + +Its main use is to avoid cloning LFS data in CI to avoid +haing to pay for LFS bandwidth because of CI needs. +It needs cmake to be available on the host. + +The action can be used as a consumer or a producer, and needs +a SHA to recover LFS data with. + +Is has the following inputs: + + - `type`: should be `producer` or `consumer`, default producer + - `repository`: the git repository to recover LFS data from, required + - `lfs_sha`: The git sha to recover LFS data from, required + - `cache_index`: An index used in the cache name, default is 0 + - `target_directory`: A target directory to copy LFS data to + +## Logic + +Producer/Consumer first use the classic cache action to recover a cache named +`lfs-data-${{lfs_sha}}-${{cache_index}}`. + +If Producer does not found it, it will clone the `repository` at `lfs_sha` commit +and upload the content as an artifact. + +If Consumer does not found it, it will try to download a potential artifact +produced earlier by the Producer. + +If it fails Consumer will clone the `repository` at `lfs_sha` commit. + +Finally, Producer/Consumer will copy the LFS data only using cmake to the `target_directory` + +## Usage + +In an first job, recover the LFS sha to recover and use the `producer` action, output the LFS sha +In a second job depending on the first, recover the LFS sha from first job and use the `consumer` action. + +``` +jobs: + +#---------------------------------------------------------------------------- +# Cache LFS: Checkout LFS data and update the cache to limit LFS bandwidth +#---------------------------------------------------------------------------- + cache_lfs: + runs-on: ubuntu-latest + name: Update LFS data cache + outputs: + lfs_sha: ${{ steps.lfs_sha_recover.outputs.lfs_sha }} + steps: + + # Checkout WITHOUT LFS as the data itself is not needed at this point + - name: Checkout + uses: actions/checkout@v3 + with: + path: 'source' + fetch-depth: 0 + lfs: false + + # Recover the last time LFS data was changed on the repository + # TODO: Update the list of directory you want to watch for changes + - name: Set LFS env var + working-directory: ${{github.workspace}}/source + id: lfs_sha_recover + shell: bash + run: echo "lfs_sha=$(git log -n 1 --pretty=format:%H -- path/to/lfs/data path/to/lfs/data/again)" >> $GITHUB_OUTPUT + + # Use producer action to recover the LFS data and upload it as cache/artifacts + - name: Cache LFS Data + uses: f3d-app/lfs-data-cache-action:latest + with: + workflow_label: 'producer' + repository: 'your/repo' + lfs_sha: ${{ steps.lfs_sha_recover.outputs.lfs_sha }} + target_directory: 'source' + + recover_lfs: + needs: cache_lfs + + # Checkout WITHOUT LFS as the data itself is not needed at this point + - name: Checkout + uses: actions/checkout@v3 + with: + path: 'source' + fetch-depth: 0 + lfs: false + + - name: Recover LFS Data + uses: f3d-app/lfs-data-cache-action:latest + with: + workflow_label: 'consumer' + repository: 'your/repo' + lfs_sha: ${{ inputs.lfs_sha}} + target_directory: 'source' +``` diff --git a/action.yml b/action.yml new file mode 100644 index 0000000..1baa1dd --- /dev/null +++ b/action.yml @@ -0,0 +1,82 @@ +name: 'Copy LFS data' +description: 'Copy LFS data using cache when possible' +inputs: + type: + description: 'Whether this action produce/consume LFS data' + default: 'producer' + repository: + description: 'Repository to recover LFS data from' + required: true + lfs_sha: + description: 'LFS sha to recover' + required: true + cache_index: + description: 'An index used in the name of the cache' + default: '0' + target_directory: + description: 'Directory to copy LFS data to' + required: true + +runs: + using: "composite" + steps: + + - name: Check required inputs + shell: bash + run: | + [[ "${{ inputs.repository }}" ]] || { echo "repository input is empty" ; exit 1; } + [[ "${{ inputs.lfs_sha }}" ]] || { echo "lfs_sha input is empty" ; exit 1; } + [[ "${{ inputs.target_directory }}" ]] || { echo "target_directory input is empty" ; exit 1; } + + - name: Cache LFS data + id: cache-lfs + uses: actions/cache@v3 + with: + path: lfs_data + key: lfs-data-${{inputs.lfs_sha}}-${{inputs.cache_index}} + + - name: Checkout LFS data + if: | + steps.cache-lfs.outputs.cache-hit != 'true' && + inputs.type == 'producer' + uses: actions/checkout@v3 + with: + path: 'lfs_data' + fetch-depth: 0 + lfs: true + + - name: Upload LFS artifact + if: | + steps.cache-lfs.outputs.cache-hit != 'true' && + inputs.type == 'producer' + uses: actions/upload-artifact@master + with: + name: lfs-data + path: lfs_data + + - name: Download LFS artifact + id: download-artifact + if: | + steps.cache-lfs.outputs.cache-hit != 'true' && + inputs.type == 'consumer' + uses: actions/download-artifact@master + continue-on-error: true + with: + name: lfs-data + path: lfs_data + + - name: Checkout LFS data + if: | + steps.cache-lfs.outputs.cache-hit != 'true' && + steps.download-artifact.outcome != 'success' && + inputs.type == 'consumer' + uses: actions/checkout@v3 + with: + path: 'lfs_data' + fetch-depth: 0 + lfs: true + + - name: Setup LFS data + working-directory: ${{github.workspace}} + shell: bash + run: cmake -P $GITHUB_ACTION_PATH/copy_lfs.cmake ./lfs_data ${{ inputs.target_directory }} diff --git a/copy_lfs.cmake b/copy_lfs.cmake new file mode 100644 index 0000000..6cc665f --- /dev/null +++ b/copy_lfs.cmake @@ -0,0 +1,12 @@ +find_package(Git QUIET) +execute_process( + COMMAND ${GIT_EXECUTABLE} lfs ls-files -n + WORKING_DIRECTORY ${CMAKE_ARGV3} + OUTPUT_VARIABLE LFS_FILES + OUTPUT_STRIP_TRAILING_WHITESPACE) +string(REPLACE "\n" ";" LFS_FILES ${LFS_FILES}) + +foreach(LFS_FILE ${LFS_FILES}) + cmake_path(GET LFS_FILE PARENT_PATH LFS_PATH) + file(COPY "${CMAKE_ARGV3}/${LFS_FILE}" DESTINATION "${CMAKE_ARGV4}/${LFS_PATH}") +endforeach() From aca29443738d836bbae55061a5d78c17ec56a2a0 Mon Sep 17 00:00:00 2001 From: Mathieu Westphal Date: Wed, 4 Sep 2024 08:43:11 +0200 Subject: [PATCH 02/14] cleanup --- action.yml | 75 +++++++++++++++++++++++++++++++++++------------------- 1 file changed, 49 insertions(+), 26 deletions(-) diff --git a/action.yml b/action.yml index 1baa1dd..3a977ff 100644 --- a/action.yml +++ b/action.yml @@ -4,18 +4,18 @@ inputs: type: description: 'Whether this action produce/consume LFS data' default: 'producer' - repository: + repository: description: 'Repository to recover LFS data from' required: true lfs_sha: - description: 'LFS sha to recover' - required: true - cache_index: - description: 'An index used in the name of the cache' - default: '0' + description: 'LFS sha to recover. If not set, target directory will be used to recover it.' + required: false + cache_postfix: + description: 'A postfix to differentiate between caches' + default: 'cache' target_directory: description: 'Directory to copy LFS data to' - required: true + default: 'source' runs: using: "composite" @@ -25,23 +25,43 @@ runs: shell: bash run: | [[ "${{ inputs.repository }}" ]] || { echo "repository input is empty" ; exit 1; } - [[ "${{ inputs.lfs_sha }}" ]] || { echo "lfs_sha input is empty" ; exit 1; } - [[ "${{ inputs.target_directory }}" ]] || { echo "target_directory input is empty" ; exit 1; } - + + - name: Create a working directory + working-directory: ${{github.workspace}} + shell: bash + run: mkdir lfs-data-cache + + - name: Checkout repository without LFS data + if: inputs.lfs_sha == '' + uses: actions/checkout@v4 + with: + repository: ${{ inputs.repository }} + path: 'lfs-data-cache/lfs_source' + fetch-depth: 0 + lfs: false + + - name: Set LFS output var without inputs + if: inputs.lfs_sha == '' + working-directory: ${{github.workspace}}/lfs-data-cache/lfs_source + id: lfs_sha_recover + shell: bash + run: echo "lfs_sha=$(git log -n 1 --pretty=format:%H -- `git-lfs ls-files -n`)" >> $GITHUB_OUTPUT + - name: Cache LFS data id: cache-lfs - uses: actions/cache@v3 + uses: actions/cache@v4 with: - path: lfs_data - key: lfs-data-${{inputs.lfs_sha}}-${{inputs.cache_index}} + path: 'lfs-data-cache/lfs_data' + key: lfs-data-${{inputs.lfs_sha == '' && ${{steps.lfs_sha_recover.outputs.lfs_sha}} || ${{inputs.lfs_sha}} }}-${{inputs.cache_postfix}} - - name: Checkout LFS data + - name: Checkout LFS data for artifact producer if: | steps.cache-lfs.outputs.cache-hit != 'true' && inputs.type == 'producer' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: - path: 'lfs_data' + repository: ${{ inputs.repository }} + path: 'lfs-data-cache/lfs_data' fetch-depth: 0 lfs: true @@ -49,34 +69,37 @@ runs: if: | steps.cache-lfs.outputs.cache-hit != 'true' && inputs.type == 'producer' - uses: actions/upload-artifact@master + uses: actions/upload-artifact@v4 with: - name: lfs-data - path: lfs_data + name: lfs-data-${{inputs.cache_postfix}} + path: 'lfs-data-cache/lfs_data' + overwrite: true + include-hidden-files: true - name: Download LFS artifact id: download-artifact if: | steps.cache-lfs.outputs.cache-hit != 'true' && inputs.type == 'consumer' - uses: actions/download-artifact@master + uses: actions/download-artifact@v4 continue-on-error: true with: - name: lfs-data - path: lfs_data + name: lfs-data-${{inputs.cache_postfix}} + path: 'lfs-data-cache/lfs_data' - - name: Checkout LFS data + - name: Checkout LFS data (last resort) if: | steps.cache-lfs.outputs.cache-hit != 'true' && steps.download-artifact.outcome != 'success' && inputs.type == 'consumer' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: - path: 'lfs_data' + repository: ${{ inputs.repository }} + path: 'lfs-data-cache/lfs_data' fetch-depth: 0 lfs: true - name: Setup LFS data working-directory: ${{github.workspace}} shell: bash - run: cmake -P $GITHUB_ACTION_PATH/copy_lfs.cmake ./lfs_data ${{ inputs.target_directory }} + run: cmake -P $GITHUB_ACTION_PATH/copy_lfs.cmake 'lfs-data-cache/lfs_data' ${{ inputs.target_directory }} From 358004077dbb8d7f7f755fc9a07fcd599f35284d Mon Sep 17 00:00:00 2001 From: Mathieu Westphal Date: Wed, 4 Sep 2024 09:04:21 +0200 Subject: [PATCH 03/14] try CI --- .github/workflows/ci.yml | 79 ++++++++++++++++++++++++++++++++++++++++ action.yml | 4 +- 2 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..9e78f16 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,79 @@ +name: CI + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + push: + branches: + - main + +concurrency: + group: '${{ github.workflow }}-${{ github.ref_name }}' + cancel-in-progress: true + +jobs: + +#---------------------------------------------------------------------------- +# Cache LFS: Checkout LFS data and update the cache to limit LFS bandwidth +#---------------------------------------------------------------------------- + cache_lfs: + runs-on: ubuntu-latest + name: Update LFS data cache + outputs: + lfs_sha: ${{ steps.lfs_sha.outputs.lfs_sha }} + steps: + + - name: Checkout + uses: actions/checkout@v4 + with: + path: 'lfs-data-cache_action' + fetch-depth: 0 + + - name: Use lfs-data-cache action + id: lfs-data-cache + uses: ./lfs-data-cache_action + with: + type: 'producer' + repository: 'f3d-app/f3d' + cache_postfix: 'ci-cache' + + - name: Set output + id: lfs_sha + shell: bash + run: echo "lfs_sha=$(steps.lfs-data-cache.outputs.lfs_sha)" >> $GITHUB_OUTPUT + + ci: + name: CI + needs: cache_lfs + + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-13] + + runs-on: ${{matrix.os}} + + steps: + + - name: Output directory + shell: bash + run: mkdir output_dir + + - name: Checkout + uses: actions/checkout@v4 + with: + path: 'lfs-data-cache_action' + fetch-depth: 0 + + - name: Use lfs-data-cache action + uses: ./lfs-data-cache_action + with: + type: 'consumer' + repository: 'f3d-app/f3d' + lfs_sha: ${{ needs.cache_lfs.outputs.lfs_sha}} + cache_postfix: 'ci-cache' + target_directory: 'output_dir' + + - name: Checkout output directory + shell: bash + run: ls output_dir/ diff --git a/action.yml b/action.yml index 3a977ff..18b7750 100644 --- a/action.yml +++ b/action.yml @@ -14,8 +14,10 @@ inputs: description: 'A postfix to differentiate between caches' default: 'cache' target_directory: - description: 'Directory to copy LFS data to' + description: 'Existing directory to copy LFS data to' default: 'source' +outputs: + lfs_sha: ${{ steps.lfs_sha_recover.outputs.lfs_sha }} runs: using: "composite" From 8a02b1bcea6bb2f76cd721b8b1094db034f54a00 Mon Sep 17 00:00:00 2001 From: Mathieu Westphal Date: Wed, 4 Sep 2024 09:06:37 +0200 Subject: [PATCH 04/14] fixup output --- action.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/action.yml b/action.yml index 18b7750..edb6524 100644 --- a/action.yml +++ b/action.yml @@ -17,7 +17,9 @@ inputs: description: 'Existing directory to copy LFS data to' default: 'source' outputs: - lfs_sha: ${{ steps.lfs_sha_recover.outputs.lfs_sha }} + lfs_sha: + description: 'The lfs_sha generated by a producer action' + value: ${{ steps.lfs_sha_recover.outputs.lfs_sha }} runs: using: "composite" From acb5e7b2edcf22a9105676717e1e508ba678eb39 Mon Sep 17 00:00:00 2001 From: Mathieu Westphal Date: Wed, 4 Sep 2024 20:11:53 +0200 Subject: [PATCH 05/14] fixup logic --- action.yml | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/action.yml b/action.yml index edb6524..74bf44e 100644 --- a/action.yml +++ b/action.yml @@ -44,19 +44,32 @@ runs: fetch-depth: 0 lfs: false - - name: Set LFS output var without inputs + - name: Set LFS sha env var without inputs if: inputs.lfs_sha == '' working-directory: ${{github.workspace}}/lfs-data-cache/lfs_source id: lfs_sha_recover shell: bash - run: echo "lfs_sha=$(git log -n 1 --pretty=format:%H -- `git-lfs ls-files -n`)" >> $GITHUB_OUTPUT + run: echo "lfs_data_cache_sha=$(git log -n 1 --pretty=format:%H -- `git-lfs ls-files -n`)" >> $GITHUB_ENV + + - name: Set LFS sha env var with inputs + if: inputs.lfs_sha != '' + working-directory: ${{github.workspace}}/lfs-data-cache/lfs_source + id: lfs_sha_recover + shell: bash + run: echo "lfs_data_cache_sha=${{inputs.lfs_sha}}" >> $GITHUB_ENV - name: Cache LFS data id: cache-lfs uses: actions/cache@v4 with: path: 'lfs-data-cache/lfs_data' - key: lfs-data-${{inputs.lfs_sha == '' && ${{steps.lfs_sha_recover.outputs.lfs_sha}} || ${{inputs.lfs_sha}} }}-${{inputs.cache_postfix}} + key: lfs-data-${{env.lfs_data_cache_sha}}-${{inputs.cache_postfix}} + + - name: Set LFS output sha + working-directory: ${{github.workspace}}/lfs-data-cache/lfs_source + id: lfs_sha_recover + shell: bash + run: echo "lfs_sha=${{env.lfs_data_cache_sha}}" >> $GITHUB_OUTPUT - name: Checkout LFS data for artifact producer if: | @@ -65,6 +78,7 @@ runs: uses: actions/checkout@v4 with: repository: ${{ inputs.repository }} + ref: ${{env.lfs_data_cache_sha}} path: 'lfs-data-cache/lfs_data' fetch-depth: 0 lfs: true @@ -99,6 +113,7 @@ runs: uses: actions/checkout@v4 with: repository: ${{ inputs.repository }} + ref: ${{env.lfs_data_cache_sha}} path: 'lfs-data-cache/lfs_data' fetch-depth: 0 lfs: true From 810d5955cedce8c3f8ab6988502034e54295419b Mon Sep 17 00:00:00 2001 From: Mathieu Westphal Date: Wed, 4 Sep 2024 20:13:25 +0200 Subject: [PATCH 06/14] fixup --- action.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/action.yml b/action.yml index 74bf44e..2493ac3 100644 --- a/action.yml +++ b/action.yml @@ -47,14 +47,12 @@ runs: - name: Set LFS sha env var without inputs if: inputs.lfs_sha == '' working-directory: ${{github.workspace}}/lfs-data-cache/lfs_source - id: lfs_sha_recover shell: bash run: echo "lfs_data_cache_sha=$(git log -n 1 --pretty=format:%H -- `git-lfs ls-files -n`)" >> $GITHUB_ENV - name: Set LFS sha env var with inputs if: inputs.lfs_sha != '' working-directory: ${{github.workspace}}/lfs-data-cache/lfs_source - id: lfs_sha_recover shell: bash run: echo "lfs_data_cache_sha=${{inputs.lfs_sha}}" >> $GITHUB_ENV From 0e17b0af46b227dd3ca0f7a8862a023a74632c1c Mon Sep 17 00:00:00 2001 From: Mathieu Westphal Date: Wed, 4 Sep 2024 20:38:11 +0200 Subject: [PATCH 07/14] Fixup --- .github/workflows/ci.yml | 4 ++-- README.md | 50 ++++++++++++++++------------------------ action.yml | 28 +++++++++++----------- 3 files changed, 36 insertions(+), 46 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9e78f16..b0f0f5c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,6 +74,6 @@ jobs: cache_postfix: 'ci-cache' target_directory: 'output_dir' - - name: Checkout output directory + - name: Check output has expected size shell: bash - run: ls output_dir/ + run: [ `cat output_dir/testing/data/f3d.vtp | wc -c` == 4104 ] diff --git a/README.md b/README.md index 6ef910d..64b215d 100644 --- a/README.md +++ b/README.md @@ -3,18 +3,22 @@ A producer/consumer github action to cache and recover LFS data. Its main use is to avoid cloning LFS data in CI to avoid -haing to pay for LFS bandwidth because of CI needs. -It needs cmake to be available on the host. +having to pay for LFS bandwidth because of CI needs. -The action can be used as a consumer or a producer, and needs -a SHA to recover LFS data with. +It expects cmake to be available on the host. + +The action can be used as a consumer or a producer, and must +provide the repository containing the LFS data to recover from. + +It is possible to provide a specific SHA to produce. +If not provided, the last commit modyfing the LFS file will be produced. Is has the following inputs: - - `type`: should be `producer` or `consumer`, default producer - - `repository`: the git repository to recover LFS data from, required - - `lfs_sha`: The git sha to recover LFS data from, required - - `cache_index`: An index used in the cache name, default is 0 + - `type`: should be `producer` or `consumer`, default to `producer` + - `repository`: the git repository to produce LFS data from, default: ${{ github.repository }} + - `lfs_sha`: The git sha to recover LFS data from, optional + - `cache_postfix`: An postfix added to the cache name, to support multiple caches, default to `cache` - `target_directory`: A target directory to copy LFS data to ## Logic @@ -34,8 +38,9 @@ Finally, Producer/Consumer will copy the LFS data only using cmake to the `targe ## Usage -In an first job, recover the LFS sha to recover and use the `producer` action, output the LFS sha -In a second job depending on the first, recover the LFS sha from first job and use the `consumer` action. +In an first job, use the `producer` action, which output the LFS sha that will be produced +In a second job, usually a matrix joib, depending on the first, +recover the LFS sha from first job and use the `consumer` action. ``` jobs: @@ -50,7 +55,7 @@ jobs: lfs_sha: ${{ steps.lfs_sha_recover.outputs.lfs_sha }} steps: - # Checkout WITHOUT LFS as the data itself is not needed at this point + # Checkout your repository WITHOUT LFS - name: Checkout uses: actions/checkout@v3 with: @@ -58,27 +63,14 @@ jobs: fetch-depth: 0 lfs: false - # Recover the last time LFS data was changed on the repository - # TODO: Update the list of directory you want to watch for changes - - name: Set LFS env var - working-directory: ${{github.workspace}}/source - id: lfs_sha_recover - shell: bash - run: echo "lfs_sha=$(git log -n 1 --pretty=format:%H -- path/to/lfs/data path/to/lfs/data/again)" >> $GITHUB_OUTPUT - # Use producer action to recover the LFS data and upload it as cache/artifacts - name: Cache LFS Data - uses: f3d-app/lfs-data-cache-action:latest - with: - workflow_label: 'producer' - repository: 'your/repo' - lfs_sha: ${{ steps.lfs_sha_recover.outputs.lfs_sha }} - target_directory: 'source' + uses: f3d-app/lfs-data-cache-action:v1 recover_lfs: needs: cache_lfs - # Checkout WITHOUT LFS as the data itself is not needed at this point + # Checkout your repository WITHOUT LFS - name: Checkout uses: actions/checkout@v3 with: @@ -87,10 +79,8 @@ jobs: lfs: false - name: Recover LFS Data - uses: f3d-app/lfs-data-cache-action:latest + uses: f3d-app/lfs-data-cache-action:v1 with: workflow_label: 'consumer' - repository: 'your/repo' - lfs_sha: ${{ inputs.lfs_sha}} - target_directory: 'source' + lfs_sha: ${{ needs.cache_lfs.outputs.lfs_sha}} ``` diff --git a/action.yml b/action.yml index 2493ac3..0b1c4a3 100644 --- a/action.yml +++ b/action.yml @@ -6,7 +6,8 @@ inputs: default: 'producer' repository: description: 'Repository to recover LFS data from' - required: true + required: false + default: ${{ github.repository }} lfs_sha: description: 'LFS sha to recover. If not set, target directory will be used to recover it.' required: false @@ -33,26 +34,25 @@ runs: - name: Create a working directory working-directory: ${{github.workspace}} shell: bash - run: mkdir lfs-data-cache + run: mkdir lfs_data_cache - name: Checkout repository without LFS data if: inputs.lfs_sha == '' uses: actions/checkout@v4 with: repository: ${{ inputs.repository }} - path: 'lfs-data-cache/lfs_source' + path: 'lfs_data_cache/lfs_source' fetch-depth: 0 lfs: false - - name: Set LFS sha env var without inputs + - name: Set LFS sha env var from repository if: inputs.lfs_sha == '' - working-directory: ${{github.workspace}}/lfs-data-cache/lfs_source + working-directory: ${{github.workspace}}/lfs_data_cache/lfs_source shell: bash run: echo "lfs_data_cache_sha=$(git log -n 1 --pretty=format:%H -- `git-lfs ls-files -n`)" >> $GITHUB_ENV - - name: Set LFS sha env var with inputs + - name: Set LFS sha env var from inputs if: inputs.lfs_sha != '' - working-directory: ${{github.workspace}}/lfs-data-cache/lfs_source shell: bash run: echo "lfs_data_cache_sha=${{inputs.lfs_sha}}" >> $GITHUB_ENV @@ -60,11 +60,11 @@ runs: id: cache-lfs uses: actions/cache@v4 with: - path: 'lfs-data-cache/lfs_data' + path: 'lfs_data_cache/lfs_data' key: lfs-data-${{env.lfs_data_cache_sha}}-${{inputs.cache_postfix}} - name: Set LFS output sha - working-directory: ${{github.workspace}}/lfs-data-cache/lfs_source + working-directory: ${{github.workspace}}/lfs_data_cache/lfs_source id: lfs_sha_recover shell: bash run: echo "lfs_sha=${{env.lfs_data_cache_sha}}" >> $GITHUB_OUTPUT @@ -77,7 +77,7 @@ runs: with: repository: ${{ inputs.repository }} ref: ${{env.lfs_data_cache_sha}} - path: 'lfs-data-cache/lfs_data' + path: 'lfs_data_cache/lfs_data' fetch-depth: 0 lfs: true @@ -88,7 +88,7 @@ runs: uses: actions/upload-artifact@v4 with: name: lfs-data-${{inputs.cache_postfix}} - path: 'lfs-data-cache/lfs_data' + path: 'lfs_data_cache/lfs_data' overwrite: true include-hidden-files: true @@ -101,7 +101,7 @@ runs: continue-on-error: true with: name: lfs-data-${{inputs.cache_postfix}} - path: 'lfs-data-cache/lfs_data' + path: 'lfs_data_cache/lfs_data' - name: Checkout LFS data (last resort) if: | @@ -112,11 +112,11 @@ runs: with: repository: ${{ inputs.repository }} ref: ${{env.lfs_data_cache_sha}} - path: 'lfs-data-cache/lfs_data' + path: 'lfs_data_cache/lfs_data' fetch-depth: 0 lfs: true - name: Setup LFS data working-directory: ${{github.workspace}} shell: bash - run: cmake -P $GITHUB_ACTION_PATH/copy_lfs.cmake 'lfs-data-cache/lfs_data' ${{ inputs.target_directory }} + run: cmake -P $GITHUB_ACTION_PATH/copy_lfs.cmake 'lfs_data_cache/lfs_data' ${{ inputs.target_directory }} From 1a741a693df971fb2ede05260c37db071529c43e Mon Sep 17 00:00:00 2001 From: Mathieu Westphal Date: Wed, 4 Sep 2024 21:29:26 +0200 Subject: [PATCH 08/14] check --- .github/workflows/ci.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b0f0f5c..19ba113 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,4 +76,11 @@ jobs: - name: Check output has expected size shell: bash - run: [ `cat output_dir/testing/data/f3d.vtp | wc -c` == 4104 ] + run: | + if [[ `cat output_dir/testing/data/f3d.vtp | wc -c` == 4104 ]]; then + echo "File is of expected size" + exit 0 + else + echo "File is not of expected size" + exit 1 + fi From 2e9935ee24d474dcff45219ae01dbf154d504718 Mon Sep 17 00:00:00 2001 From: Mathieu Westphal Date: Wed, 4 Sep 2024 21:40:04 +0200 Subject: [PATCH 09/14] fixup again --- .github/workflows/check_size.cmake | 5 +++++ .github/workflows/ci.yml | 9 +-------- 2 files changed, 6 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/check_size.cmake diff --git a/.github/workflows/check_size.cmake b/.github/workflows/check_size.cmake new file mode 100644 index 0000000..b20cf29 --- /dev/null +++ b/.github/workflows/check_size.cmake @@ -0,0 +1,5 @@ +# Check that a LFS data file is big enough to be an actual file +file(SIZE "${CMAKE_ARGV3}" lfs_file_size) +if (lfs_file_size LESS_EQUAL 500) + message(FATAL_ERROR "File is is small than 500 bytes, lfs data not correctly recovered") +endif() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 19ba113..cc2f773 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,11 +76,4 @@ jobs: - name: Check output has expected size shell: bash - run: | - if [[ `cat output_dir/testing/data/f3d.vtp | wc -c` == 4104 ]]; then - echo "File is of expected size" - exit 0 - else - echo "File is not of expected size" - exit 1 - fi + run: cmake -P ./lfs-data-cache_action/.github/workflows/check_size.cmake output_dir/testing/data/f3d.vtp From 099b93847c0b3a701afb21cc291269125a9802ae Mon Sep 17 00:00:00 2001 From: Mathieu Westphal Date: Wed, 4 Sep 2024 21:44:39 +0200 Subject: [PATCH 10/14] typos --- .github/workflows/ci.yml | 4 ---- README.md | 6 +++++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cc2f773..4b76e20 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,10 +12,6 @@ concurrency: cancel-in-progress: true jobs: - -#---------------------------------------------------------------------------- -# Cache LFS: Checkout LFS data and update the cache to limit LFS bandwidth -#---------------------------------------------------------------------------- cache_lfs: runs-on: ubuntu-latest name: Update LFS data cache diff --git a/README.md b/README.md index 64b215d..a1bc2fc 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ Finally, Producer/Consumer will copy the LFS data only using cmake to the `targe ## Usage In an first job, use the `producer` action, which output the LFS sha that will be produced -In a second job, usually a matrix joib, depending on the first, +In a second job, usually a matrix job, depending on the first, recover the LFS sha from first job and use the `consumer` action. ``` @@ -67,6 +67,10 @@ jobs: - name: Cache LFS Data uses: f3d-app/lfs-data-cache-action:v1 +#---------------------------------------------------------------------------- +# Actual CI: Recover LFS data first +#---------------------------------------------------------------------------- + recover_lfs: needs: cache_lfs From 5e818810aa9de602521b44fba6b9b1d58bb606a8 Mon Sep 17 00:00:00 2001 From: Mathieu Westphal Date: Wed, 4 Sep 2024 22:33:15 +0200 Subject: [PATCH 11/14] Update .github/workflows/check_size.cmake Co-authored-by: Michael MIGLIORE --- .github/workflows/check_size.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check_size.cmake b/.github/workflows/check_size.cmake index b20cf29..ebe97d6 100644 --- a/.github/workflows/check_size.cmake +++ b/.github/workflows/check_size.cmake @@ -1,5 +1,5 @@ # Check that a LFS data file is big enough to be an actual file file(SIZE "${CMAKE_ARGV3}" lfs_file_size) if (lfs_file_size LESS_EQUAL 500) - message(FATAL_ERROR "File is is small than 500 bytes, lfs data not correctly recovered") + message(FATAL_ERROR "File is smaller than 500 bytes, lfs data not correctly recovered") endif() From 4c3853a0d00461375c0d20d28af60be9e5b580c7 Mon Sep 17 00:00:00 2001 From: Mathieu Westphal Date: Thu, 5 Sep 2024 07:49:54 +0200 Subject: [PATCH 12/14] fixup working dir --- action.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/action.yml b/action.yml index 0b1c4a3..5c736dc 100644 --- a/action.yml +++ b/action.yml @@ -46,10 +46,12 @@ runs: lfs: false - name: Set LFS sha env var from repository + working-directory: ${{github.workspace}} if: inputs.lfs_sha == '' - working-directory: ${{github.workspace}}/lfs_data_cache/lfs_source shell: bash - run: echo "lfs_data_cache_sha=$(git log -n 1 --pretty=format:%H -- `git-lfs ls-files -n`)" >> $GITHUB_ENV + run: | + cd lfs_data_cache/lfs_source + echo "lfs_data_cache_sha=$(git log -n 1 --pretty=format:%H -- `git-lfs ls-files -n`)" >> $GITHUB_ENV - name: Set LFS sha env var from inputs if: inputs.lfs_sha != '' @@ -64,7 +66,6 @@ runs: key: lfs-data-${{env.lfs_data_cache_sha}}-${{inputs.cache_postfix}} - name: Set LFS output sha - working-directory: ${{github.workspace}}/lfs_data_cache/lfs_source id: lfs_sha_recover shell: bash run: echo "lfs_sha=${{env.lfs_data_cache_sha}}" >> $GITHUB_OUTPUT From eda1ccc9040b6cd14340a10399f5180f01d8f77f Mon Sep 17 00:00:00 2001 From: Mathieu Westphal Date: Thu, 5 Sep 2024 07:52:11 +0200 Subject: [PATCH 13/14] fixup again --- action.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/action.yml b/action.yml index 5c736dc..ed62a75 100644 --- a/action.yml +++ b/action.yml @@ -46,12 +46,10 @@ runs: lfs: false - name: Set LFS sha env var from repository - working-directory: ${{github.workspace}} + working-directory: ${{github.workspace}}/lfs_data_cache/lfs_source if: inputs.lfs_sha == '' shell: bash - run: | - cd lfs_data_cache/lfs_source - echo "lfs_data_cache_sha=$(git log -n 1 --pretty=format:%H -- `git-lfs ls-files -n`)" >> $GITHUB_ENV + run: echo "lfs_data_cache_sha=$(git log -n 1 --pretty=format:%H -- `git-lfs ls-files -n`)" >> $GITHUB_ENV - name: Set LFS sha env var from inputs if: inputs.lfs_sha != '' From 2515e608d60a7541863bae7f14b1e49f963a9eb5 Mon Sep 17 00:00:00 2001 From: Mathieu Westphal Date: Thu, 5 Sep 2024 07:56:09 +0200 Subject: [PATCH 14/14] fixup readme --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a1bc2fc..57e00f0 100644 --- a/README.md +++ b/README.md @@ -60,11 +60,12 @@ jobs: uses: actions/checkout@v3 with: path: 'source' - fetch-depth: 0 + fetch-depth: 1 lfs: false # Use producer action to recover the LFS data and upload it as cache/artifacts - name: Cache LFS Data + id: lfs_sha_recover uses: f3d-app/lfs-data-cache-action:v1 #----------------------------------------------------------------------------