fix: make oci_tarball work with genrule (#621)

.github/workflows/ci.yaml

@@ -129,8 +129,11 @@ jobs:
 
       - name: Configure Remote Docker Host
         if: ${{ matrix.os == 'macos-13' }}
-        run: echo "DOCKER_HOST=$(grep 'tc.host' ~/.testcontainers.properties | cut -d '=' -f2 | xargs)" >> $GITHUB_ENV
-
+        run: |
+          echo "DOCKER_HOST=$(grep 'tc.host' ~/.testcontainers.properties | cut -d '=' -f2 | xargs)" >> $GITHUB_ENV
+          curl -fsSL https://download.docker.com/mac/static/stable/x86_64/docker-23.0.0.tgz | tar -xOvf - docker/docker > /usr/local/bin/docker
+          chmod +x /usr/local/bin/docker
+
       - name: bazel test //...
         working-directory: ${{ matrix.folder }}
         env:

examples/assert.bzl

@@ -104,6 +104,7 @@ def assert_oci_config(
         out = name,
     )
 
+# buildifier: disable=function-docstring-args
 def assert_oci_image_command(
         name,
         image,
@@ -121,24 +122,14 @@ def assert_oci_image_command(
         tags = tags + ["manual"],
     )
 
-    native.filegroup(
-        name = name + "_tarball_archive",
-        srcs = [name + "_tarball"],
-        output_group = "tarball",
-        tags = tags + ["manual"],
-    )
-
     docker_args = " ".join(['"' + arg + '"' for arg in ([tag] + args)])
 
     native.genrule(
         name = name + "_gen",
-        srcs = [
-            name + "_tarball_archive",
-        ],
         output_to_bindir = True,
         cmd = """
 docker=$(location //examples:docker_cli)
-$$docker load -i $(location :{name}_tarball_archive)
+$(location :{name}_tarball)
 container_id=$$($$docker run -d {docker_args})
 $$docker wait $$container_id > $(location :{name}_exit_code)
 $$docker logs $$container_id > $(location :{name}_output)
@@ -149,7 +140,7 @@ $$docker logs $$container_id > $(location :{name}_output)
             name + "_exit_code",
         ],
         target_compatible_with = TARGET_COMPATIBLE_WITH,
-        tools = ["//examples:docker_cli"],
+        tools = [name + "_tarball", "//examples:docker_cli"],
     )
 
     if output_eq:

examples/assertion/BUILD.bazel

@@ -290,6 +290,28 @@ assert_json_matches(
     filter1 = ".[0].RepoTags",
 )
 
+# Case 10: An oci_image directly fed into oci_tarball
+oci_image(
+    name = "case10",
+    architecture = "arm64",
+    os = "linux",
+)
+
+oci_tarball(
+    name = "case10_tarball",
+    image = ":case10",
+    repo_tags = ["case10:example"],
+)
+
+genrule(
+    name = "case10_run",
+    outs = ["out.txt"],
+    cmd = """
+$(location :case10_tarball) && echo "worked" > $@
+""",
+    tools = [":case10_tarball"],
+)
+
 # build them as test.
 build_test(
     name = "test",
@@ -299,5 +321,6 @@ build_test(
         ":case3",
         ":case4_tarball_tar",
         ":case5_tarball_tar",
+        ":case10_run",
     ],
 )

oci/private/BUILD.bazel

@@ -17,7 +17,10 @@ bzl_library(
         "//docs:__pkg__",
         "//oci:__subpackages__",
     ],
-    deps = [":util"],
+    deps = [
+        ":util",
+        "@aspect_bazel_lib//lib:paths",
+    ],
 )
 
 bzl_library(

oci/private/tarball.bzl

@@ -19,6 +19,7 @@ docker run --rm my-repository:latest
 ```
 """
 
+load("@aspect_bazel_lib//lib:paths.bzl", "BASH_RLOCATION_FUNCTION", "to_rlocation_path")
 load("//oci/private:util.bzl", "util")
 
 doc = """Creates tarball from OCI layouts that can be loaded into docker daemon without needing to publish the image first.
@@ -93,28 +94,29 @@ attrs = {
         allow_single_file = True,
     ),
     "_tarball_sh": attr.label(allow_single_file = True, default = "//oci/private:tarball.sh.tpl"),
+    "_runfiles": attr.label(default = "@bazel_tools//tools/bash/runfiles"),
     "_windows_constraint": attr.label(default = "@platforms//os:windows"),
 }
 
 def _tarball_impl(ctx):
-    jq = ctx.toolchains["@aspect_bazel_lib//lib:jq_toolchain_type"].jqinfo
+    jq = ctx.toolchains["@aspect_bazel_lib//lib:jq_toolchain_type"]
+    bsdtar = ctx.toolchains["@aspect_bazel_lib//lib:tar_toolchain_type"]
 
     image = ctx.file.image
+    repo_tags = ctx.file.repo_tags
+
     mtree_spec = ctx.actions.declare_file("{}/tarball.spec".format(ctx.label.name))
-    bsdtar = ctx.toolchains["@aspect_bazel_lib//lib:tar_toolchain_type"]
     executable = ctx.actions.declare_file("{}/tarball.sh".format(ctx.label.name))
+    manifest_json = ctx.actions.declare_file("{}/manifest.json".format(ctx.label.name))
 
     # Represents either manifest.json or index.json depending on the image format
-    image_json = ctx.actions.declare_file("{}/_tarball.json".format(ctx.label.name))
-    repo_tags = ctx.file.repo_tags
-
     substitutions = {
         "{{format}}": ctx.attr.format,
-        "{{jq_path}}": jq.bin.path,
+        "{{jq_path}}": jq.jqinfo.bin.path,
         "{{tar}}": bsdtar.tarinfo.binary.path,
         "{{image_dir}}": image.path,
         "{{output}}": mtree_spec.path,
-        "{{json_out}}": image_json.path,
+        "{{json_out}}": manifest_json.path,
     }
 
     if ctx.attr.repo_tags:
@@ -131,28 +133,15 @@ def _tarball_impl(ctx):
         direct = [image, repo_tags, executable],
         transitive = [bsdtar.default.files],
     )
-    mtree_outputs = [mtree_spec, image_json]
+    mtree_outputs = [mtree_spec, manifest_json]
     ctx.actions.run(
         executable = util.maybe_wrap_launcher_for_windows(ctx, executable),
         inputs = mtree_inputs,
         outputs = mtree_outputs,
-        tools = [jq.bin],
+        tools = [jq.jqinfo.bin],
         mnemonic = "OCITarballManifest",
     )
 
-    exe = ctx.actions.declare_file(ctx.label.name + ".sh")
-
-    ctx.actions.expand_template(
-        template = ctx.file._run_template,
-        output = exe,
-        substitutions = {
-            "{{TAR}}": bsdtar.tarinfo.binary.short_path,
-            "{{mtree_path}}": mtree_spec.path,
-            "{{loader}}": ctx.file.loader.path if ctx.file.loader else "",
-        },
-        is_executable = True,
-    )
-
     # This action produces a large output and should rarely be used as it puts load on the cache.
     # It will only run if the "tarball" output_group is explicitly requested
     tarball = ctx.actions.declare_file("{}/tarball.tar".format(ctx.label.name))
@@ -169,11 +158,36 @@ def _tarball_impl(ctx):
         mnemonic = "OCITarball",
     )
 
+    # Create an executable runner script that will create the tarball at runtime,
+    # as opposed to at build to avoid uploading large artifacts to remote cache.
+    runnable_loader = ctx.actions.declare_file(ctx.label.name + ".sh")
+
+    runtime_deps = []
+    if ctx.file.loader:
+        runtime_deps.append(ctx.file.loader)
+    runfiles = ctx.runfiles(runtime_deps, transitive_files = tar_inputs)
+    runfiles = runfiles.merge(ctx.attr._runfiles.default_runfiles)
+
+    ctx.actions.expand_template(
+        template = ctx.file._run_template,
+        output = runnable_loader,
+        substitutions = {
+            "{{BASH_RLOCATION_FUNCTION}}": BASH_RLOCATION_FUNCTION,
+            "{{tar}}": to_rlocation_path(ctx, bsdtar.tarinfo.binary),
+            "{{mtree_path}}": to_rlocation_path(ctx, mtree_spec),
+            "{{loader}}": to_rlocation_path(ctx, ctx.file.loader) if ctx.file.loader else "",
+            "{{manifest_root}}": manifest_json.root.path,
+            "{{image_root}}": image.root.path,
+            "{{workspace_name}}": ctx.workspace_name,
+        },
+        is_executable = True,
+    )
+
     return [
         DefaultInfo(
             files = depset([mtree_spec]),
-            runfiles = ctx.runfiles(files = [ctx.file.loader] if ctx.file.loader else [], transitive_files = tar_inputs),
-            executable = exe,
+            runfiles = runfiles,
+            executable = runnable_loader,
         ),
         OutputGroupInfo(tarball = depset([tarball])),
     ]

oci/private/tarball_run.sh.tpl

@@ -1,8 +1,14 @@
 #!/usr/bin/env bash
 set -o pipefail -o errexit -o nounset
 
-if [ -e "{{loader}}" ]; then
-    CONTAINER_CLI="{{loader}}"
+{{BASH_RLOCATION_FUNCTION}}
+
+readonly TAR="$(rlocation "{{tar}}")"
+readonly MTREE="$(rlocation "{{mtree_path}}")"
+readonly LOADER="$(rlocation "{{loader}}")"
+
+if [ -e "$LOADER}" ]; then
+    CONTAINER_CLI="$LOADER"
 elif command -v docker &> /dev/null; then
     CONTAINER_CLI="docker"
 elif command -v podman &> /dev/null; then
@@ -13,31 +19,13 @@ else
     exit 1
 fi
 
-
-# The execroot detection code is copied from https://github.com/aspect-build/rules_js/blob/d4ac7025a83192d011b7dd7447975a538e34c49b/js/private/js_binary.sh.tpl#L169-L217
-if [[ "$PWD" == *"/bazel-out/"* ]]; then
-    bazel_out_segment="/bazel-out/"
-elif [[ "$PWD" == *"/BAZEL-~1/"* ]]; then
-    bazel_out_segment="/BAZEL-~1/"
-elif [[ "$PWD" == *"/bazel-~1/"* ]]; then
-    bazel_out_segment="/bazel-~1/"
-fi
-
-if [[ "${bazel_out_segment:-}" ]]; then
-    # We are in runfiles and we don't yet know the execroot
-    rest="${PWD#*"$bazel_out_segment"}"
-    index=$((${#PWD} - ${#rest} - ${#bazel_out_segment}))
-    if [ ${index} -lt 0 ]; then
-        echo "No 'bazel-out' folder found in path '${PWD}'" >&2
-        exit 1
-    fi
-    EXECROOT="${PWD:0:$index}"
-else
-    # We are in execroot or in some other context all or a manually run oci_tarball.
-    EXECROOT="${PWD}"
-fi
-
+# Strip manifest root and image root from mtree to make it compatible with runfiles layout.
+image_root="{{image_root}}/"
+manifest_root="{{manifest_root}}/"
+mtree_contents="$(cat $MTREE)"
+mtree_contents="${mtree_contents//"$image_root"/}"
+mtree_contents="${mtree_contents//"$manifest_root"/}"
 
 "$CONTAINER_CLI" load --input <(
-    {{TAR}} --cd "$EXECROOT" --create --no-xattr --no-mac-metadata @"{{mtree_path}}"
+    "$TAR" --cd "$RUNFILES_DIR/{{workspace_name}}" --create --no-xattr --no-mac-metadata @- <<< "$mtree_contents"
 )