diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 2112216..98c995a 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -30,17 +30,6 @@ jobs:
         uses: docker/metadata-action@v5
         with:
           images: ${{ env.IMAGE_NAME }}
-      - name: Set up Bazel cache
-        uses: actions/cache@v3
-        with:
-          path: bazel-cache
-          key: bazel-${{ matrix.platform }}-${{ hashFiles('Dockerfile') }}
-          restore-keys: bazel-${{ matrix.platform }}
-      - name: Inject Bazel cache into Docker
-        uses: reproducible-containers/buildkit-cache-dance@v2.1.3
-        with:
-          cache-source: bazel-cache
-          cache-target: /root/.cache/bazel
       - name: Build and push
         id: build
         uses: docker/build-push-action@v5
diff --git a/Dockerfile b/Dockerfile
index 4312e50..6665127 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,62 +1,11 @@
 # syntax=docker/dockerfile:1.4
 FROM debian:bookworm AS builder
 ARG TARGETARCH
-ARG LLVM_VERSION=17
-
-RUN <<EOT
-  rm -f /etc/apt/apt.conf.d/docker-clean
-  echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
-EOT
-
-# Install LLVM
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
-  --mount=type=cache,target=/var/lib/apt,sharing=locked <<EOT
-  apt-get update
-  apt-get install -y --no-install-recommends curl gnupg git patch ca-certificates python3 python3-distutils tcl
-  echo "
-deb http://apt.llvm.org/bookworm/ llvm-toolchain-bookworm-${LLVM_VERSION} main
-deb-src http://apt.llvm.org/bookworm/ llvm-toolchain-bookworm-${LLVM_VERSION} main
-" > /etc/apt/sources.list.d/llvm.list
-  curl https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
-  apt-get update
-  apt-get install -y --no-install-recommends \
-    llvm-${LLVM_VERSION} \
-    lld-${LLVM_VERSION} \
-    clang-${LLVM_VERSION} \
-    libc++-${LLVM_VERSION}-dev \
-    libc++abi-${LLVM_VERSION}-dev \
-    libunwind-${LLVM_VERSION}-dev
-EOT
-
-# Install bazelisk
-RUN <<EOT
-  curl -L https://github.com/bazelbuild/bazelisk/releases/download/v1.17.0/bazelisk-linux-${TARGETARCH} -o /usr/local/bin/bazelisk
-  chmod +x /usr/local/bin/bazelisk
-EOT
-
-# Download workerd source code
+RUN apt-get update && apt-get install -y curl
 WORKDIR /workspace
-RUN curl -L https://github.com/cloudflare/workerd/archive/refs/tags/v1.20240605.0.tar.gz | tar -zx --strip-component=1 -C ./
-
-# Patch workerd
-COPY workerd.patch ./
-RUN patch -p1 < workerd.patch
-
-# Build workerd
-RUN <<EOT
-  echo "
-build:linux --action_env=CC=/usr/lib/llvm-${LLVM_VERSION}/bin/clang --action_env=CXX=/usr/lib/llvm-${LLVM_VERSION}/bin/clang++
-build:linux --host_action_env=CC=/usr/lib/llvm-${LLVM_VERSION}/bin/clang --host_action_env=CXX=/usr/lib/llvm-${LLVM_VERSION}/bin/clang++
-" >> .bazelrc
-EOT
-
-RUN --mount=type=cache,target=/root/.cache/bazelisk \
-  --mount=type=cache,target=/root/.cache/bazel <<EOT
-  bazelisk build --config=thin-lto -c opt //src/workerd/server:workerd
-  strip -S bazel-bin/src/workerd/server/workerd
-  cp bazel-bin/src/workerd/server/workerd /usr/local/bin/workerd
-EOT
+COPY download-workerd.sh ./
+RUN ARCH=${TARGETARCH} ./download-workerd.sh
 
 FROM gcr.io/distroless/cc-debian12:nonroot
-COPY --from=builder /usr/local/bin/workerd /usr/local/bin/workerd
+COPY --from=builder /workspace/bin/workerd /usr/local/bin/workerd
 ENTRYPOINT ["workerd"]
diff --git a/download-workerd.sh b/download-workerd.sh
new file mode 100755
index 0000000..13f905a
--- /dev/null
+++ b/download-workerd.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+pkg_version=1.20240605.0
+target_path=./bin/workerd
+
+if [[ $ARCH == arm* ]]; then
+  pkg_arch=arm64
+else
+  pkg_arch=64
+fi
+
+set -x
+mkdir -p "$(dirname $target_path)"
+curl -L "https://github.com/cloudflare/workerd/releases/download/v${pkg_version}/workerd-linux-${pkg_arch}.gz" | gzip -d > "$target_path"
+chmod +x "$target_path"
diff --git a/workerd.patch b/workerd.patch
deleted file mode 100644
index 03fc3f3..0000000
--- a/workerd.patch
+++ /dev/null
@@ -1,13 +0,0 @@
-diff --git a/src/workerd/jsg/setup.c++ b/src/workerd/jsg/setup.c++
-index 7724606..c3eeea2 100644
---- a/src/workerd/jsg/setup.c++
-+++ b/src/workerd/jsg/setup.c++
-@@ -112,7 +112,7 @@ V8System::V8System(kj::Own<v8::Platform> platformParam, kj::ArrayPtr<const kj::S
-   //
-   // (It turns out you can call v8::V8::SetFlagsFromString() as many times as you want to add
-   // more flags.)
--  v8::V8::SetFlagsFromString("--noincremental-marking");
-+  // v8::V8::SetFlagsFromString("--noincremental-marking");
- 
- #ifdef __APPLE__
-   // On macOS arm64, we find that V8 can be collecting pages that contain compiled code when