Add spark img

.github/workflows/oneflow-spark.yml

@@ -0,0 +1,65 @@
+name: Build OneFlow Spark
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - "dev-oneflow-spark"
+
+concurrency:
+  group: oneflow-spark-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  REGION_ID: cn-beijing
+  ACR_REGISTRY: registry.cn-beijing.aliyuncs.com
+  ACR_NAMESPACE: oneflow
+  DOCKER_HUB_NAMESPACE: oneflowinc
+
+jobs:
+  build_img:
+    name: Build ${{ matrix.tag-suffix }}
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        tag-suffix: ["oneflow-spark-cuda11.2"]
+        include:
+          - tag-suffix: "oneflow-spark-cuda11.2"
+            BASE_IMAGE: "oneflowinc/oneflow:nightly-cuda11.2"
+    env:
+      DOCKER_REPO: "${{ matrix.tag-suffix }}"
+      BASE_IMAGE: "${{ matrix.BASE_IMAGE }}"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      # 1.1 Login to ACR
+      - name: Login to ACR with the AccessKey pair
+        uses: aliyun/acr-login@v1
+        with:
+          login-server: https://registry.${{env.REGION_ID}}.aliyuncs.com
+          username: "${{ secrets.ACR_USERNAME }}"
+          password: "${{ secrets.ACR_PASSWORD }}"
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          push: true
+          tags: |
+            ${{ env.DOCKER_HUB_NAMESPACE }}/${{ env.DOCKER_REPO }}:latest
+            ${{ env.DOCKER_HUB_NAMESPACE }}/${{ env.DOCKER_REPO }}:${{ github.sha }}
+            ${{ env.ACR_REGISTRY }}/${{ env.ACR_NAMESPACE }}/${{ env.DOCKER_REPO }}:latest
+            ${{ env.ACR_REGISTRY }}/${{ env.ACR_NAMESPACE }}/${{ env.DOCKER_REPO }}:${{ github.sha }}
+          cache-from: type=registry,ref=${{ env.DOCKER_HUB_NAMESPACE }}/${{ env.DOCKER_REPO }}:latest
+          cache-to: type=inline
+          context: ./oneflow-spark
+          build-args: |
+            BASE_IMAGE

oneflow-spark/Dockerfile

@@ -0,0 +1,16 @@
+FROM oneflowinc/oneflow:nightly-cuda11.2
+
+RUN apt-get update && \
+    apt-get install -y apt-transport-https && \
+    apt-get install -y --no-install-recommends && \
+    apt install -y scala default-jdk git
+
+RUN wget https://dlcdn.apache.org/spark/spark-3.2.1/spark-3.2.1-bin-hadoop3.2.tgz \
+    && tar xvf spark-3.2.1-bin-hadoop3.2.tgz \
+    && mv spark-3.2.1-bin-hadoop3.2 /opt/spark
+
+ENV SPARK_HOME="/opt/spark"
+ENV PATH=$PATH:$SPARK_HOME/bin:$SPARK_HOME/sbin
+
+RUN pip3 install petastorm psutil
+

openvino/Dockerfile

@@ -22,6 +22,7 @@ RUN --mount=type=cache,id=apt-dev,target=/var/cache/apt \
 
 RUN --mount=type=cache,id=apt-dev,target=/var/cache/apt \
     apt-get update && apt-get install -y --no-install-recommends \
+    lld \
     python3-pip python3-dev python3-setuptools python3-wheel ninja-build libopenblas-dev nasm g++ gcc autoconf automake libtool lsb-release software-properties-common \
     gpg-agent && \
     rm -rf /var/lib/apt/lists/*