[GLUTEN-10926][VL] Refine docker image build (#11126)

This patch refines the docker image build by

split the merge job
speedup the downloading for celeborn/uniffle/hadoop with faster apache mirror

---------

Signed-off-by: Yuan <[email protected]>

Author: zhouyuan

.github/workflows/docker_image.yml

@@ -385,12 +385,11 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        digests: [ vcpkg-centos-8, vcpkg-centos-9, centos-8-jdk8, centos-8-jdk11, centos-8-jdk17, centos-9-jdk8, centos-9-jdk11, centos-9-jdk17 ]
+        digests: [ vcpkg-centos-8, vcpkg-centos-9 ]
     needs:
       - build-vcpkg-centos-8
       - build-vcpkg-centos-9
-      - build-centos-8
-      - build-centos-9
+
     steps:
       - name: Download digests
         uses: actions/download-artifact@v4
@@ -466,3 +465,47 @@ jobs:
       - name: Inspect image
         run: |
           docker buildx imagetools inspect ${{ env.DOCKERHUB_REPO }}:${{ steps.meta.outputs.version }}
+
+  merge-dynamic:
+    if: ${{ startsWith(github.repository, 'apache/') }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        digests: [ centos-8-jdk8, centos-8-jdk11, centos-8-jdk17, centos-9-jdk8, centos-9-jdk11, centos-9-jdk17 ]
+    needs:
+      - build-centos-8
+      - build-centos-9
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: ${{ runner.temp }}/digests
+          pattern: digests-${{ matrix.digests }}-*
+          merge-multiple: true
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.DOCKERHUB_REPO }}
+          tags: ${{ matrix.digests }}
+
+      - name: Create manifest list and push
+        working-directory: ${{ runner.temp }}/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.DOCKERHUB_REPO }}@sha256:%s ' *)
+
+      - name: Inspect image
+        run: |
+          docker buildx imagetools inspect ${{ env.DOCKERHUB_REPO }}:${{ steps.meta.outputs.version }}
+

dev/docker/Dockerfile.centos8-dynamic-build

@@ -40,10 +40,10 @@ RUN set -ex; \
     tar -xvf ${local_binary}; \
     mv apache-maven-${maven_version} /usr/lib/maven; \
     rm -rf ${local_binary}; \
-    wget -nv https://archive.apache.org/dist/celeborn/celeborn-0.5.4/apache-celeborn-0.5.4-bin.tgz -P /opt/; \
-    wget -nv https://archive.apache.org/dist/celeborn/celeborn-0.6.1/apache-celeborn-0.6.1-bin.tgz -P /opt/; \
-    wget -nv https://archive.apache.org/dist/incubator/uniffle/0.9.2/apache-uniffle-0.9.2-incubating-bin.tar.gz -P /opt/; \
-    wget -nv https://archive.apache.org/dist/hadoop/common/hadoop-2.8.5/hadoop-2.8.5.tar.gz -P /opt/; \
+    wget -nv ${mirror_host}/celeborn/celeborn-0.5.4/apache-celeborn-0.5.4-bin.tgz?action=download -P /opt/; \
+    wget -nv ${mirror_host}/celeborn/celeborn-0.6.1/apache-celeborn-0.6.1-bin.tgz?action=download -P /opt/; \
+    wget -nv ${mirror_host}/uniffle/0.9.2/apache-uniffle-0.9.2-incubating-bin.tar.gz?action=download -P /opt/; \
+    wget -nv ${mirror_host}/hadoop/common/hadoop-2.8.5/hadoop-2.8.5.tar.gz -P /opt/; \
     git clone --depth=1 https://github.com/apache/incubator-gluten /opt/gluten; \
     cd /opt/gluten/.github/workflows/util/; \
     ./install-spark-resources.sh 3.2; \

dev/docker/Dockerfile.centos9-dynamic-build

@@ -38,10 +38,10 @@ RUN set -ex; \
     tar -xvf ${local_binary}; \
     mv apache-maven-${maven_version} /usr/lib/maven; \
     rm -rf ${local_binary}; \
-    wget -nv https://archive.apache.org/dist/celeborn/celeborn-0.5.4/apache-celeborn-0.5.4-bin.tgz -P /opt/; \
-    wget -nv https://archive.apache.org/dist/celeborn/celeborn-0.6.1/apache-celeborn-0.6.1-bin.tgz -P /opt/; \
-    wget -nv https://archive.apache.org/dist/incubator/uniffle/0.9.2/apache-uniffle-0.9.2-incubating-bin.tar.gz -P /opt/; \
-    wget -nv https://archive.apache.org/dist/hadoop/common/hadoop-2.8.5/hadoop-2.8.5.tar.gz -P /opt/; \
+    wget -nv ${mirror_host}/celeborn/celeborn-0.5.4/apache-celeborn-0.5.4-bin.tgz?action=download -P /opt/; \
+    wget -nv ${mirror_host}/celeborn/celeborn-0.6.1/apache-celeborn-0.6.1-bin.tgz?action=download -P /opt/; \
+    wget -nv ${mirror_host}/uniffle/0.9.2/apache-uniffle-0.9.2-incubating-bin.tar.gz?action=download -P /opt/; \
+    wget -nv ${mirror_host}/hadoop/common/hadoop-2.8.5/hadoop-2.8.5.tar.gz -P /opt/; \
     git clone --depth=1 https://github.com/apache/incubator-gluten /opt/gluten; \
     cd /opt/gluten/.github/workflows/util/; \
     ./install-spark-resources.sh 3.2; \