[KYUUBI #7205] Use Apache closer.lua syntax for kyuubi-download to obtain engine archives

### Why are the changes needed?

Speed up download Spark, Flink, and Hive tarballs, for the latest versions.

https://infra.apache.org/release-download-pages.html#download-scripts

### How was this patch tested?

Observe GHA.

### Was this patch authored or co-authored using generative AI tooling?

No.

Closes #7205 from pan3793/closer-lua.

Closes #7205

10ffa34 [Cheng Pan] Use Apache closer.lua syntax for kyuubi-download to obtain engine archives

Authored-by: Cheng Pan <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>

Author: pan3793

.github/workflows/master.yml

@@ -62,17 +62,17 @@ jobs:
             comment: 'normal'
           - java: 8
             spark: '3.5'
-            spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.3.3 -Dspark.archive.name=spark-3.3.3-bin-hadoop3.tgz -Pzookeeper-3.6'
+            spark-archive: '-Dspark.archive.mirror=https://www.apache.org/dyn/closer.lua/spark/spark-3.3.3 -Dspark.archive.name=spark-3.3.3-bin-hadoop3.tgz -Pzookeeper-3.6'
             exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
             comment: 'verify-on-spark-3.3-binary'
           - java: 8
             spark: '3.5'
-            spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.4.3 -Dspark.archive.name=spark-3.4.3-bin-hadoop3.tgz -Pzookeeper-3.6'
+            spark-archive: '-Dspark.archive.mirror=https://www.apache.org/dyn/closer.lua/spark/spark-3.4.3 -Dspark.archive.name=spark-3.4.3-bin-hadoop3.tgz -Pzookeeper-3.6'
             exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
             comment: 'verify-on-spark-3.4-binary'
           - java: 17
             spark: '3.5'
-            spark-archive: '-Pscala-2.13 -Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-4.0.1 -Dspark.archive.name=spark-4.0.1-bin-hadoop3.tgz'
+            spark-archive: '-Pscala-2.13 -Dspark.archive.mirror=https://www.apache.org/dyn/closer.lua/spark/spark-4.0.1 -Dspark.archive.name=spark-4.0.1-bin-hadoop3.tgz'
             exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
             comment: 'verify-on-spark-4.0-binary'
     env:
@@ -247,15 +247,15 @@ jobs:
         include:
           - java: 8
             flink: '1.20'
-            flink-archive: '-Dflink.archive.mirror=https://archive.apache.org/dist/flink/flink-1.17.2 -Dflink.archive.name=flink-1.17.2-bin-scala_2.12.tgz'
+            flink-archive: '-Dflink.archive.mirror=https://www.apache.org/dyn/closer.lua/flink/flink-1.17.2 -Dflink.archive.name=flink-1.17.2-bin-scala_2.12.tgz'
             comment: 'verify-on-flink-1.17-binary'
           - java: 8
             flink: '1.20'
-            flink-archive: '-Dflink.archive.mirror=https://archive.apache.org/dist/flink/flink-1.18.1 -Dflink.archive.name=flink-1.18.1-bin-scala_2.12.tgz'
+            flink-archive: '-Dflink.archive.mirror=https://www.apache.org/dyn/closer.lua/flink/flink-1.18.1 -Dflink.archive.name=flink-1.18.1-bin-scala_2.12.tgz'
             comment: 'verify-on-flink-1.18-binary'
           - java: 8
             flink: '1.20'
-            flink-archive: '-Dflink.archive.mirror=https://archive.apache.org/dist/flink/flink-1.19.1 -Dflink.archive.name=flink-1.19.1-bin-scala_2.12.tgz'
+            flink-archive: '-Dflink.archive.mirror=https://www.apache.org/dyn/closer.lua/flink/flink-1.19.1 -Dflink.archive.name=flink-1.19.1-bin-scala_2.12.tgz'
             comment: 'verify-on-flink-1.19-binary'
     steps:
       - uses: actions/checkout@v4
@@ -310,10 +310,10 @@ jobs:
         comment: [ "normal" ]
         include:
           - java: 8
-            hive-archive: '-Dhive.archive.mirror=https://archive.apache.org/dist/hive/hive-2.3.10 -Dhive.archive.name=apache-hive-2.3.10-bin.tar.gz'
+            hive-archive: '-Dhive.archive.mirror=https://www.apache.org/dyn/closer.lua/hive/hive-2.3.10 -Dhive.archive.name=apache-hive-2.3.10-bin.tar.gz'
             comment: 'verify-on-hive-2.3-binary'
           - java: 8
-            hive-archive: '-Dhive.archive.mirror=https://github.com/pan3793/cdh-hive/releases/download/cdh6.3.2-release -Dhive.archive.name=apache-hive-2.1.1-cdh6.3.2-bin.tar.gz'
+            hive-archive: '-Dhive.archive.mirror=https://github.com/pan3793/cdh-hive/releases/download/cdh6.3.2-release -Dhive.archive.name=apache-hive-2.1.1-cdh6.3.2-bin.tar.gz -Dhive.archive.query='
             comment: 'verify-on-hive-2.1-cdh6-binary'
     steps:
       - uses: actions/checkout@v4

docs/contributing/code/building.md

@@ -82,7 +82,7 @@ For the Scala version for Spark engines, the server will look up the `SPARK_SCAL
 
 ## Building With Apache dlcdn Site
 
-By default, we use `https://archive.apache.org/dist/` to download the built-in release packages of engines,
+By default, we use [`closer.lua`](https://infra.apache.org/release-download-pages.html#download-scripts) to download the built-in release packages of engines,
 such as Spark or Flink.
 But sometimes, you may find it hard to reach, or the download speed is too slow,
 then you can define the `apache.archive.dist` by `-Pmirror-cdn` to accelerate to download speed.

externals/kyuubi-download/pom.xml

@@ -51,7 +51,8 @@
                         <phase>compile</phase>
                         <configuration>
                             <skip>${spark.archive.download.skip}</skip>
-                            <url>${spark.archive.mirror}/${spark.archive.name}</url>
+                            <url>${spark.archive.mirror}/${spark.archive.name}${spark.archive.query}</url>
+                            <outputFileName>${spark.archive.name}</outputFileName>
                         </configuration>
                     </execution>
                     <execution>
@@ -62,7 +63,8 @@
                         <phase>compile</phase>
                         <configuration>
                             <skip>${flink.archive.download.skip}</skip>
-                            <url>${flink.archive.mirror}/${flink.archive.name}</url>
+                            <url>${flink.archive.mirror}/${flink.archive.name}${flink.archive.query}</url>
+                            <outputFileName>${flink.archive.name}</outputFileName>
                         </configuration>
                     </execution>
                     <execution>
@@ -73,7 +75,8 @@
                         <phase>compile</phase>
                         <configuration>
                             <skip>${hive.archive.download.skip}</skip>
-                            <url>${hive.archive.mirror}/${hive.archive.name}</url>
+                            <url>${hive.archive.mirror}/${hive.archive.name}${hive.archive.query}</url>
+                            <outputFileName>${hive.archive.name}</outputFileName>
                         </configuration>
                     </execution>
                 </executions>

pom.xml

@@ -126,7 +126,7 @@
         <!-- Please don't upgrade the version to 4.10+, it depends on JDK 11 -->
         <antlr4.version>4.9.3</antlr4.version>
         <antlr.st4.version>4.3.4</antlr.st4.version>
-        <apache.archive.dist>https://archive.apache.org/dist</apache.archive.dist>
+        <apache.archive.dist>https://www.apache.org/dyn/closer.lua</apache.archive.dist>
         <atlas.version>2.3.0</atlas.version>
         <byte-buddy.version>1.17.6</byte-buddy.version>
         <bouncycastle.version>1.78</bouncycastle.version>
@@ -144,6 +144,7 @@
         <flink.version>1.20.0</flink.version>
         <flink.archive.name>flink-${flink.version}-bin-scala_2.12.tgz</flink.archive.name>
         <flink.archive.mirror>${apache.archive.dist}/flink/flink-${flink.version}</flink.archive.mirror>
+        <flink.archive.query>?action=download</flink.archive.query>
         <flink.archive.download.skip>false</flink.archive.download.skip>
         <google.jsr305.version>3.0.2</google.jsr305.version>
         <grpc.version>1.65.1</grpc.version>
@@ -155,6 +156,7 @@
         <hive.version>3.1.3</hive.version>
         <hive.archive.name>apache-hive-${hive.version}-bin.tar.gz</hive.archive.name>
         <hive.archive.mirror>${apache.archive.dist}/hive/hive-${hive.version}</hive.archive.mirror>
+        <hive.archive.query>?action=download</hive.archive.query>
         <hive.archive.download.skip>false</hive.archive.download.skip>
         <httpclient.version>4.5.14</httpclient.version>
         <httpcore.version>4.4.16</httpcore.version>
@@ -208,6 +210,7 @@
         <spark.archive.scala.suffix></spark.archive.scala.suffix>
         <spark.archive.name>spark-${spark.version}-bin-hadoop3${spark.archive.scala.suffix}.tgz</spark.archive.name>
         <spark.archive.mirror>${apache.archive.dist}/spark/spark-${spark.version}</spark.archive.mirror>
+        <spark.archive.query>?action=download</spark.archive.query>
         <spark.archive.download.skip>false</spark.archive.download.skip>
         <sqlite.version>3.46.1.3</sqlite.version>
         <supercsv.version>2.2.0</supercsv.version>
@@ -1955,6 +1958,9 @@
             <properties>
                 <!-- the apache cdn mirror works only for latest apache releases -->
                 <apache.archive.dist>https://dlcdn.apache.org</apache.archive.dist>
+                <spark.archive.query></spark.archive.query>
+                <flink.archive.query></flink.archive.query>
+                <hive.archive.query></hive.archive.query>
                 <nodeDownloadRoot>https://npmmirror.com/mirrors/node/</nodeDownloadRoot>
                 <pnpmDownloadRoot>https://registry.npmmirror.com/pnpm/-/</pnpmDownloadRoot>
             </properties>