Merge branch 'master' into stack/snapshotmanager-wireframe

Author: TimothyW553

.github/workflows/connectors_test.yaml

@@ -1,5 +1,5 @@
 name: "Delta Iceberg Latest"
-on: [push, pull_request]
+on: [] # [push, pull_request]
 jobs:
   test:
     name: "DIL: Scala ${{ matrix.scala }}"
@@ -25,7 +25,7 @@ jobs:
         uses: actions/setup-java@v3
         with:
           distribution: "zulu"
-          java-version: "11"
+          java-version: "17"
       - name: Cache Scala, SBT
         uses: actions/cache@v3
         with:

.github/workflows/iceberg_test.yaml

@@ -37,11 +37,12 @@ jobs:
           echo "Runner arch: ${{ runner.arch }}"
       - name: Checkout code
         uses: actions/checkout@v4
+      # Run unit tests with JDK 17. These unit tests depend on Spark, and Spark 4.0+ is JDK 17.
       - name: install java
         uses: actions/setup-java@v4
         with:
           distribution: "zulu"
-          java-version: "11"
+          java-version: "17"
       - name: Cache SBT and dependencies
         id: cache-sbt
         uses: actions/cache@v4
@@ -59,7 +60,7 @@ jobs:
           else
             echo "❌ Cache MISS - will download dependencies"
           fi
-      - name: Run tests
+      - name: Run unit tests
         run: |
           python run-tests.py --group kernel --coverage --shard ${{ matrix.shard }}
 
@@ -68,6 +69,7 @@ jobs:
     runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v3
+      # Run integration tests with JDK 11, as they have no Spark dependency
       - name: install java
         uses: actions/setup-java@v3
         with:

.github/workflows/kernel_test.yaml

@@ -22,7 +22,7 @@ jobs:
         uses: actions/setup-java@v3
         with:
           distribution: "zulu"
-          java-version: "11"
+          java-version: "17"
         if: steps.git-diff.outputs.diff
       - name: Run Unity tests with coverage
         run: |

.github/workflows/kernel_unitycatalog_test.yaml

@@ -24,7 +24,7 @@ jobs:
         uses: actions/setup-java@v3
         with:
           distribution: "zulu"
-          java-version: "11"
+          java-version: "17"
       - name: Cache Scala, SBT
         uses: actions/cache@v3
         with:

.github/workflows/spark_examples_test.yaml

@@ -25,7 +25,7 @@ jobs:
         uses: actions/setup-java@v3
         with:
           distribution: "zulu"
-          java-version: "11"
+          java-version: "17"
       - name: Cache Scala, SBT
         uses: actions/cache@v3
         with:
@@ -53,33 +53,36 @@ jobs:
           export PATH="~/.pyenv/bin:$PATH"
           eval "$(pyenv init -)"
           eval "$(pyenv virtualenv-init -)"
-          pyenv install 3.8.18
-          pyenv global system 3.8.18
-          pipenv --python 3.8 install
+          pyenv install 3.9
+          pyenv global system 3.9
+          pipenv --python 3.9 install
           # Update the pip version to 24.0. By default `pyenv.run` installs the latest pip version
           # available. From version 24.1, `pip` doesn't allow installing python packages
           # with version string containing `-`. In Delta-Spark case, the pypi package generated has
           # `-SNAPSHOT` in version (e.g. `3.3.0-SNAPSHOT`) as the version is picked up from
           # the`version.sbt` file.
           pipenv run pip install pip==24.0 setuptools==69.5.1 wheel==0.43.0
-          # Install PySpark without bundled Scala 2.12 JARs - read more in the future note below
-          pipenv run pip install pyspark==3.5.3 --no-deps
-          pipenv run pip install py4j==0.10.9.7
-          pipenv run pip install flake8==3.5.0 pypandoc==1.3.3
-          pipenv run pip install black==23.9.1
+          pipenv run pip install pyspark==4.0.1
+          pipenv run pip install flake8==3.9.0
+          pipenv run pip install black==23.12.1
           pipenv run pip install importlib_metadata==3.10.0
-          # The mypy versions 0.982 and 1.8.0 have conflicting rules (cannot get style checks to
-          # pass for both versions on the same file) so we upgrade this to match Spark 4.0
           pipenv run pip install mypy==1.8.0
           pipenv run pip install mypy-protobuf==3.3.0
           pipenv run pip install cryptography==37.0.4
           pipenv run pip install twine==4.0.1
           pipenv run pip install wheel==0.33.4
           pipenv run pip install setuptools==41.1.0
           pipenv run pip install pydocstyle==3.0.0
-          pipenv run pip install pandas==1.1.3
-          pipenv run pip install pyarrow==8.0.0
-          pipenv run pip install numpy==1.20.3
+          pipenv run pip install pandas==2.2.0
+          pipenv run pip install pyarrow==11.0.0
+          pipenv run pip install pypandoc==1.3.3
+          pipenv run pip install numpy==1.22.4
+          pipenv run pip install grpcio==1.67.0
+          pipenv run pip install grpcio-status==1.67.0
+          pipenv run pip install googleapis-common-protos==1.65.0
+          pipenv run pip install protobuf==5.29.1
+          pipenv run pip install googleapis-common-protos-stubs==2.2.0
+          pipenv run pip install grpc-stubs==1.24.11
         if: steps.git-diff.outputs.diff
       - name: Run Python tests
         # when changing TEST_PARALLELISM_COUNT make sure to also change it in spark_master_test.yaml

.github/workflows/spark_master_test.yaml

@@ -29,7 +29,7 @@ jobs:
         uses: actions/setup-java@v3
         with:
           distribution: "zulu"
-          java-version: "11"
+          java-version: "17"
       - name: Cache Scala, SBT
         uses: actions/cache@v3
         with:
@@ -57,29 +57,36 @@ jobs:
           export PATH="~/.pyenv/bin:$PATH"
           eval "$(pyenv init -)"
           eval "$(pyenv virtualenv-init -)"
-          pyenv install 3.8.18
-          pyenv global system 3.8.18
-          pipenv --python 3.8 install
+          pyenv install 3.9
+          pyenv global system 3.9
+          pipenv --python 3.9 install
           # Update the pip version to 24.0. By default `pyenv.run` installs the latest pip version
           # available. From version 24.1, `pip` doesn't allow installing python packages
           # with version string containing `-`. In Delta-Spark case, the pypi package generated has
           # `-SNAPSHOT` in version (e.g. `3.3.0-SNAPSHOT`) as the version is picked up from
           # the`version.sbt` file.
           pipenv run pip install pip==24.0 setuptools==69.5.1 wheel==0.43.0
-          pipenv run pip install pyspark==3.5.3
-          pipenv run pip install flake8==3.5.0 pypandoc==1.3.3
-          pipenv run pip install black==23.9.1
+          pipenv run pip install pyspark==4.0.1
+          pipenv run pip install flake8==3.9.0
+          pipenv run pip install black==23.12.1
           pipenv run pip install importlib_metadata==3.10.0
-          pipenv run pip install mypy==0.982
+          pipenv run pip install mypy==1.8.0
           pipenv run pip install mypy-protobuf==3.3.0
           pipenv run pip install cryptography==37.0.4
           pipenv run pip install twine==4.0.1
           pipenv run pip install wheel==0.33.4
           pipenv run pip install setuptools==41.1.0
           pipenv run pip install pydocstyle==3.0.0
-          pipenv run pip install pandas==1.1.3
-          pipenv run pip install pyarrow==8.0.0
-          pipenv run pip install numpy==1.20.3
+          pipenv run pip install pandas==2.2.0
+          pipenv run pip install pyarrow==11.0.0
+          pipenv run pip install pypandoc==1.3.3
+          pipenv run pip install numpy==1.22.4
+          pipenv run pip install grpcio==1.67.0
+          pipenv run pip install grpcio-status==1.67.0
+          pipenv run pip install googleapis-common-protos==1.65.0
+          pipenv run pip install protobuf==5.29.1
+          pipenv run pip install googleapis-common-protos-stubs==2.2.0
+          pipenv run pip install grpc-stubs==1.24.11
         if: steps.git-diff.outputs.diff
       - name: Scala structured logging style check
         run: |

.github/workflows/spark_python_test.yaml

@@ -13,7 +13,7 @@
           uses: actions/setup-java@v3
           with:
             distribution: "zulu"
-            java-version: "11"
+            java-version: "17"
         - uses: actions/checkout@v3
         - name: generate unidoc
           run: build/sbt "++ ${{ matrix.scala }}" unidoc