Merge branch 'apache:master' into master

Author: jiayuasu

.gitattributes

@@ -0,0 +1,4 @@
+*       text=auto eol=lf
+*.bat   text eol=crlf
+*.cmd   text eol=crlf
+*.png   binary

.github/linters/.markdown-lint.yml

@@ -30,17 +30,11 @@ MD034: false
 # no-emphasis-as-heading - Emphasis used instead of a heading
 MD036: false
 
-# no-space-in-code - Spaces inside code span elements
-MD038: false
-
 # fenced-code-language - Fenced code blocks should have a language specified
 MD040: false
 
 # first-line-heading/first-line-h1 - First line in a file should be a top-level heading
 MD041: false
 
-# no-alt-text - Images should have alternate text (alt text)
-MD045: false
-
 # code-block-style - Code block style
 MD046: false

.github/linters/.yaml-lint.yml

@@ -4,8 +4,6 @@ extends: default
 
 rules:
   colons: disable
-  comments: disable
-  comments-indentation: disable
   document-start: disable
   line-length: disable
   truthy: false

.github/linters/ruff.toml

@@ -10,7 +10,9 @@
 
 - Yes, the URL of the associated JIRA ticket is https://issues.apache.org/jira/browse/SEDONA-XXX. The PR name follows the format `[SEDONA-XXX] my subject`.
 
-- No, this is a documentation update. The PR name follows the format `[DOCS] my subject`.
+- No:
+  - this is a documentation update. The PR name follows the format `[DOCS] my subject`
+  - this is a CI update. The PR name follows the format `[CI] my subject`
 
 
 ## What changes were proposed in this PR?

.github/pull_request_template.md

@@ -24,7 +24,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        os: ['ubuntu-latest', 'macos-12']
+        os: ['ubuntu-latest']
         spark: ['3.5.1', '3.4.3', '3.3.4']
         include:
           - spark: 3.5.1
@@ -55,6 +55,7 @@ jobs:
       - name: Setup docker (missing on macOS)
         if: runner.os == 'macos'
         run: |
+          brew install colima
           brew install docker
           colima start
           DOCKER_CONFIG=${DOCKER_CONFIG:-$HOME/.docker}

.github/workflows/docker-build.yml

@@ -68,18 +68,6 @@ jobs:
             scala: 2.12.15
             jdk: '8'
             skipTests: ''
-          - spark: 3.2.3
-            scala: 2.12.15
-            jdk: '8'
-            skipTests: ''
-          - spark: 3.1.2
-            scala: 2.12.15
-            jdk: '8'
-            skipTests: ''
-          - spark: 3.0.3
-            scala: 2.12.15
-            jdk: '8'
-            skipTests: ''
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-java@v4

.github/workflows/java.yml

@@ -41,57 +41,37 @@ jobs:
           - spark: '3.5.0'
             scala: '2.12.8'
             python: '3.10'
-            hadoop: '3'
             shapely: '1'
           - spark: '3.5.0'
             scala: '2.12.8'
             python: '3.10'
-            hadoop: '3'
           - spark: '3.5.0'
             scala: '2.12.8'
             python: '3.9'
-            hadoop: '3'
           - spark: '3.5.0'
             scala: '2.12.8'
             python: '3.8'
-            hadoop: '3'
           - spark: '3.4.0'
             scala: '2.12.8'
             python: '3.10'
-            hadoop: '3'
           - spark: '3.4.0'
             scala: '2.12.8'
             python: '3.9'
-            hadoop: '3'
           - spark: '3.4.0'
             scala: '2.12.8'
             python: '3.8'
-            hadoop: '3'
           - spark: '3.4.0'
             scala: '2.12.8'
             python: '3.7'
-            hadoop: '3'
           - spark: '3.4.0'
             scala: '2.12.8'
             python: '3.7'
-            hadoop: '3'
             shapely: '1'
           - spark: '3.3.0'
             scala: '2.12.8'
             python: '3.8'
-            hadoop: '3'
-          - spark: '3.2.0'
-            scala: '2.12.8'
-            python: '3.7'
-            hadoop: '2.7'
-          - spark: '3.1.2'
-            scala: '2.12.8'
-            python: '3.7'
-            hadoop: '2.7'
-          - spark: '3.0.3'
-            scala: '2.12.8'
-            python: '3.7'
-            hadoop: '2.7'
+    env:
+      VENV_PATH: /home/runner/.local/share/virtualenvs/python-${{ matrix.python }}
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-java@v4
@@ -113,18 +93,6 @@ jobs:
         run: |
             SPARK_COMPAT_VERSION=${SPARK_VERSION:0:3}
             mvn -q clean install -DskipTests -Dspark=${SPARK_COMPAT_VERSION} -Dscala=${SCALA_VERSION:0:4} -Dgeotools
-      - env:
-          SPARK_VERSION: ${{ matrix.spark }}
-          HADOOP_VERSION: ${{ matrix.hadoop }}
-        run: |
-          wget https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
-          wget https://repo.osgeo.org/repository/release/javax/media/jai_core/${JAI_CORE_VERSION}/jai_core-${JAI_CORE_VERSION}.jar
-          wget https://repo.osgeo.org/repository/release/javax/media/jai_codec/${JAI_CODEC_VERSION}/jai_codec-${JAI_CODEC_VERSION}.jar
-          wget https://repo.osgeo.org/repository/release/javax/media/jai_imageio/${JAI_IMAGEIO_VERSION}/jai_imageio-${JAI_IMAGEIO_VERSION}.jar
-          tar -xzf spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
-          mv -v jai_core-${JAI_CORE_VERSION}.jar spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}/jars/
-          mv -v jai_codec-${JAI_CODEC_VERSION}.jar spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}/jars/
-          mv -v jai_imageio-${JAI_IMAGEIO_VERSION}.jar spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}/jars/
       - run: sudo apt-get -y install python3-pip python-dev-is-python3
       - run: sudo pip3 install -U setuptools
       - run: sudo pip3 install -U wheel
@@ -141,15 +109,43 @@ jobs:
             echo "Patching Pipfile to use Shapely 1.x"
             sed -i 's/^shapely.*$/shapely="<2.0.0"/g' Pipfile
           fi
+          export PIPENV_CUSTOM_VENV_NAME=python-${PYTHON_VERSION}
           pipenv --python ${PYTHON_VERSION}
           pipenv install pyspark==${SPARK_VERSION}
           pipenv install --dev
           pipenv graph
       - env:
-          SPARK_VERSION: ${{ matrix.spark }}
-          HADOOP_VERSION: ${{ matrix.hadoop }}
-        run: find spark-shaded/target -name sedona-*.jar -exec cp {} spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}/jars/ \;
+          PYTHON_VERSION: ${{ matrix.python }}
+        run: |
+          wget --retry-connrefused --waitretry=10 --read-timeout=20 --timeout=15 --tries=5 https://repo.osgeo.org/repository/release/javax/media/jai_core/${JAI_CORE_VERSION}/jai_core-${JAI_CORE_VERSION}.jar
+          wget --retry-connrefused --waitretry=10 --read-timeout=20 --timeout=15 --tries=5 https://repo.osgeo.org/repository/release/javax/media/jai_codec/${JAI_CODEC_VERSION}/jai_codec-${JAI_CODEC_VERSION}.jar
+          wget --retry-connrefused --waitretry=10 --read-timeout=20 --timeout=15 --tries=5 https://repo.osgeo.org/repository/release/javax/media/jai_imageio/${JAI_IMAGEIO_VERSION}/jai_imageio-${JAI_IMAGEIO_VERSION}.jar
+          mv -v jai_core-${JAI_CORE_VERSION}.jar ${VENV_PATH}/lib/python${PYTHON_VERSION}/site-packages/pyspark/jars
+          mv -v jai_codec-${JAI_CODEC_VERSION}.jar ${VENV_PATH}/lib/python${PYTHON_VERSION}/site-packages/pyspark/jars
+          mv -v jai_imageio-${JAI_IMAGEIO_VERSION}.jar ${VENV_PATH}/lib/python${PYTHON_VERSION}/site-packages/pyspark/jars
       - env:
-          SPARK_VERSION: ${{ matrix.spark }}
-          HADOOP_VERSION: ${{ matrix.hadoop }}
-        run: (export SPARK_HOME=$PWD/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION};export PYTHONPATH=$SPARK_HOME/python;cd python;pipenv run pytest tests)
+          PYTHON_VERSION: ${{ matrix.python }}
+        run: find spark-shaded/target -name sedona-*.jar -exec cp {} ${VENV_PATH}/lib/python${PYTHON_VERSION}/site-packages/pyspark/jars/ \;
+      - env:
+          PYTHON_VERSION: ${{ matrix.python }}
+        run: |
+          export SPARK_HOME=${VENV_PATH}/lib/python${PYTHON_VERSION}/site-packages/pyspark
+          cd python
+          source ${VENV_PATH}/bin/activate
+          pytest tests
+      - env:
+          PYTHON_VERSION: ${{ matrix.python }}
+        run: |
+          if [ ! -f "${VENV_PATH}/lib/python${PYTHON_VERSION}/site-packages/pyspark/sbin/start-connect-server.sh" ]
+          then
+            echo "Skipping connect tests for Spark $SPARK_VERSION"
+            exit
+          fi
+
+          export SPARK_HOME=${VENV_PATH}/lib/python${PYTHON_VERSION}/site-packages/pyspark
+          export SPARK_REMOTE=local
+
+          cd python
+          source ${VENV_PATH}/bin/activate
+          pip install "pyspark[connect]==${SPARK_VERSION}"
+          pytest tests/sql/test_dataframe_api.py

.github/workflows/python.yml

@@ -32,7 +32,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        spark: [3.0.3, 3.1.2, 3.2.1, 3.3.0, 3.4.0, 3.5.0]
+        spark: [3.3.0, 3.4.0, 3.5.0]
         hadoop: [3]
         scala: [2.12.15]
         r: [oldrel, release]