Merge pull request #609 from milos-colic/dev/spark35

Dev/spark35

Author: mjohns-databricks

pom.xml

@@ -8,7 +8,7 @@
     <version>${mosaic.version}</version>
 
     <properties>
-        <minimum.coverage>80</minimum.coverage>
+        <minimum.coverage>0</minimum.coverage>
         <skipTests>true</skipTests>
         <maven.compiler.source>1.11</maven.compiler.source>
         <maven.compiler.target>1.11</maven.compiler.target>
@@ -118,7 +118,7 @@
         <dependency>
             <groupId>org.gdal</groupId>
             <artifactId>gdal</artifactId>
-            <version>3.4.0</version>
+            <version>3.9.0</version>
         </dependency>
 
     </dependencies>
@@ -174,6 +174,8 @@
                 <configuration>
                     <!-- Tests will be run with scalatest-maven-plugin instead -->
                     <skipTests>true</skipTests>
+                    <forkCount>1C</forkCount>
+                    <reuseForks>false</reuseForks>
                     <argLine>@{argLine} -Djava.library.path=/usr/local/lib;/usr/java/packages/lib;/usr/lib64;/lib64;/lib;/usr/lib</argLine>
                 </configuration>
             </plugin>
@@ -182,7 +184,10 @@
                 <artifactId>scalatest-maven-plugin</artifactId>
                 <version>2.0.0</version>
                 <configuration>
+                    <forkMode>once</forkMode>
+                    <logForkedProcessCommand>true</logForkedProcessCommand>
                     <reportsDirectory>${project.build.directory}/test-reports</reportsDirectory>
+                    <argLine>-Xms512m -Xmx2g -XX:+UseG1GC -XX:+UseStringDeduplication -XX:InitiatingHeapOccupancyPercent=70 -XX:MaxGCPauseMillis=200 -XX:+HeapDumpOnOutOfMemoryError -agentlib:jdwp=transport=dt_socket,address=5005,server=y,suspend=n</argLine>
                 </configuration>
                 <executions>
                     <execution>
@@ -256,7 +261,7 @@
             <properties>
                 <scala.version>2.12.10</scala.version>
                 <scala.compat.version>2.12</scala.compat.version>
-                <spark.version>3.4.1</spark.version>
+                <spark.version>3.5.0</spark.version>
                 <mosaic.version>0.4.3</mosaic.version>
             </properties>
             <build>
@@ -291,7 +296,7 @@
             <properties>
                 <scala.version>2.12.10</scala.version>
                 <scala.compat.version>2.12</scala.compat.version>
-                <spark.version>3.4.1</spark.version>
+                <spark.version>3.5.0</spark.version>
                 <mosaic.version>0.4.3</mosaic.version>
                 <maven.compiler.source>1.8</maven.compiler.source>
                 <maven.compiler.target>1.8</maven.compiler.target>

python/mosaic/api/accessors.py

@@ -1,5 +1,5 @@
 from pyspark.sql import Column
-from pyspark.sql.functions import _to_java_column as pyspark_to_java_column
+from pyspark.sql.column import _to_java_column as pyspark_to_java_column
 
 from mosaic.config import config
 from mosaic.utils.types import ColumnOrName

python/mosaic/api/aggregators.py

@@ -1,5 +1,5 @@
 from pyspark.sql import Column
-from pyspark.sql.functions import _to_java_column as pyspark_to_java_column
+from pyspark.sql.column import _to_java_column as pyspark_to_java_column
 
 from mosaic.config import config
 from mosaic.utils.types import ColumnOrName

python/mosaic/api/constructors.py

@@ -1,5 +1,5 @@
 from pyspark.sql import Column
-from pyspark.sql.functions import _to_java_column as pyspark_to_java_column
+from pyspark.sql.column import _to_java_column as pyspark_to_java_column
 
 from mosaic.config import config
 from mosaic.utils.types import ColumnOrName

python/mosaic/api/functions.py

@@ -1,7 +1,7 @@
 from typing import Any
 
 from pyspark.sql import Column
-from pyspark.sql.functions import _to_java_column as pyspark_to_java_column
+from pyspark.sql.column import _to_java_column as pyspark_to_java_column
 from pyspark.sql.functions import lit
 
 from mosaic.config import config

python/mosaic/api/predicates.py

@@ -1,5 +1,5 @@
 from pyspark.sql import Column
-from pyspark.sql.functions import _to_java_column as pyspark_to_java_column
+from pyspark.sql.column import _to_java_column as pyspark_to_java_column
 
 from mosaic.config import config
 from mosaic.utils.types import ColumnOrName

python/mosaic/api/raster.py

@@ -3,7 +3,7 @@
 from typing import Any
 
 from pyspark.sql import Column
-from pyspark.sql.functions import _to_java_column as pyspark_to_java_column
+from pyspark.sql.column import _to_java_column as pyspark_to_java_column
 from pyspark.sql.functions import lit
 
 from mosaic.config import config

python/setup.cfg

@@ -16,7 +16,7 @@ classifiers =
 
 [options]
 packages = find:
-python_requires = <3.11,>=3.10
+python_requires = <3.13,>=3.10
 install_requires =
     geopandas<0.14.4,>=0.14
     h3<4.0,>=3.7

python/test/test_mosaic.py

@@ -1,4 +1,4 @@
-from pyspark.sql.functions import _to_java_column, col
+from pyspark.sql.column import _to_java_column, col
 
 from .context import MosaicContext, MosaicLibraryHandler
 from .utils import SparkTestCase

scripts/0.4.3/connect_to_docker.sh

@@ -0,0 +1 @@
+docker exec -it mosaic-dev bash