chore(dev-deps): support pyspark in nix

Author: cpcloud

flake.nix

@@ -65,6 +65,7 @@
 
         # necessary for quarto and quartodoc
         export PYTHONPATH=''${PWD}''${PYTHONPATH:+:}''${PYTHONPATH}:''${PWD}/docs
+        export PYSPARK_PYTHON="$(which python)"
       '';
 
       preCommitDeps = with pkgs; [

nix/ibis.nix

@@ -7,6 +7,8 @@
 , ibisTestingData
 }:
 let
+  # pyspark could be added here, but it doesn't handle parallel test execution
+  # well and serially it takes on the order of 7-8 minutes to execute serially
   backends = [ "datafusion" "duckdb" "pandas" "polars" "sqlite" ]
     # dask version has a show-stopping bug for Python >=3.11
     ++ lib.optionals (python3.pythonOlder "3.11") [ "dask" ];