Add python dataproc library

Author: david-zlai

api/python/requirements/base.in

@@ -8,4 +8,5 @@ crcmod==1.7
 glom
 boto3
 importlib-resources==6.5.2
-rich
+rich
+google-cloud-dataproc

api/python/requirements/base.txt

@@ -1,4 +1,4 @@
-# SHA1:f6642699c69070a051b23fe523edcec65b717c6f
+# SHA1:5e8a030f43231740326a93b6159be25c736d7e23
 #
 # This file is autogenerated by pip-compile-multi
 # To update, run:
@@ -11,14 +11,16 @@ boltons==25.0.0
     # via
     #   face
     #   glom
-boto3==1.37.6
+boto3==1.37.27
     # via -r requirements/base.in
-botocore==1.37.6
+botocore==1.37.27
     # via
     #   boto3
     #   s3transfer
-cachetools==5.5.0
+cachetools==5.5.2
     # via google-auth
+certifi==2025.1.31
+    # via requests
 charset-normalizer==3.4.1
     # via requests
 click==8.1.8
@@ -28,68 +30,108 @@ crcmod==1.7
 face==24.0.0
     # via glom
 glom==24.11.0
-    # via -r requirements/base.
-google-api-core==2.24.0
+    # via -r requirements/base.in
+google-api-core[grpc]==2.24.2
     # via
+    #   google-cloud-bigquery-storage
     #   google-cloud-core
+    #   google-cloud-dataproc
     #   google-cloud-storage
-google-auth==2.37.0
+google-auth==2.38.0
     # via
     #   google-api-core
+    #   google-cloud-bigquery-storage
     #   google-cloud-core
+    #   google-cloud-dataproc
     #   google-cloud-storage
-google-cloud-core==2.4.1
+google-cloud-bigquery-storage==2.30.0
+    # via -r requirements/base.in
+google-cloud-core==2.4.3
     # via google-cloud-storage
+google-cloud-dataproc==5.18.1
+    # via -r requirements/base.in
 google-cloud-storage==2.19.0
     # via -r requirements/base.in
-google-crc32c==1.6.0
+google-crc32c==1.7.1
     # via
     #   google-cloud-storage
     #   google-resumable-media
 google-resumable-media==2.7.2
     # via google-cloud-storage
-googleapis-common-protos==1.66.0
+googleapis-common-protos[grpc]==1.69.2
+    # via
+    #   google-api-core
+    #   grpc-google-iam-v1
+    #   grpcio-status
+grpc-google-iam-v1==0.14.2
+    # via google-cloud-dataproc
+grpcio==1.71.0
+    # via
+    #   google-api-core
+    #   googleapis-common-protos
+    #   grpc-google-iam-v1
+    #   grpcio-status
+grpcio-status==1.71.0
     # via google-api-core
 idna==3.10
     # via requests
+importlib-resources==6.5.2
+    # via -r requirements/base.in
 jmespath==1.0.1
     # via
     #   boto3
     #   botocore
 markdown-it-py==3.0.0
     # via rich
-proto-plus==1.25.0
-    # via google-api-core
-protobuf==5.29.3
+mdurl==0.1.2
+    # via markdown-it-py
+proto-plus==1.26.1
+    # via
+    #   google-api-core
+    #   google-cloud-bigquery-storage
+    #   google-cloud-dataproc
+protobuf==5.29.4
     # via
     #   google-api-core
+    #   google-cloud-bigquery-storage
+    #   google-cloud-dataproc
     #   googleapis-common-protos
+    #   grpc-google-iam-v1
+    #   grpcio-status
     #   proto-plus
+py4j==0.10.9.7
+    # via pyspark
 pyasn1==0.6.1
     # via
     #   pyasn1-modules
     #   rsa
-pyasn1-modules==0.4.1
+pyasn1-modules==0.4.2
     # via google-auth
 pygments==2.19.1
     # via rich
+pyspark==3.5.4
+    # via -r requirements/base.in
 python-dateutil==2.9.0.post0
     # via botocore
 requests==2.32.3
     # via
     #   google-api-core
     #   google-cloud-storage
-rich==13.9.4
+rich==14.0.0
     # via -r requirements/base.in
 rsa==4.9
     # via google-auth
 s3transfer==0.11.4
     # via boto3
 six==1.17.0
-    # via thrift
+    # via
+    #   python-dateutil
+    #   thrift
+sqlglot==26.12.1
+    # via -r requirements/base.in
 thrift==0.21.0
     # via -r requirements/base.in
 urllib3==2.3.0
     # via
     #   botocore
-    #   requests
+    #   requests

api/python/requirements/dev.txt

@@ -8,29 +8,29 @@
 -r base.txt
 autoflake==2.3.1
     # via -r requirements/dev.in
-black==24.10.0
+black==25.1.0
     # via -r requirements/dev.in
 cfgv==3.4.0
     # via pre-commit
 chardet==5.2.0
     # via tox
 colorama==0.4.6
     # via tox
-coverage[toml]==7.6.10
+coverage[toml]==7.8.0
     # via pytest-cov
 distlib==0.3.9
     # via virtualenv
-filelock==3.16.1
+filelock==3.18.0
     # via
     #   tox
     #   virtualenv
-identify==2.6.5
+identify==2.6.9
     # via pre-commit
 importlib-metadata==8.4.0
     # via -r requirements/dev.in
-iniconfig==2.0.0
+iniconfig==2.1.0
     # via pytest
-isort==5.13.2
+isort==6.0.1
     # via -r requirements/dev.in
 mypy-extensions==1.0.0
     # via black
@@ -44,7 +44,7 @@ packaging==24.2
     #   tox
 pathspec==0.12.1
     # via black
-platformdirs==4.3.6
+platformdirs==4.3.7
     # via
     #   black
     #   tox
@@ -53,21 +53,21 @@ pluggy==1.5.0
     # via
     #   pytest
     #   tox
-pre-commit==4.0.1
+pre-commit==4.2.0
     # via -r requirements/dev.in
-pyflakes==3.2.0
+pyflakes==3.3.2
     # via autoflake
-pyproject-api==1.8.0
+pyproject-api==1.9.0
     # via tox
-pytest==8.3.4
+pytest==8.3.5
     # via pytest-cov
-pytest-cov==6.0.0
+pytest-cov==6.1.0
     # via -r requirements/dev.in
 pyyaml==6.0.2
     # via pre-commit
-tox==4.23.2
+tox==4.25.0
     # via -r requirements/dev.in
-virtualenv==20.28.1
+virtualenv==20.30.0
     # via
     #   pre-commit
     #   tox