-
Notifications
You must be signed in to change notification settings - Fork 5
/
Copy pathspark-defaults.conf
21 lines (18 loc) · 1.12 KB
/
spark-defaults.conf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
spark.sql.defaultCatalog hms
spark.sql.extensions org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions
spark.sql.catalog.hms org.apache.iceberg.spark.SparkCatalog
spark.sql.catalog.hms.type hive
spark.sql.catalog.hms.s3.endpoint http://minio:9000
spark.sql.catalog.hms.warehouse s3a://lakehouse/warehouse
spark.sql.catalog.hms.uri thrift://hive-metastore:9083
spark.sql.catalog.hms.io-impl org.apache.iceberg.aws.s3.S3FileIO
spark.sql.catalog.hms.hadoop.fs.s3a.access.key cTI5BM9ecjv6qISgGaHP
spark.sql.catalog.hms.hadoop.fs.s3a.access.secret gJslk7jC1IJqOpDAVoV0fPXFS0WKDcSX9zBGd3f1
spark.eventLog.enabled true
spark.eventLog.dir /home/iceberg/spark-events
spark.history.fs.logDirectory /home/iceberg/spark-events
# For accesing MinIO directly (without using metastore)
# Endpoint requires fixed IP, not working with docker container names
spark.hadoop.fs.s3a.endpoint http://10.10.5.100:9000
spark.hadoop.fs.s3a.access.key cTI5BM9ecjv6qISgGaHP
spark.hadoop.fs.s3a.access.secret gJslk7jC1IJqOpDAVoV0fPXFS0WKDcSX9zBGd3f1