forked from ajayky-os/iceberg-benchmark-java
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathspark-sql.sh
More file actions
executable file
·30 lines (29 loc) · 2.04 KB
/
Copy pathspark-sql.sh
File metadata and controls
executable file
·30 lines (29 loc) · 2.04 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
export SPARK_MASTER_URL=spark://10.182.0.93:7077
export ICEBERG_SPARK_JAR="gs://gcs-hyd-iceberg-benchmark-warehouse/jars/iceberg-spark-runtime-4.0_2.13-1.11.0-SNAPSHOT.jar,gs://gcs-hyd-iceberg-benchmark-warehouse/jars/iceberg-gcp-bundle-1.11.0-SNAPSHOT.jar"
export CATALOG_NAME="gcs_prod"
export PARTITIONED_CATALOG_NAME="gcs_prod_partitioned"
export CATALOG_WAREHOUSE="gs://gcs-hyd-iceberg-benchmark-warehouse/warehouse"
export PARTITIONED_CATALOG_WAREHOUSE="gs://gcs-hyd-iceberg-benchmark-warehouse/partitioned_warehouse"
/opt/spark/bin/spark-sql \
--master $SPARK_MASTER_URL \
--jars $ICEBERG_SPARK_JAR \
--conf spark.hadoop.fs.AbstractFileSystem.gs.impl=com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS \
--conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions \
--conf spark.sql.catalog.$CATALOG_NAME=org.apache.iceberg.spark.SparkCatalog \
--conf spark.sql.catalog.$CATALOG_NAME.type=hadoop \
--conf spark.sql.catalog.$CATALOG_NAME.warehouse=$CATALOG_WAREHOUSE \
--conf spark.sql.catalog.$CATALOG_NAME.hadoop.fs.AbstractFileSystem.gs.impl=com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS \
--conf spark.sql.catalog.$CATALOG_NAME.io-impl=org.apache.iceberg.gcp.gcs.GCSFileIO \
--conf spark.sql.catalog.$CATALOG_NAME.gcs.analytics.core.enabled=true \
--conf spark.sql.catalog.$PARTITIONED_CATALOG_NAME=org.apache.iceberg.spark.SparkCatalog \
--conf spark.sql.catalog.$PARTITIONED_CATALOG_NAME.type=hadoop \
--conf spark.sql.catalog.$PARTITIONED_CATALOG_NAME.io-impl=org.apache.iceberg.gcp.gcs.GCSFileIO \
--conf spark.sql.catalog.$PARTITIONED_CATALOG_NAME.warehouse=$PARTITIONED_CATALOG_WAREHOUSE \
--conf spark.sql.catalog.$PARTITIONED_CATALOG_NAME.hadoop.fs.AbstractFileSystem.gs.impl=com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS \
--conf spark.sql.catalog.$PARTITIONED_CATALOG_NAME.gcs.analytics-core.enabled=true \
--conf spark.hadoop.hive.cli.print.header=true \
--conf spark.sql.iceberg.vectorization.enabled=true \
--driver-memory 32g \
--executor-memory 32g \
--executor-cores 5 \
--num-executors 29