diff --git a/scripts/beam-sql.sh b/scripts/beam-sql.sh index c9ec365b99bc..9df48f6c18d4 100755 --- a/scripts/beam-sql.sh +++ b/scripts/beam-sql.sh @@ -36,6 +36,8 @@ MAVEN_DISTRIBUTION_URL="https://repo.maven.apache.org/maven2/org/apache/maven/ap # Maven Plugin Configuration MAVEN_SHADE_PLUGIN_VERSION="3.5.1" +ICEBERG_VERSION="${ICEBERG_VERSION:-1.10.0}" +APACHE_SNAPSHOT_REPOSITORY_URL="https://repository.apache.org/content/repositories/snapshots/" mkdir -p "${CACHE_DIR}" # Create a temporary directory for our Maven project. @@ -84,6 +86,38 @@ function setup_maven_wrapper() { MAVEN_CMD="${mvnw_script}" } +function add_beam_dependency() { + local artifact_id="$1" + cat >> "${POM_FILE}" << EOL + + org.apache.beam + ${artifact_id} + \${beam.version} + +EOL +} + +function add_dependency() { + local group_id="$1" + local artifact_id="$2" + local version="$3" + cat >> "${POM_FILE}" << EOL + + ${group_id} + ${artifact_id} + ${version} + +EOL +} + +function normalize_beam_version() { + case "${BEAM_VERSION}" in + *-[sS][nN][aA][pP][sS][hH][oO][tT]) + BEAM_VERSION="${BEAM_VERSION%-*}-SNAPSHOT" + ;; + esac +} + function usage() { echo "Usage: $0 [--version ] [--runner ] [--io ] [--list-versions] [--list-ios] [--list-runners] [--debug] [-h|--help]" echo "" @@ -91,6 +125,7 @@ function usage() { echo "" echo "Options:" echo " --version Specify the Apache Beam version (default: ${DEFAULT_BEAM_VERSION})." + echo " SNAPSHOT versions are resolved from Apache's Maven snapshot repository." echo " --runner Specify the Beam runner to use (default: direct)." echo " Supported runners:" echo " direct - DirectRunner (runs locally, good for development)" @@ -208,7 +243,7 @@ function list_runners() { echo "✅ Available runners for Beam ${BEAM_VERSION}:" echo "" - + # Process each runner and provide descriptions while IFS= read -r runner; do case "$runner" in @@ -217,7 +252,7 @@ function list_runners() { echo " Runs locally on your machine. Good for development and testing." ;; "google-cloud-dataflow-java") - echo " dataflow - DataflowRunner" + echo " dataflow - DataflowRunner" echo " Runs on Google Cloud Dataflow for production workloads." ;; flink-*) @@ -273,7 +308,7 @@ function list_runners() { ;; esac done <<< "$runners" - + echo "" echo "💡 Usage: ./beam-sql.sh --runner " echo " Default: direct" @@ -291,7 +326,7 @@ DEBUG_MODE=false while [[ "$#" -gt 0 ]]; do case $1 in - --version) BEAM_VERSION="$2"; shift ;; + --version) BEAM_VERSION="$2"; normalize_beam_version; shift ;; --runner) BEAM_RUNNER=$(echo "$2" | tr '[:upper:]' '[:lower:]'); shift ;; --io) IO_CONNECTORS+=("$2"); shift ;; --list-versions) list_versions; exit 0 ;; @@ -363,7 +398,15 @@ else EOL # Add IO and Runner dependencies for io in "${IO_CONNECTORS[@]}"; do - echo " org.apache.beambeam-sdks-java-io-${io}\${beam.version}" >> "${POM_FILE}" + add_beam_dependency "beam-sdks-java-io-${io}" + case "${io}" in + iceberg) + add_beam_dependency "beam-sdks-java-extensions-sql-iceberg" + add_dependency "org.apache.iceberg" "iceberg-aws" "${ICEBERG_VERSION}" + add_dependency "org.apache.iceberg" "iceberg-aws-bundle" "${ICEBERG_VERSION}" + add_dependency "org.apache.iceberg" "iceberg-gcp" "${ICEBERG_VERSION}" + ;; + esac done RUNNER_ARTIFACT="" case "${BEAM_RUNNER}" in @@ -372,12 +415,25 @@ EOL *) echo "❌ Error: Unsupported runner '${BEAM_RUNNER}'." >&2; exit 1 ;; esac if [ -n "${RUNNER_ARTIFACT}" ]; then - echo " org.apache.beam${RUNNER_ARTIFACT}\${beam.version}" >> "${POM_FILE}" + add_beam_dependency "${RUNNER_ARTIFACT}" fi # Complete the POM with the build section for the maven-shade-plugin cat >> "${POM_FILE}" << EOL + + + apache.snapshots + ${APACHE_SNAPSHOT_REPOSITORY_URL} + + false + + + true + always + + + ${BEAM_VERSION} @@ -395,6 +451,11 @@ cat >> "${POM_FILE}" << EOL + + + true + +