From 9ea27be7f41f5946b56fef07591b493da04b867e Mon Sep 17 00:00:00 2001 From: Ahmed Abualsaud Date: Tue, 16 Jun 2026 22:23:01 -0400 Subject: [PATCH 1/2] add iceberg sql dep when using icebergio --- scripts/beam-sql.sh | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/scripts/beam-sql.sh b/scripts/beam-sql.sh index c9ec365b99bc..b304646ff97d 100755 --- a/scripts/beam-sql.sh +++ b/scripts/beam-sql.sh @@ -84,6 +84,17 @@ function setup_maven_wrapper() { MAVEN_CMD="${mvnw_script}" } +function add_beam_dependency() { + local artifact_id="$1" + cat >> "${POM_FILE}" << EOL + + org.apache.beam + ${artifact_id} + \${beam.version} + +EOL +} + function usage() { echo "Usage: $0 [--version ] [--runner ] [--io ] [--list-versions] [--list-ios] [--list-runners] [--debug] [-h|--help]" echo "" @@ -363,7 +374,12 @@ else EOL # Add IO and Runner dependencies for io in "${IO_CONNECTORS[@]}"; do - echo " org.apache.beambeam-sdks-java-io-${io}\${beam.version}" >> "${POM_FILE}" + add_beam_dependency "beam-sdks-java-io-${io}" + case "${io}" in + iceberg) + add_beam_dependency "beam-sdks-java-extensions-sql-iceberg" + ;; + esac done RUNNER_ARTIFACT="" case "${BEAM_RUNNER}" in @@ -372,7 +388,7 @@ EOL *) echo "❌ Error: Unsupported runner '${BEAM_RUNNER}'." >&2; exit 1 ;; esac if [ -n "${RUNNER_ARTIFACT}" ]; then - echo " org.apache.beam${RUNNER_ARTIFACT}\${beam.version}" >> "${POM_FILE}" + add_beam_dependency "${RUNNER_ARTIFACT}" fi # Complete the POM with the build section for the maven-shade-plugin From 5219487ff3e119bedebb1f1dd8edf4c8c029678d Mon Sep 17 00:00:00 2001 From: Ahmed Abualsaud Date: Wed, 17 Jun 2026 16:20:40 -0400 Subject: [PATCH 2/2] add maven snapshot repo --- scripts/beam-sql.sh | 53 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 49 insertions(+), 4 deletions(-) diff --git a/scripts/beam-sql.sh b/scripts/beam-sql.sh index b304646ff97d..9df48f6c18d4 100755 --- a/scripts/beam-sql.sh +++ b/scripts/beam-sql.sh @@ -36,6 +36,8 @@ MAVEN_DISTRIBUTION_URL="https://repo.maven.apache.org/maven2/org/apache/maven/ap # Maven Plugin Configuration MAVEN_SHADE_PLUGIN_VERSION="3.5.1" +ICEBERG_VERSION="${ICEBERG_VERSION:-1.10.0}" +APACHE_SNAPSHOT_REPOSITORY_URL="https://repository.apache.org/content/repositories/snapshots/" mkdir -p "${CACHE_DIR}" # Create a temporary directory for our Maven project. @@ -95,6 +97,27 @@ function add_beam_dependency() { EOL } +function add_dependency() { + local group_id="$1" + local artifact_id="$2" + local version="$3" + cat >> "${POM_FILE}" << EOL + + ${group_id} + ${artifact_id} + ${version} + +EOL +} + +function normalize_beam_version() { + case "${BEAM_VERSION}" in + *-[sS][nN][aA][pP][sS][hH][oO][tT]) + BEAM_VERSION="${BEAM_VERSION%-*}-SNAPSHOT" + ;; + esac +} + function usage() { echo "Usage: $0 [--version ] [--runner ] [--io ] [--list-versions] [--list-ios] [--list-runners] [--debug] [-h|--help]" echo "" @@ -102,6 +125,7 @@ function usage() { echo "" echo "Options:" echo " --version Specify the Apache Beam version (default: ${DEFAULT_BEAM_VERSION})." + echo " SNAPSHOT versions are resolved from Apache's Maven snapshot repository." echo " --runner Specify the Beam runner to use (default: direct)." echo " Supported runners:" echo " direct - DirectRunner (runs locally, good for development)" @@ -219,7 +243,7 @@ function list_runners() { echo "✅ Available runners for Beam ${BEAM_VERSION}:" echo "" - + # Process each runner and provide descriptions while IFS= read -r runner; do case "$runner" in @@ -228,7 +252,7 @@ function list_runners() { echo " Runs locally on your machine. Good for development and testing." ;; "google-cloud-dataflow-java") - echo " dataflow - DataflowRunner" + echo " dataflow - DataflowRunner" echo " Runs on Google Cloud Dataflow for production workloads." ;; flink-*) @@ -284,7 +308,7 @@ function list_runners() { ;; esac done <<< "$runners" - + echo "" echo "💡 Usage: ./beam-sql.sh --runner " echo " Default: direct" @@ -302,7 +326,7 @@ DEBUG_MODE=false while [[ "$#" -gt 0 ]]; do case $1 in - --version) BEAM_VERSION="$2"; shift ;; + --version) BEAM_VERSION="$2"; normalize_beam_version; shift ;; --runner) BEAM_RUNNER=$(echo "$2" | tr '[:upper:]' '[:lower:]'); shift ;; --io) IO_CONNECTORS+=("$2"); shift ;; --list-versions) list_versions; exit 0 ;; @@ -378,6 +402,9 @@ EOL case "${io}" in iceberg) add_beam_dependency "beam-sdks-java-extensions-sql-iceberg" + add_dependency "org.apache.iceberg" "iceberg-aws" "${ICEBERG_VERSION}" + add_dependency "org.apache.iceberg" "iceberg-aws-bundle" "${ICEBERG_VERSION}" + add_dependency "org.apache.iceberg" "iceberg-gcp" "${ICEBERG_VERSION}" ;; esac done @@ -394,6 +421,19 @@ EOL # Complete the POM with the build section for the maven-shade-plugin cat >> "${POM_FILE}" << EOL + + + apache.snapshots + ${APACHE_SNAPSHOT_REPOSITORY_URL} + + false + + + true + always + + + ${BEAM_VERSION} @@ -411,6 +451,11 @@ cat >> "${POM_FILE}" << EOL + + + true + +