Skip to content

[Beam SQL Shell] Add Iceberg deps when using icebergio #38994

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ahmedabu98 merged 2 commits into from
Jun 17, 2026
+67 −6
Merged

[Beam SQL Shell] Add Iceberg deps when using icebergio #38994

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9ea27be
add iceberg sql dep when using icebergio
ahmedabu98 Jun 17, 2026
5219487
add maven snapshot repo
ahmedabu98 Jun 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 67 additions & 6 deletions scripts/beam-sql.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ MAVEN_DISTRIBUTION_URL="https://repo.maven.apache.org/maven2/org/apache/maven/ap

# Maven Plugin Configuration
MAVEN_SHADE_PLUGIN_VERSION="3.5.1"
ICEBERG_VERSION="${ICEBERG_VERSION:-1.10.0}"
APACHE_SNAPSHOT_REPOSITORY_URL="https://repository.apache.org/content/repositories/snapshots/"
mkdir -p "${CACHE_DIR}"

# Create a temporary directory for our Maven project.
Expand Down Expand Up @@ -84,13 +86,46 @@ function setup_maven_wrapper() {
MAVEN_CMD="${mvnw_script}"
}

function add_beam_dependency() {
local artifact_id="$1"
cat >> "${POM_FILE}" << EOL
<dependency>
<groupId>org.apache.beam</groupId>
<artifactId>${artifact_id}</artifactId>
<version>\${beam.version}</version>
</dependency>
EOL
}

function add_dependency() {
local group_id="$1"
local artifact_id="$2"
local version="$3"
cat >> "${POM_FILE}" << EOL
<dependency>
<groupId>${group_id}</groupId>
<artifactId>${artifact_id}</artifactId>
<version>${version}</version>
</dependency>
EOL
}

function normalize_beam_version() {
case "${BEAM_VERSION}" in
*-[sS][nN][aA][pP][sS][hH][oO][tT])
BEAM_VERSION="${BEAM_VERSION%-*}-SNAPSHOT"
;;
esac
}

function usage() {
echo "Usage: $0 [--version <beam_version>] [--runner <runner_name>] [--io <io_connector>] [--list-versions] [--list-ios] [--list-runners] [--debug] [-h|--help]"
echo ""
echo "A self-contained launcher for the Apache Beam SQL Shell."
echo ""
echo "Options:"
echo " --version Specify the Apache Beam version (default: ${DEFAULT_BEAM_VERSION})."
echo " SNAPSHOT versions are resolved from Apache's Maven snapshot repository."
echo " --runner Specify the Beam runner to use (default: direct)."
echo " Supported runners:"
echo " direct - DirectRunner (runs locally, good for development)"
Expand Down Expand Up @@ -208,7 +243,7 @@ function list_runners() {

echo "✅ Available runners for Beam ${BEAM_VERSION}:"
echo ""

# Process each runner and provide descriptions
while IFS= read -r runner; do
case "$runner" in
Expand All @@ -217,7 +252,7 @@ function list_runners() {
echo " Runs locally on your machine. Good for development and testing."
;;
"google-cloud-dataflow-java")
echo " dataflow - DataflowRunner"
echo " dataflow - DataflowRunner"
echo " Runs on Google Cloud Dataflow for production workloads."
;;
flink-*)
Expand Down Expand Up @@ -273,7 +308,7 @@ function list_runners() {
;;
esac
done <<< "$runners"

echo ""
echo "💡 Usage: ./beam-sql.sh --runner <runner_name>"
echo " Default: direct"
Expand All @@ -291,7 +326,7 @@ DEBUG_MODE=false

while [[ "$#" -gt 0 ]]; do
case $1 in
--version) BEAM_VERSION="$2"; shift ;;
--version) BEAM_VERSION="$2"; normalize_beam_version; shift ;;
--runner) BEAM_RUNNER=$(echo "$2" | tr '[:upper:]' '[:lower:]'); shift ;;
--io) IO_CONNECTORS+=("$2"); shift ;;
--list-versions) list_versions; exit 0 ;;
Expand Down Expand Up @@ -363,7 +398,15 @@ else
EOL
# Add IO and Runner dependencies
for io in "${IO_CONNECTORS[@]}"; do
echo " <dependency><groupId>org.apache.beam</groupId><artifactId>beam-sdks-java-io-${io}</artifactId><version>\${beam.version}</version></dependency>" >> "${POM_FILE}"
add_beam_dependency "beam-sdks-java-io-${io}"
case "${io}" in
iceberg)
add_beam_dependency "beam-sdks-java-extensions-sql-iceberg"
add_dependency "org.apache.iceberg" "iceberg-aws" "${ICEBERG_VERSION}"
add_dependency "org.apache.iceberg" "iceberg-aws-bundle" "${ICEBERG_VERSION}"
add_dependency "org.apache.iceberg" "iceberg-gcp" "${ICEBERG_VERSION}"
;;
esac
done
RUNNER_ARTIFACT=""
case "${BEAM_RUNNER}" in
Expand All @@ -372,12 +415,25 @@ EOL
*) echo "❌ Error: Unsupported runner '${BEAM_RUNNER}'." >&2; exit 1 ;;
esac
if [ -n "${RUNNER_ARTIFACT}" ]; then
echo " <dependency><groupId>org.apache.beam</groupId><artifactId>${RUNNER_ARTIFACT}</artifactId><version>\${beam.version}</version></dependency>" >> "${POM_FILE}"
add_beam_dependency "${RUNNER_ARTIFACT}"
fi

# Complete the POM with the build section for the maven-shade-plugin
cat >> "${POM_FILE}" << EOL
</dependencies>
<repositories>
<repository>
<id>apache.snapshots</id>
<url>${APACHE_SNAPSHOT_REPOSITORY_URL}</url>
<releases>
<enabled>false</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
<updatePolicy>always</updatePolicy>
</snapshots>
</repository>
</repositories>
<properties>
<beam.version>${BEAM_VERSION}</beam.version>
</properties>
Expand All @@ -395,6 +451,11 @@ cat >> "${POM_FILE}" << EOL
</goals>
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<manifestEntries>
<Multi-Release>true</Multi-Release>
</manifestEntries>
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
</transformers>
<filters>
Expand Down
Loading