|
| 1 | +# syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7 |
| 2 | +# check=error=true |
| 3 | + |
| 4 | +# The purpose of this stage is to gather jars and environment variables needed in the final stage. |
| 5 | +# These are collected in the /stackable/spark directory. |
| 6 | +FROM local-image/java-devel AS spark-source-builder |
| 7 | + |
| 8 | +ARG RELEASE_VERSION |
| 9 | +ARG SPARK_VERSION |
| 10 | +ARG STACKABLE_USER_UID |
| 11 | + |
| 12 | +WORKDIR /stackable |
| 13 | + |
| 14 | +COPY --chown=${STACKABLE_USER_UID}:0 spark-k8s/stackable/patches/patchable.toml /stackable/src/spark-k8s/stackable/patches/patchable.toml |
| 15 | +COPY --chown=${STACKABLE_USER_UID}:0 spark-k8s/stackable/patches/${SPARK_VERSION} /stackable/src/spark-k8s/stackable/patches/${SPARK_VERSION} |
| 16 | + |
| 17 | +RUN <<EOF |
| 18 | +cd "$(/stackable/patchable --images-repo-root=src checkout spark-k8s ${SPARK_VERSION})" |
| 19 | + |
| 20 | +# NEW_VERSION="${SPARK_VERSION}-stackable${RELEASE_VERSION}" |
| 21 | + |
| 22 | +# mvn versions:set -DnewVersion=$NEW_VERSION |
| 23 | + |
| 24 | +mkdir -p /stackable/spark/jars |
| 25 | + |
| 26 | +# Download log4j-slf4j-impl-x.x.x.jar containing the StaticLoggerBinder |
| 27 | +# which is required by the connector. |
| 28 | +# Spark contains only log4j-slf4j2-impl-x.x.x.jar but not |
| 29 | +# log4j-slf4j-impl-x.x.x.jar. It is okay to have both JARs in the |
| 30 | +# classpath as long as they have the same version. |
| 31 | +mvn --non-recursive --file pom.xml \ |
| 32 | + dependency:copy \ |
| 33 | + -Dartifact=org.apache.logging.log4j:log4j-slf4j-impl:'${log4j.version}' \ |
| 34 | + -DoutputDirectory=/stackable/spark/jars |
| 35 | + |
| 36 | + |
| 37 | +# Create an environment files with properties needed to build the connector |
| 38 | +# Get the Scala version used by Spark |
| 39 | +SCALA_VERSION=$(grep "scala.version" pom.xml | head -n1 | awk -F '[<>]' '{print $3}') |
| 40 | + |
| 41 | +# Get the Scala binary version used by Spark |
| 42 | +SCALA_BINARY_VERSION=$(grep "scala.binary.version" pom.xml | head -n1 | awk -F '[<>]' '{print $3}') |
| 43 | + |
| 44 | +echo "SCALA_VERSION=${SCALA_VERSION}" > /stackable/spark/env |
| 45 | +echo "SCALA_BINARY_VERSION=${SCALA_BINARY_VERSION}" >> /stackable/spark/env |
| 46 | +echo "SPARK_VERSION=${SPARK_VERSION}" >> /stackable/spark/env |
| 47 | +EOF |
| 48 | + |
| 49 | +# hbase-connectors-builder: Build the Spark HBase connector and copy |
| 50 | +# required JARs into /stackable/spark/jars |
| 51 | +FROM local-image/java-devel AS final |
| 52 | + |
| 53 | +ARG PRODUCT_VERSION |
| 54 | +ARG RELEASE_VERSION |
| 55 | +ARG HADOOP_VERSION |
| 56 | +ARG HBASE_VERSION |
| 57 | +ARG STACKABLE_USER_UID |
| 58 | + |
| 59 | +# Patch the hbase-connectors source code |
| 60 | +WORKDIR /stackable |
| 61 | + |
| 62 | +COPY --chown=${STACKABLE_USER_UID}:0 spark-k8s/hbase-connectors/stackable/patches/patchable.toml /stackable/src/spark-k8s/hbase-connectors/stackable/patches/patchable.toml |
| 63 | +COPY --chown=${STACKABLE_USER_UID}:0 spark-k8s/hbase-connectors/stackable/patches/${PRODUCT_VERSION} /stackable/src/spark-k8s/hbase-connectors/stackable/patches/${PRODUCT_VERSION} |
| 64 | + |
| 65 | +# Copy jars and env from spark-source-builder |
| 66 | +COPY --chown=${STACKABLE_USER_UID}:0 --from=spark-source-builder \ |
| 67 | + /stackable/spark/jars \ |
| 68 | + spark/jars |
| 69 | +COPY --chown=${STACKABLE_USER_UID}:0 --from=spark-source-builder \ |
| 70 | + /stackable/spark/env \ |
| 71 | + spark/env |
| 72 | + |
| 73 | +RUN <<EOF |
| 74 | + |
| 75 | +cd "$(/stackable/patchable --images-repo-root=src checkout spark-k8s/hbase-connectors ${PRODUCT_VERSION})"/spark |
| 76 | + |
| 77 | +NEW_VERSION="${PRODUCT_VERSION}-stackable${RELEASE_VERSION}" |
| 78 | + |
| 79 | +mvn versions:set -DnewVersion=$NEW_VERSION |
| 80 | + |
| 81 | +# Create snapshot of the source code including custom patches |
| 82 | +tar -czf /stackable/hbase-connector-${NEW_VERSION}-src.tar.gz . |
| 83 | + |
| 84 | +# Building the hbase-connectors with JDK 17 is not yet supported, see |
| 85 | +# https://github.com/apache/hbase-connectors/pull/132. |
| 86 | +# As there are no JDK profiles, access to the non-public elements must |
| 87 | +# be enabled with --add-opens, see https://openjdk.org/jeps/403 and |
| 88 | +# https://openjdk.org/jeps/261#Breaking-encapsulation. |
| 89 | +export JDK_JAVA_OPTIONS="\ |
| 90 | + --add-opens java.base/java.lang=ALL-UNNAMED \ |
| 91 | + --add-opens java.base/java.util=ALL-UNNAMED" |
| 92 | + |
| 93 | +source /stackable/spark/env |
| 94 | + |
| 95 | +# Build the Spark HBase connector |
| 96 | +# Skip the tests because the MiniHBaseCluster does not get ready for |
| 97 | +# whatever reason: |
| 98 | +# Caused by: java.lang.RuntimeException: Master not active after 30000ms |
| 99 | +# at org.apache.hadoop.hbase.util.JVMClusterUtil.waitForEvent(JVMClusterUtil.java:221) |
| 100 | +# at org.apache.hadoop.hbase.util.JVMClusterUtil.startup(JVMClusterUtil.java:177) |
| 101 | +# at org.apache.hadoop.hbase.LocalHBaseCluster.startup(LocalHBaseCluster.java:407) |
| 102 | +# at org.apache.hadoop.hbase.MiniHBaseCluster.init(MiniHBaseCluster.java:250) |
| 103 | +mvn \ |
| 104 | + --batch-mode \ |
| 105 | + --no-transfer-progress \ |
| 106 | + --define spark.version="${SPARK_VERSION}" \ |
| 107 | + --define scala.version="${SCALA_VERSION}" \ |
| 108 | + --define scala.binary.version="${SCALA_BINARY_VERSION}" \ |
| 109 | + --define hadoop-three.version="${HADOOP_VERSION}" \ |
| 110 | + --define hbase.version="${HBASE_VERSION}" \ |
| 111 | + --define skipTests \ |
| 112 | + --define maven.test.skip=true \ |
| 113 | + clean package |
| 114 | + |
| 115 | + |
| 116 | +cp "$(pwd)/hbase-spark/target/hbase-spark-${NEW_VERSION}.jar" /stackable/spark/jars/hbase-spark-${NEW_VERSION}.jar |
| 117 | + |
| 118 | +chmod g=u /stackable/hbase-connector-${NEW_VERSION}-src.tar.gz . |
| 119 | +EOF |
0 commit comments