Merge branch 'main' into chore/omid-1076

Author: razvan

CHANGELOG.md

@@ -25,6 +25,7 @@ All notable changes to this project will be documented in this file.
   `check-permissions-ownership.sh` provided in stackable-base image ([#1027]).
 - nifi: Add [nifi-iceberg-bundle] for NiFi `2.4.0` ([#1060], [#1106]).
 - nifi: Add `2.4.0` ([#1114]).
+- nifi: Add git-sync ([#1107]).
 - opa: check for correct permissions and ownerships in /stackable folder via
   `check-permissions-ownership.sh` provided in stackable-base image ([#1038]).
 - opa: Add `1.4.2` ([#1103]).
@@ -56,6 +57,7 @@ All notable changes to this project will be documented in this file.
 - yq: Bump products to use `4.45.2` ([#1090]).
 - cyclonedx-bom: Bump airflow and superset to use `6.0.0` ([#1090]).
 - vector: Bump to `0.46.1` ([#1098]).
+- spark: update dependencies for 3.5.5 ([#1094])
 
 ### Fixed
 
@@ -81,6 +83,7 @@ All notable changes to this project will be documented in this file.
   Also remove the old release workflow.
 - zookeeper: Remove 3.9.2 ([#1093]).
 - Remove ubi8-rust-builder image ([#1091]).
+- spark: remove 3.5.2 ([#1094])
 - hadoop: Remove `3.3.4` and `3.4.0` ([#1099]).
 - opa: Remove `0.67.1` ([#1103]).
 - opa: Remove legacy bundle-builder from container build ([#1103]).
@@ -112,6 +115,7 @@ All notable changes to this project will be documented in this file.
 [#1090]: https://github.com/stackabletech/docker-images/pull/1090
 [#1091]: https://github.com/stackabletech/docker-images/pull/1091
 [#1093]: https://github.com/stackabletech/docker-images/pull/1093
+[#1094]: https://github.com/stackabletech/docker-images/pull/1094
 [#1097]: https://github.com/stackabletech/docker-images/pull/1097
 [#1098]: https://github.com/stackabletech/docker-images/pull/1098
 [#1099]: https://github.com/stackabletech/docker-images/pull/1099
@@ -120,6 +124,7 @@ All notable changes to this project will be documented in this file.
 [#1103]: https://github.com/stackabletech/docker-images/pull/1103
 [#1105]: https://github.com/stackabletech/docker-images/pull/1105
 [#1106]: https://github.com/stackabletech/docker-images/pull/1106
+[#1107]: https://github.com/stackabletech/docker-images/pull/1107
 [#1108]: https://github.com/stackabletech/docker-images/pull/1108
 [#1110]: https://github.com/stackabletech/docker-images/pull/1110
 [#1114]: https://github.com/stackabletech/docker-images/pull/1114

nifi/Dockerfile

@@ -1,5 +1,9 @@
 # syntax=docker/dockerfile:1.10.0@sha256:865e5dd094beca432e8c0a1d5e1c465db5f998dca4e439981029b3b81fb39ed5
-# check=error=true
+# check=error=true;skip=InvalidDefaultArgInFrom
+
+ARG GIT_SYNC
+
+FROM oci.stackable.tech/sdp/git-sync/git-sync:${GIT_SYNC} AS git-sync-image
 
 FROM stackable/image/java-devel AS nifi-builder
 
@@ -32,6 +36,7 @@ WORKDIR /stackable
 
 COPY --chown=${STACKABLE_USER_UID}:0 nifi/stackable/patches/patchable.toml /stackable/src/nifi/stackable/patches/patchable.toml
 COPY --chown=${STACKABLE_USER_UID}:0 nifi/stackable/patches/${PRODUCT} /stackable/src/nifi/stackable/patches/${PRODUCT}
+COPY --chown=${STACKABLE_USER_UID}:0 --from=git-sync-image /git-sync /stackable/git-sync
 
 RUN <<EOF
 # This used to be located in /bin/stackable-bcrypt.jar. We create a softlink for /bin/stackable-bcrypt.jar in the main container for backwards compatibility.
@@ -125,6 +130,7 @@ COPY --chown=${STACKABLE_USER_UID}:0 --from=nifi-builder /stackable/nifi-${PRODU
 COPY --chown=${STACKABLE_USER_UID}:0 --from=nifi-builder /stackable/stackable-bcrypt.jar /stackable/stackable-bcrypt.jar
 COPY --chown=${STACKABLE_USER_UID}:0 --from=nifi-iceberg-bundle-builder /stackable/*.nar /stackable/nifi-${PRODUCT}/lib/
 COPY --chown=${STACKABLE_USER_UID}:0 --from=nifi-iceberg-bundle-builder /stackable/*.sbom.json /stackable/nifi-${PRODUCT}/lib/
+COPY --chown=${STACKABLE_USER_UID}:0 --from=nifi-builder /stackable/git-sync /stackable/git-sync
 
 COPY --chown=${STACKABLE_USER_UID}:0 nifi/stackable/bin /stackable/bin
 COPY --chown=${STACKABLE_USER_UID}:0 nifi/licenses /licenses
@@ -133,9 +139,11 @@ COPY --chown=${STACKABLE_USER_UID}:0 nifi/python /stackable/python
 RUN <<EOF
 microdnf update
 
+# git: Required for git-sync
 # python-pip: Required to install Python packages
 # Note: Python is also required for the Python processors (ExecuteScript and other for NiFi < 2.0 and custom Python processors for NiFi > 2.0)
 microdnf install \
+  git \
   python-pip
 
 microdnf clean all

nifi/versions.py

@@ -3,16 +3,19 @@
         "product": "1.27.0",
         "java-base": "11",
         "java-devel": "11",  # There is an error when trying to use the jdk 21 (since nifi 1.26.0)
+        "git_sync": "v4.4.0",
     },
     {
         "product": "1.28.1",
         "java-base": "11",
         "java-devel": "11",
+        "git_sync": "v4.4.0",
     },
     {
         "product": "2.4.0",
         "java-base": "21",
         "java-devel": "21",
+        "git_sync": "v4.4.0",
         "nifi_iceberg_bundle": "0.0.4",
     },
 ]

spark-k8s/Dockerfile

@@ -155,7 +155,7 @@ WORKDIR /stackable/spark-${PRODUCT}/dist/jars
 # Copy modules required for s3a://
 COPY --from=hadoop-builder --chown=${STACKABLE_USER_UID}:0 \
     /stackable/hadoop/share/hadoop/tools/lib/hadoop-aws-${HADOOP}.jar \
-    /stackable/hadoop/share/hadoop/tools/lib/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE}.jar \
+    /stackable/hadoop/share/hadoop/tools/lib/bundle-${AWS_JAVA_SDK_BUNDLE}.jar \
     ./
 
 # Copy modules required for abfs://

spark-k8s/stackable/patches/3.5.2/0001-Update-CycloneDX-plugin.patch

@@ -1,31 +1,13 @@
 versions = [
-    {
-        "product": "3.5.2",
-        "java-base": "17",
-        "java-devel": "17",
-        "python": "3.11",
-        "hadoop": "3.3.6",  # Hadoop version defined in ../hbase/versions.py to reduce build time and disk requirements
-        "hbase": "2.6.1",  # current Stackable LTS version
-        "aws_java_sdk_bundle": "1.12.367",  # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.6
-        "azure_storage": "7.0.1",  # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.3.6
-        "azure_keyvault_core": "1.0.0",  # https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1
-        "jackson_dataformat_xml": "2.15.2",  # https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/3.5.1
-        "stax2_api": "4.2.1",  # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2
-        "woodstox_core": "6.5.1",  # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2
-        "vector": "0.46.1",
-        "jmx_exporter": "1.2.0",
-        "tini": "0.19.0",
-        "hbase_connector": "1.0.1",
-    },
     {
         "product": "3.5.5",
         "java-base": "17",
         "java-devel": "17",
         "python": "3.11",
-        "hadoop": "3.3.6",  # Hadoop version defined in ../hbase/versions.py to reduce build time and disk requirements
-        "hbase": "2.6.1",  # current Stackable LTS version
-        "aws_java_sdk_bundle": "1.12.367",  # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.6
-        "azure_storage": "7.0.1",  # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.3.6
+        "hadoop": "3.4.1",  # Current Stackable LTS version. Source of the AWS and Azure artifacts to Spark's classpath.
+        "hbase": "2.6.2",  # Current Stackable LTS version. Used to build the HBase connector.
+        "aws_java_sdk_bundle": "2.24.6",  # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.4.1
+        "azure_storage": "7.0.1",  # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.4.1
         "azure_keyvault_core": "1.0.0",  # https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1
         "jackson_dataformat_xml": "2.15.2",  # https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/3.5.2
         "stax2_api": "4.2.1",  # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2