Merge branch 'apache:main' into MDB_STABLE

.github/workflows/dependency-submission.yml

@@ -23,7 +23,7 @@ name: Dependency Submission
 
 on:
   push:
-    branches: [ 'main' ]
+    branches: [ 'main', 'MDB_STABLE' ]
 
 permissions:
   contents: write

LICENSE

@@ -215,6 +215,84 @@ This product is derived from software originally developed by:
    terms and conditions of the subcomponent's license, as noted in the LICENSE file.
 
 
+The Greenplum Platform Extension Framework includes:
+
+----------------------------
+   Apache License - Version 2.0
+
+The following files are licensed under the Apache License, Version 2.0:
+
+FDW Module:
+   fdw/libchurl.c
+   fdw/libchurl.h
+   fdw/pxf_bridge.c
+   fdw/pxf_bridge.h
+   fdw/pxf_filter.c
+   fdw/pxf_filter.h
+   fdw/pxf_header.c
+   fdw/pxf_header.h
+
+External Table Module:
+   external-table/src/gpdbwritableformatter.c
+   external-table/src/libchurl.c
+   external-table/src/libchurl.h
+   external-table/src/pxfbridge.c
+   external-table/src/pxfbridge.h
+   external-table/src/pxffilters.c
+   external-table/src/pxffilters.h
+   external-table/src/pxfheaders.c
+   external-table/src/pxfheaders.h
+   external-table/src/pxfprotocol.c
+   external-table/src/pxfuriparser.c
+   external-table/src/pxfuriparser.h
+   external-table/test/pxffilters_test.c
+   external-table/test/pxfheaders_test.c
+   external-table/test/pxfprotocol_test.c
+   external-table/test/pxfuriparser_test.c
+
+Server Module (Java Sources):
+   server/build.gradle
+   server/gradle.properties
+   server/settings.gradle
+   server/pxf-api/src/main/java/org/apache/cloudberry/pxf/api/*.java
+   server/pxf-api/src/test/java/org/apache/cloudberry/pxf/api/**/*.java
+   server/pxf-hbase/src/main/java/org/apache/cloudberry/pxf/plugins/hbase/**/*.java
+   server/pxf-hbase/src/test/java/org/apache/cloudberry/pxf/plugins/hbase/**/*.java
+   server/pxf-hdfs/src/main/java/org/apache/cloudberry/pxf/plugins/hdfs/**/*.java
+   server/pxf-hdfs/src/test/java/org/apache/cloudberry/pxf/plugins/hdfs/**/*.java
+   server/pxf-hive/src/main/java/org/apache/cloudberry/pxf/plugins/hive/**/*.java
+   server/pxf-hive/src/test/java/org/apache/cloudberry/pxf/plugins/hive/**/*.java
+   server/pxf-jdbc/src/main/java/org/apache/cloudberry/pxf/plugins/jdbc/**/*.java
+   server/pxf-jdbc/src/test/java/org/apache/cloudberry/pxf/plugins/jdbc/**/*.java
+   server/pxf-json/src/main/java/org/apache/cloudberry/pxf/plugins/json/**/*.java
+   server/pxf-json/src/test/java/org/apache/cloudberry/pxf/plugins/json/**/*.java
+   server/pxf-service/src/main/java/org/apache/cloudberry/pxf/service/**/*.java
+   server/pxf-service/src/test/java/org/apache/cloudberry/pxf/service/**/*.java
+
+Documentation Templates:
+   docs/content/*.html.md.erb
+
+Configuration Files:
+   server/pxf-api/src/test/resources/pxf-profiles-default.xml
+   server/pxf-hive/src/test/resources/pxf-profiles-default.xml
+   server/pxf-jdbc/src/test/resources/log4j.properties
+   server/pxf-json/src/test/resources/log4j.properties
+   server/pxf-service/src/main/resources/pxf-profiles-default.xml
+   server/pxf-service/src/templates/conf/pxf-profiles.xml
+   server/pxf-service/src/test/resources/pxf-profiles-default.xml
+
+CI/Test Templates:
+   automation/src/test/resources/templates/zk/zoo.cfg
+   ci/singlecluster/templates/hadoop/etc/hadoop/core-site.xml
+   ci/singlecluster/templates/hadoop/etc/hadoop/hdfs-site.xml
+   ci/singlecluster/templates/hadoop/etc/hadoop/yarn-env.sh
+   ci/singlecluster/templates/hbase/conf/hbase-env.sh
+   ci/singlecluster/templates/hbase/conf/hbase-site.xml
+   ci/singlecluster/templates/ranger/install.properties
+   ci/singlecluster/templates/tez/conf/tez-site.xml
+   ci/singlecluster/templates/usersync/install.properties
+
+
 The Greenplum Platform Extension Framework includes:
 
 ----------------------------

Makefile

@@ -92,7 +92,7 @@ ifeq ($(SKIP_FDW_PACKAGE_REASON),)
 else
 	@echo "Skipping staging FDW extension because $(SKIP_FDW_PACKAGE_REASON)"
 endif
-	make -C cli stage  
+	make -C cli stage
 	make -C server stage
 ifneq ($(SKIP_EXTERNAL_TABLE_PACKAGE_REASON),)
 	@echo "Skipping staging external-table extension because $(SKIP_EXTERNAL_TABLE_PACKAGE_REASON)"

server/Makefile

@@ -40,7 +40,10 @@ help:
 	@echo	"  - test - runs unit tests for all PXF modules"
 	@echo   "  - coverage - runs unit tests for all PXF modules and gives a coverage report"
 	@echo	"  - install - setup PXF in the configured deployPath"
-	@echo	"  - stage - build PXF server and stage resulting artifacts for packaging"
+	@echo	"  - stage - build PXF server and stage resulting artifacts for packaging (no JDBC drivers in build/stage/lib)"
+	@echo	"  - stage-notest - same as stage but skip tests (no JDBC drivers in build/stage/lib)"
+	@echo	"  - stage-jdbc-drivers - copy JDBC driver JARs to build/stage/lib"
+	@echo	"  - install-jdbc-drivers - setup PXF JDBC drivers in the configured deployPath"
 	@echo	"  - doc - creates aggregate javadoc under docs"
 
 all: prepare-gradle-wrapper
@@ -95,6 +98,18 @@ stage-notest: prepare-gradle-wrapper
 	install -m 700 -d "build/stage/run"
 	install -m 700 -d "build/stage/keytabs"
 
+.PHONY: stage-jdbc-drivers
+stage-jdbc-drivers: prepare-gradle-wrapper
+	./gradlew $(PXF_GRADLE_PROPERTIES) stageJdbcDrivers
+
+.PHONY: install
+install-jdbc-drivers: stage-jdbc-drivers
+	@if [ -z "$(PXF_HOME)" ]; then \
+		echo "ERROR: PXF_HOME is not set"; exit 2; \
+	fi
+	mkdir -p "$(PXF_HOME)"/lib
+	cp -R build/stage/lib "$(PXF_HOME)"/lib
+
 clean: prepare-gradle-wrapper
 	./gradlew clean
 	rm -rf build

server/build.gradle

@@ -212,8 +212,9 @@ configure(javaProjects) {
                 entry("avro")
                 entry("avro-mapred")
             }
-            // Zstd support for Avro/Parquet
+            // Compression support for Avro/Parquet
             dependency("com.github.luben:zstd-jni:1.5.7-6")
+            dependency("at.yawk.lz4:lz4-java:1.10.3")
 
             // Jackson 1.x dependencies
             dependencySet(group:"org.codehaus.jackson", version:"1.9.13") {
@@ -374,3 +375,11 @@ task stage(type: Copy) {
 
     doLast { new File("${buildDir}/stage/version").text = "${version}\n" }
 }
+
+task stageJdbcDrivers(type: Copy) {
+    description "Copy JDBC driver JARs from pxf-jdbc-drivers to build/stage/lib."
+    dependsOn ":pxf-jdbc-drivers:jar"
+    from(project(':pxf-jdbc-drivers').configurations.runtimeClasspath)
+    into("${buildDir}/stage/lib")
+    duplicatesStrategy = DuplicatesStrategy.INCLUDE
+}

server/pxf-jdbc-drivers/build.gradle

@@ -0,0 +1,34 @@
+dependencies {
+    // PostgreSQL driver is used by PXF itself. 
+    // Its version is defined in server/build.gradle
+
+    // Clickhouse
+    implementation('com.clickhouse:clickhouse-jdbc:0.6.5')
+    implementation('com.clickhouse:clickhouse-jdbc:0.6.5:http')
+    implementation('at.yawk.lz4:lz4-java')
+    implementation('org.apache.httpcomponents.client5:httpclient5:5.3.1')     { transitive = false }
+       implementation('org.apache.httpcomponents.core5:httpcore5:5.2.4')    { transitive = false }
+       implementation('org.apache.httpcomponents.core5:httpcore5-h2:5.2.4') { transitive = false }
+
+
+    // DB2
+    implementation('net.sf.jt400:jt400:11.2')
+
+    // MSSQL
+    implementation('com.microsoft.sqlserver:mssql-jdbc:8.2.2.jre11')
+
+    // MySQL
+    implementation('mysql:mysql-connector-java:8.0.26')       { transitive = false }
+
+    // Snowflake
+    implementation('net.snowflake:snowflake-jdbc:3.13.9')
+
+    // ORA
+    implementation('com.oracle.database.jdbc:ojdbc11:23.3.0.23.09')
+    implementation('com.oracle.database.jdbc:ucp11:23.3.0.23.09')
+    implementation('com.oracle.database.nls:orai18n:23.3.0.23.09')
+
+    // trino
+    implementation('io.trino:trino-jdbc:403')
+
+}