Update HBASE to 2.3.7 in PXF (#51)

ostinru · web-flow · commit 34f6de376146 · 2026-02-18T19:43:20.000+05:00
### hbase-client update

Update `hbase-client` from `1.3.2` to `2.3.7` (latest version with minimal 3rd-party library updates) with following decision made:
* Use hadoop 2 version - Despite of the fact that we are using hadoop 3 in automation tests, we still using hadoop2 libs for all other connectors.
* Use shaded version of libraries when it relocates java-packages to new namespaces
* Use non-shaded version of libraries when it just bundles several jars to single fat-jar. Fat jars hides dependencies from gradle, however puts classes on classpath. This will lead to unpredictable issues.

Fortunately, `hbase-client:2.3.7` depends on `hadoop-2.10.0` that we are using! No changes here.

### update details

```
runtimeClasspath - Runtime classpath of source set 'main'.
\--- org.apache.hbase:hbase-client:2.3.7
     +--- org.apache.hbase.thirdparty:hbase-shaded-protobuf:3.3.0
     +--- org.apache.hbase:hbase-common:2.3.7
     |    +--- org.apache.hbase:hbase-logging:2.3.7
     |    +--- org.apache.hbase.thirdparty:hbase-shaded-miscellaneous:3.3.0
     |    |    \--- com.google.errorprone:error_prone_annotations:2.3.4
     |    +--- org.apache.hbase.thirdparty:hbase-shaded-gson:3.3.0
     |    +--- org.apache.hbase.thirdparty:hbase-shaded-netty:3.3.0
     |    +--- commons-codec:commons-codec:1.13
     |    +--- org.apache.commons:commons-lang3:3.9
     |    +--- commons-io:commons-io:2.11.0
     |    +--- com.google.protobuf:protobuf-java:2.5.0
     |    +--- org.apache.htrace:htrace-core4:4.2.0-incubating
     |    +--- org.apache.commons:commons-crypto:1.0.0
     |    +--- org.apache.yetus:audience-annotations:0.5.0
     |    \--- org.apache.hadoop:hadoop-common:2.10.0 -&gt; 2.10.2
     +--- org.apache.hbase:hbase-hadoop-compat:2.3.7
     |    +--- org.apache.hbase.thirdparty:hbase-shaded-miscellaneous:3.3.0 (*)
     |    +--- org.apache.hbase:hbase-metrics-api:2.3.7
     |    |    +--- org.apache.hbase:hbase-common:2.3.7 (*)
     |    |    +--- org.apache.commons:commons-lang3:3.9
     |    |    +--- org.apache.hbase.thirdparty:hbase-shaded-miscellaneous:3.3.0 (*)
     +--- org.apache.hbase:hbase-hadoop2-compat:2.3.7
     |    +--- org.apache.hbase:hbase-hadoop-compat:2.3.7 (*)
     |    +--- org.apache.hbase:hbase-common:2.3.7 (*)
     |    +--- org.apache.hbase:hbase-metrics:2.3.7
     |    |    +--- org.apache.hbase.thirdparty:hbase-shaded-miscellaneous:3.3.0 (*)
     |    |    +--- org.apache.hbase:hbase-common:2.3.7 (*)
     |    |    +--- org.apache.hbase:hbase-metrics-api:2.3.7 (*)
     |    |    +--- io.dropwizard.metrics:metrics-core:3.2.6
     |    +--- org.apache.hbase:hbase-metrics-api:2.3.7 (*)
     |    +--- org.apache.hadoop:hadoop-common:2.10.0 -&gt; 2.10.2 (*)
     |    +--- javax.activation:javax.activation-api:1.2.0
     |    +--- org.apache.commons:commons-lang3:3.9
     |    +--- org.apache.hbase.thirdparty:hbase-shaded-miscellaneous:3.3.0 (*)
     +--- org.apache.hbase:hbase-protocol-shaded:2.3.7
     |    +--- org.apache.hbase.thirdparty:hbase-shaded-protobuf:3.3.0
     +--- org.apache.hbase:hbase-protocol:2.3.7
     |    +--- com.google.protobuf:protobuf-java:2.5.0
     +--- commons-codec:commons-codec:1.13
     +--- commons-io:commons-io:2.11.0
     +--- org.apache.commons:commons-lang3:3.9
     +--- org.slf4j:slf4j-api:1.7.30 -&gt; 1.7.36
     +--- org.apache.hbase.thirdparty:hbase-shaded-miscellaneous:3.3.0 (*)
     +--- com.google.protobuf:protobuf-java:2.5.0
     +--- org.apache.hbase.thirdparty:hbase-shaded-netty:3.3.0
     +--- org.apache.zookeeper:zookeeper:3.5.7 (*)
     +--- org.apache.htrace:htrace-core4:4.2.0-incubating
     +--- org.jruby.jcodings:jcodings:1.0.18
     +--- org.jruby.joni:joni:2.1.11
     |    \--- org.jruby.jcodings:jcodings:1.0.13 -&gt; 1.0.18
     +--- io.dropwizard.metrics:metrics-core:3.2.6 (*)
     +--- org.apache.commons:commons-crypto:1.0.0
     +--- org.apache.yetus:audience-annotations:0.5.0
     +--- org.apache.hadoop:hadoop-auth:2.10.0 -&gt; 2.10.2 (*)
     \--- org.apache.hadoop:hadoop-common:2.10.0 -&gt; 2.10.2 (*)

(*) - dependencies omitted (listed previously)
```

### Other changes

* Automation tests: upgrade hbase to 2.3.7
diff --git a/automation/docker/singlecluster/Dockerfile b/automation/docker/singlecluster/Dockerfile
@@ -10,14 +10,14 @@ RUN apt-get update &&  \
 ENV    HADOOP_VERSION=3.1.2
 ENV      HIVE_VERSION=3.1.3
 ENV ZOOKEEPER_VERSION=3.5.9
-ENV     HBASE_VERSION=2.0.6
+ENV     HBASE_VERSION=2.3.7
 ENV       TEZ_VERSION=0.9.2
 
 # checksums from archive.apache.org
 ENV    HADOOP_SHA512="0e0ee817c89b3c4eb761eca7f16640742a83b0e99b6fda26c1bee2baabedad93aab86e252bf5f1e2381c6d464bc4003d10c7cc0f61b2062f4c59732ca24d1bd9"
 ENV      HIVE_SHA256="0c9b6a6359a7341b6029cc9347435ee7b379f93846f779d710b13f795b54bb16"
 ENV ZOOKEEPER_SHA512="0e5a64713abc6f36d961dd61a06f681868171a9d9228366e512a01324806d263e05508029c94d8e18307811867cdc39d848e736c252bf56c461273ef74c66a45"
-ENV     HBASE_SHA512="a0e10904ecf7f059b77bc0ce704254046a978126db720cc7e55dc53b87097715da64b8391fe3cc94348bc432871ad8f29891dc8df1ea052eb628da0fdca97c93"
+ENV     HBASE_SHA512="1032521025660daa70260cdc931f52a26c87596be444451fe1fa88b526ede55e9d6b4220e91ff6f7422bec11f30d64fa6745e95a9c36971fdb1a264a2c745693"
 ENV      TEZ_SHA512="a2d94bd9fa778d42a8bac9d9da8e263e469ddfef93968b06434716554995f490231de5607541ac236e770aa0158b64250c38bc1cd57dbfa629fea705f2ffa2f5"
 
 # faster mirror:
@@ -59,7 +59,7 @@ RUN mkdir -p $ZOOKEEPER_ROOT && \
 RUN mkdir -p $HBASE_ROOT && \
     curl -fSL "$HBASE_URL" -o hbase.tar.gz && \
     echo "$HBASE_SHA512 hbase.tar.gz" | sha512sum -c && \
-    tar xvf hbase.tar.gz -C $HBASE_ROOT --strip-components 1 --exclude="docs/*" && \
+    tar xvf hbase.tar.gz -C $HBASE_ROOT --strip-components 1 --exclude="docs/*" --exclude="lib/*-tests.jar" --exclude="lib/shaded-clients" && \
     rm hbase.tar.gz
 
 RUN mkdir -p $TEZ_ROOT && \
diff --git a/automation/docker/singlecluster/README.HDP3.md b/automation/docker/singlecluster/README.HDP3.md
@@ -7,7 +7,7 @@ It contains the following versions:
 - Hadoop 3.3.6
 - Hive 3.1.3
 - Zookeeper 3.5.9
-- HBase 2.0.6
+- HBase 2.3.7
 - Tez 0.9.2
 
 This version of Single cluster requires users to make some manual changes to the configuration files once the tarball has been unpacked (see Initialization steps below).
diff --git a/server/build.gradle b/server/build.gradle
@@ -90,15 +90,17 @@ configure(javaProjects) {
             dependency("commons-configuration:commons-configuration:1.10")
             dependency("commons-io:commons-io:2.7")
             dependency("commons-lang:commons-lang:2.6")
+            dependency("commons-lang:commons-lang3:3.9")
             dependency("commons-logging:commons-logging:1.1.3")
             dependency("io.airlift:aircompressor:0.27")
             dependency("javax.jdo:jdo-api:3.0.1")
             dependency("joda-time:joda-time:2.8.1")
             dependency("net.sf.opencsv:opencsv:2.3")
             dependency("org.antlr:antlr-runtime:3.5.2")
             dependency("org.apache.commons:commons-compress:1.20")
+            dependency("org.apache.commons:commons-crypto:1.0.0")
             dependency("org.apache.htrace:htrace-core:3.1.0-incubating")
-            dependency("org.apache.htrace:htrace-core4:4.0.1-incubating")
+            dependency("org.apache.htrace:htrace-core4:4.2.0-incubating")
 
             dependency("org.apache.zookeeper:zookeeper:3.4.6")
             dependency("org.codehaus.woodstox:stax2-api:3.1.4")
@@ -135,11 +137,28 @@ configure(javaProjects) {
 
             // HBase dependencies
             dependencySet(group:"org.apache.hbase", version:"${hbaseVersion}") {
-                entry("hbase-annotations")
                 entry("hbase-client")
                 entry("hbase-common")
                 entry("hbase-protocol")
+                entry("hbase-protocol-shaded")
+                entry("hbase-logging")
+                entry("hbase-hadoop-compat")
+                entry("hbase-hadoop2-compat")
+                entry("hbase-metrics-api")
+                entry("hbase-metrics")
             }
+            dependencySet(group:"org.apache.hbase.thirdparty", version:"3.3.0") {
+                entry("hbase-shaded-protobuf")
+                entry("hbase-shaded-miscellaneous")
+                entry("hbase-shaded-gson")
+                entry("hbase-shaded-netty")
+                entry("hbase-unsafe")
+            }
+            dependency("org.apache.yetus:audience-annotations:0.5.0")
+            dependency("io.opentelemetry:opentelemetry-api:1.49.0")
+            dependency("io.opentelemetry:opentelemetry-context:1.49.0")
+            dependency("io.opentelemetry.semconv:opentelemetry-semconv:1.29.0-alpha")
+            dependency("io.dropwizard.metrics:metrics-core:3.2.6")
 
             // Hive dependencies
             dependency("org.apache.hive:hive-storage-api:${hiveStorageApiVersion}")
@@ -230,15 +249,15 @@ configure(javaProjects) {
         options.compilerArgs += [
                 "-g", "-Xlint:varargs", "-Xlint:cast", "-Xlint:classfile", "-Xlint:dep-ann", "-Xlint:divzero",
                 "-Xlint:empty", "-Xlint:finally", "-Xlint:overrides", "-Xlint:path", "-Xlint:-processing", "-Xlint:static",
-                "-Xlint:try", "-Xlint:fallthrough", "-Xlint:deprecation", "-Xlint:unchecked", "-Xlint:-options", "-Werror"
+                "-Xlint:try", "-Xlint:fallthrough", "-Xlint:unchecked", "-Xlint:-options", "-Werror"
         ]
     }
 
     compileTestJava {
         options.compilerArgs += [
                 "-g", "-Xlint:varargs", "-Xlint:cast", "-Xlint:classfile", "-Xlint:dep-ann", "-Xlint:divzero",
                 "-Xlint:empty", "-Xlint:finally", "-Xlint:overrides", "-Xlint:path", "-Xlint:-processing", "-Xlint:static",
-                "-Xlint:try", "-Xlint:fallthrough", "-Xlint:deprecation", "-Xlint:unchecked", "-Xlint:-options", "-Werror"
+                "-Xlint:try", "-Xlint:fallthrough", "-Xlint:unchecked", "-Xlint:-options", "-Werror"
         ]
     }
 
diff --git a/server/gradle.properties b/server/gradle.properties
@@ -21,7 +21,7 @@ license=ASL 2.0
 hadoopVersion=2.10.2
 hiveVersion=2.3.8
 hiveStorageApiVersion=2.7.3
-hbaseVersion=1.3.2
+hbaseVersion=2.3.7
 junitVersion=4.11
 parquetVersion=1.11.1
 awsJavaSdk=1.12.261
diff --git a/server/pxf-hbase/build.gradle b/server/pxf-hbase/build.gradle
@@ -14,23 +14,47 @@ dependencies {
      *******************************/
 
     compileOnly("com.google.code.findbugs:annotations")
-    compileOnly("org.apache.hbase:hbase-annotations")
 
     /*******************************
-     * Implementation Dependencies
+     * Project Dependencies
      *******************************/
 
     implementation(project(':pxf-api'))
-    implementation("com.google.protobuf:protobuf-java")
     implementation("commons-collections:commons-collections")
-    implementation("org.apache.hbase:hbase-client")                  { transitive = false }
-    implementation("org.apache.hbase:hbase-common")                  { transitive = false }
-    implementation("org.apache.hbase:hbase-protocol")                { transitive = false }
-    implementation("org.apache.htrace:htrace-core")                  { transitive = false }
-    implementation("org.apache.zookeeper:zookeeper")                 { transitive = false }
-    implementation("io.netty:netty-common")                          { transitive = false }
-    implementation("io.netty:netty-transport")                       { transitive = false }
-    implementation("com.yammer.metrics:metrics-core")                { transitive = false }
+
+    /*******************************
+     * Hbase
+     *******************************/
+
+    implementation("org.apache.hbase:hbase-client")                          { transitive = false }
+    implementation("org.apache.hbase.thirdparty:hbase-shaded-protobuf")      { transitive = false }
+    implementation("org.apache.hbase:hbase-common")                          { transitive = false }
+    implementation("org.apache.hbase:hbase-logging")                         { transitive = false }
+    implementation("org.apache.hbase.thirdparty:hbase-shaded-miscellaneous") { transitive = false }
+    implementation("org.apache.hbase.thirdparty:hbase-shaded-gson")          { transitive = false }
+    implementation("org.apache.hbase.thirdparty:hbase-shaded-netty")         { transitive = false }
+    implementation("org.apache.commons:commons-lang3")                       { transitive = false }
+    implementation("org.apache.commons:commons-crypto")                      { transitive = false }
+    implementation("org.apache.hadoop:hadoop-common")                        { transitive = false }
+    implementation("org.apache.hadoop:hadoop-auth")                          { transitive = false }
+    implementation("org.apache.hbase:hbase-hadoop-compat")                   { transitive = false }
+    implementation("org.apache.hbase:hbase-metrics-api")                     { transitive = false }
+    implementation("org.apache.hbase:hbase-metrics")                         { transitive = false }
+    implementation("org.apache.hbase:hbase-hadoop2-compat")                  { transitive = false }
+    implementation("org.apache.hbase:hbase-protocol-shaded")                 { transitive = false }
+    implementation("org.apache.hbase:hbase-protocol")                        { transitive = false }
+    implementation("com.google.protobuf:protobuf-java")                      { transitive = false }
+    implementation("org.apache.zookeeper:zookeeper")                         { transitive = false }
+    implementation("io.netty:netty-common")                                  { transitive = false }
+    implementation("io.netty:netty-transport")                               { transitive = false }
+//  skip JRuby - it is part of interactive shell
+//    implementation("org.jruby.jcodings:jcodings:1.0.58")                     { transitive = false }
+//    implementation("org.jruby.joni:joni:2.2.1")                              { transitive = false }
+    implementation("org.apache.yetus:audience-annotations")                  { transitive = false }
+    implementation("io.opentelemetry:opentelemetry-api")                     { transitive = false }
+    implementation("io.opentelemetry:opentelemetry-context")                 { transitive = false }
+    implementation("io.opentelemetry.semconv:opentelemetry-semconv")         { transitive = false }
+    implementation("io.dropwizard.metrics:metrics-core:3.2.6")               { transitive = false }
 
     implementation("org.springframework.boot:spring-boot-starter-log4j2")
 
@@ -39,7 +63,6 @@ dependencies {
      *******************************/
 
     testCompileOnly("com.google.code.findbugs:annotations")
-    testCompileOnly("org.apache.hbase:hbase-annotations")
     testImplementation("com.esotericsoftware:minlog")
     testImplementation("com.esotericsoftware:reflectasm")
     testImplementation('org.springframework.boot:spring-boot-starter-test')
diff --git a/server/pxf-hbase/src/main/java/org/greenplum/pxf/plugins/hbase/HBaseDataFragmenter.java b/server/pxf-hbase/src/main/java/org/greenplum/pxf/plugins/hbase/HBaseDataFragmenter.java
@@ -81,7 +81,6 @@ public FragmentStats getFragmentStats() {
     public List<Fragment> getFragments() throws Exception {
 
         // check that Zookeeper and HBase master are available
-        HBaseAdmin.checkHBaseAvailable(configuration);
         connection = ConnectionFactory.createConnection(configuration);
         Admin hbaseAdmin = connection.getAdmin();
         if (!HBaseUtilities.isTableAvailable(hbaseAdmin, context.getDataSource())) {
diff --git a/server/pxf-hbase/src/main/java/org/greenplum/pxf/plugins/hbase/utilities/HBaseDoubleComparator.java b/server/pxf-hbase/src/main/java/org/greenplum/pxf/plugins/hbase/utilities/HBaseDoubleComparator.java
@@ -19,11 +19,11 @@
  * under the License.
  */
 
-import com.google.protobuf.ByteString;
-import com.google.protobuf.InvalidProtocolBufferException;
+import org.apache.hbase.thirdparty.com.google.protobuf.ByteString;
+import org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
 import org.apache.hadoop.hbase.exceptions.DeserializationException;
 import org.apache.hadoop.hbase.filter.ByteArrayComparable;
-import org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ComparatorProtos;
 import org.apache.hadoop.hbase.util.Bytes;
 
 public class HBaseDoubleComparator extends ByteArrayComparable {
diff --git a/server/pxf-hbase/src/main/java/org/greenplum/pxf/plugins/hbase/utilities/HBaseFloatComparator.java b/server/pxf-hbase/src/main/java/org/greenplum/pxf/plugins/hbase/utilities/HBaseFloatComparator.java
@@ -19,11 +19,11 @@
  * under the License.
  */
 
-import com.google.protobuf.ByteString;
-import com.google.protobuf.InvalidProtocolBufferException;
+import org.apache.hbase.thirdparty.com.google.protobuf.ByteString;
+import org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
 import org.apache.hadoop.hbase.exceptions.DeserializationException;
 import org.apache.hadoop.hbase.filter.ByteArrayComparable;
-import org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ComparatorProtos;
 import org.apache.hadoop.hbase.util.Bytes;
 
 public class HBaseFloatComparator extends ByteArrayComparable{
diff --git a/server/pxf-hbase/src/main/java/org/greenplum/pxf/plugins/hbase/utilities/HBaseIntegerComparator.java b/server/pxf-hbase/src/main/java/org/greenplum/pxf/plugins/hbase/utilities/HBaseIntegerComparator.java
@@ -23,11 +23,11 @@
 import org.apache.hadoop.hbase.exceptions.DeserializationException;
 import org.apache.hadoop.hbase.filter.ByteArrayComparable;
 import org.apache.hadoop.hbase.filter.SubstringComparator;
-import org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ComparatorProtos;
 import org.apache.hadoop.hbase.util.Bytes;
 
-import com.google.protobuf.ByteString;
-import com.google.protobuf.InvalidProtocolBufferException;
+import org.apache.hbase.thirdparty.com.google.protobuf.ByteString;
+import org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
 
 /**
  * This is a Filter comparator for HBase It is external to PXF HBase code.
