[FSTORE-789] multi modular hsfs java client (#975)

multi modular hsfs java client

Author: davitbzh

.github/workflows/mkdocs-master.yml

@@ -35,9 +35,15 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-maven-
 
+      - name: Set up JDK 8
+        uses: actions/setup-java@v3
+        with:
+          java-version: '8'
+          distribution: 'adopt'
+
       - name: Build java doc documentation
         working-directory: ./java
-        run: mvn clean package javadoc:javadoc -DskipTests && cp -r target/apidocs ../docs/javadoc
+        run: mvn clean install javadoc:javadoc javadoc:aggregate -DskipTests && cp -r target/site/apidocs ../docs/javadoc
 
       - name: setup git
         run: |

.github/workflows/mkdocs-release.yml

@@ -42,9 +42,15 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-maven-
 
+      - name: Set up JDK 8
+        uses: actions/setup-java@v3
+        with:
+          java-version: '8'
+          distribution: 'adopt'
+
       - name: Build java doc documentation
         working-directory: ./java
-        run: mvn clean package javadoc:javadoc -DskipTests && cp -r target/apidocs ../docs/javadoc
+        run: mvn clean install javadoc:javadoc javadoc:aggregate -DskipTests && cp -r target/site/apidocs ../docs/javadoc
 
       - name: setup git
         run: |

java/hsfs/pom.xml

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <parent>
+    <artifactId>hsfs-sdk</artifactId>
+    <groupId>com.logicalclocks</groupId>
+    <version>3.2.0-SNAPSHOT</version>
+  </parent>
+  <modelVersion>4.0.0</modelVersion>
+
+  <artifactId>hsfs</artifactId>
+
+
+  <properties>
+    <javax.version>2.2.11</javax.version>
+  </properties>
+
+  <dependencies>
+    <!-- https://mvnrepository.com/artifact/javax.xml.bind/jaxb-api -->
+    <dependency>
+      <groupId>javax.xml.bind</groupId>
+      <artifactId>jaxb-api</artifactId>
+      <version>${javax.version}</version>
+      <scope>provided</scope>
+    </dependency>
+  </dependencies>
+</project>

java/src/main/java/com/logicalclocks/hsfs/DataFormat.java renamed to java/hsfs/src/main/java/com/logicalclocks/hsfs/DataFormat.java

@@ -23,20 +23,16 @@
 import com.google.common.base.Strings;
 import com.logicalclocks.hsfs.metadata.Option;
 import com.logicalclocks.hsfs.metadata.StorageConnectorApi;
-
 import com.logicalclocks.hsfs.util.Constants;
-import com.logicalclocks.hsfs.spark.engine.SparkEngine;
+
 import lombok.AllArgsConstructor;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.Setter;
 import lombok.ToString;
 
 import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Paths;
 import java.time.Instant;
-import java.util.Base64;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -491,17 +487,11 @@ public static class BigqueryConnector extends StorageConnector {
     /**
      * Set spark options specific to BigQuery.
      * @return Map
-     * @throws IOException IOException
      */
     @Override
-    public Map<String, String> sparkOptions() throws IOException {
+    public Map<String, String> sparkOptions() {
       Map<String, String> options = new HashMap<>();
 
-      // Base64 encode the credentials file
-      String localKeyPath = SparkEngine.getInstance().addFile(keyPath);
-      byte[] fileContent = Files.readAllBytes(Paths.get(localKeyPath));
-      options.put(Constants.BIGQ_CREDENTIALS, Base64.getEncoder().encodeToString(fileContent));
-
       options.put(Constants.BIGQ_PARENT_PROJECT, parentProject);
       if (!Strings.isNullOrEmpty(materializationDataset)) {
         options.put(Constants.BIGQ_MATERIAL_DATASET, materializationDataset);

java/src/main/java/com/logicalclocks/hsfs/DeltaStreamerJobConf.java renamed to java/hsfs/src/main/java/com/logicalclocks/hsfs/DeltaStreamerJobConf.java

@@ -18,9 +18,9 @@
 package com.logicalclocks.hsfs.engine;
 
 import com.logicalclocks.hsfs.metadata.FeatureGroupApi;
-import com.logicalclocks.hsfs.FeatureGroupBase;
 import com.logicalclocks.hsfs.metadata.HopsworksClient;
 import com.logicalclocks.hsfs.metadata.TagsApi;
+import com.logicalclocks.hsfs.FeatureGroupBase;
 import com.logicalclocks.hsfs.EntityEndpointType;
 import com.logicalclocks.hsfs.Feature;
 import com.logicalclocks.hsfs.FeatureStoreException;

java/src/main/java/com/logicalclocks/hsfs/EntityEndpointType.java renamed to java/hsfs/src/main/java/com/logicalclocks/hsfs/EntityEndpointType.java

@@ -17,16 +17,16 @@
 
 package com.logicalclocks.hsfs.engine;
 
-import com.logicalclocks.hsfs.Feature;
-import com.logicalclocks.hsfs.FeatureStoreBase;
 import com.logicalclocks.hsfs.metadata.FeatureGroupApi;
-import com.logicalclocks.hsfs.FeatureGroupBase;
 import com.logicalclocks.hsfs.metadata.HopsworksClient;
 import com.logicalclocks.hsfs.metadata.KafkaApi;
+import com.logicalclocks.hsfs.metadata.Subject;
+import com.logicalclocks.hsfs.Feature;
+import com.logicalclocks.hsfs.FeatureStoreBase;
+import com.logicalclocks.hsfs.FeatureGroupBase;
 import com.logicalclocks.hsfs.FeatureGroupCommit;
 import com.logicalclocks.hsfs.FeatureStoreException;
 
-import com.logicalclocks.hsfs.metadata.Subject;
 import lombok.SneakyThrows;
 import org.apache.avro.Schema;
 import org.apache.avro.SchemaBuilder;

java/src/main/java/com/logicalclocks/hsfs/ExternalDataFormat.java renamed to java/hsfs/src/main/java/com/logicalclocks/hsfs/ExternalDataFormat.java

@@ -20,15 +20,15 @@
 import com.google.common.base.Strings;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
+import com.logicalclocks.hsfs.constructor.Join;
+import com.logicalclocks.hsfs.constructor.QueryBase;
+import com.logicalclocks.hsfs.metadata.FeatureViewApi;
+import com.logicalclocks.hsfs.metadata.TagsApi;
 import com.logicalclocks.hsfs.Feature;
 import com.logicalclocks.hsfs.FeatureGroupBaseForApi;
 import com.logicalclocks.hsfs.Split;
 import com.logicalclocks.hsfs.TrainingDatasetBase;
-import com.logicalclocks.hsfs.constructor.Join;
-import com.logicalclocks.hsfs.constructor.QueryBase;
 import com.logicalclocks.hsfs.FeatureGroupBase;
-import com.logicalclocks.hsfs.metadata.FeatureViewApi;
-import com.logicalclocks.hsfs.metadata.TagsApi;
 import com.logicalclocks.hsfs.EntityEndpointType;
 import com.logicalclocks.hsfs.FeatureStoreBase;
 import com.logicalclocks.hsfs.FeatureStoreException;

java/src/main/java/com/logicalclocks/hsfs/Feature.java renamed to java/hsfs/src/main/java/com/logicalclocks/hsfs/Feature.java

@@ -18,10 +18,10 @@
 package com.logicalclocks.hsfs.metadata;
 
 import com.damnhandy.uri.template.UriTemplate;
-import com.logicalclocks.hsfs.FeatureGroupBase;
 import com.logicalclocks.hsfs.constructor.FeatureGroupAlias;
 import com.logicalclocks.hsfs.constructor.FsQueryBase;
 import com.logicalclocks.hsfs.constructor.QueryBase;
+import com.logicalclocks.hsfs.FeatureGroupBase;
 import com.logicalclocks.hsfs.FeatureStoreBase;
 import com.logicalclocks.hsfs.FeatureStoreException;
 import org.apache.http.HttpHeaders;

java/src/main/java/com/logicalclocks/hsfs/FeatureGroupBase.java renamed to java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureGroupBase.java

@@ -31,9 +31,6 @@
 
 import java.io.IOException;
 
-import static com.logicalclocks.hsfs.metadata.HopsworksClient.PROJECT_PATH;
-import static com.logicalclocks.hsfs.metadata.HopsworksClient.getInstance;
-
 public class StatisticsApi {
 
   private static final String ENTITY_ROOT_PATH = "{/entityType}";
@@ -65,7 +62,7 @@ public Statistics post(TrainingDatasetBase trainingDatasetBase, Statistics stati
 
   private Statistics post(Integer projectId, Integer featurestoreId, Integer entityId, Statistics statistics)
       throws FeatureStoreException, IOException {
-    String pathTemplate = PROJECT_PATH + FeatureStoreApi.FEATURE_STORE_PATH + STATISTICS_PATH;
+    String pathTemplate = HopsworksClient.PROJECT_PATH + FeatureStoreApi.FEATURE_STORE_PATH + STATISTICS_PATH;
 
     String uri = UriTemplate.fromTemplate(pathTemplate)
         .set("projectId", projectId)
@@ -79,7 +76,7 @@ private Statistics post(Integer projectId, Integer featurestoreId, Integer entit
   public Statistics post(FeatureViewBase featureViewBase,
                          Integer trainingDataVersion, Statistics statistics)
       throws FeatureStoreException, IOException {
-    String pathTemplate = PROJECT_PATH + FeatureStoreApi.FEATURE_STORE_PATH + FV_STATISTICS_PATH;
+    String pathTemplate = HopsworksClient.PROJECT_PATH + FeatureStoreApi.FEATURE_STORE_PATH + FV_STATISTICS_PATH;
 
     String uri = UriTemplate.fromTemplate(pathTemplate)
         .set("projectId", featureViewBase.getFeatureStore().getProjectId())
@@ -92,7 +89,7 @@ public Statistics post(FeatureViewBase featureViewBase,
   }
 
   private Statistics post(String uri, Statistics statistics) throws FeatureStoreException, IOException {
-    HopsworksClient hopsworksClient = getInstance();
+    HopsworksClient hopsworksClient = HopsworksClient.getInstance();
     LOGGER.info("Sending metadata request: " + uri);
     HttpPost postRequest = new HttpPost(uri);
     postRequest.setEntity(hopsworksClient.buildStringEntity(statistics));
@@ -114,7 +111,7 @@ public Statistics get(TrainingDatasetBase trainingDatasetBase, String commitTime
   private Statistics get(Integer projectId, Integer featurestoreId, Integer entityId, String commitTime)
       throws FeatureStoreException, IOException {
     HopsworksClient hopsworksClient = HopsworksClient.getInstance();
-    String pathTemplate = PROJECT_PATH
+    String pathTemplate = HopsworksClient.PROJECT_PATH
         + FeatureStoreApi.FEATURE_STORE_PATH
         + STATISTICS_PATH;
 
@@ -152,7 +149,7 @@ public Statistics getLast(TrainingDatasetBase trainingDatasetBase) throws Featur
   private Statistics getLast(Integer projectId, Integer featurestoreId, Integer entityId)
       throws FeatureStoreException, IOException {
     HopsworksClient hopsworksClient = HopsworksClient.getInstance();
-    String pathTemplate = PROJECT_PATH
+    String pathTemplate = HopsworksClient.PROJECT_PATH
         + FeatureStoreApi.FEATURE_STORE_PATH
         + STATISTICS_PATH;
 

java/src/main/java/com/logicalclocks/hsfs/FeatureGroupBaseForApi.java renamed to java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureGroupBaseForApi.java

@@ -38,16 +38,14 @@
 import java.util.Map;
 import java.util.Optional;
 
-import static com.logicalclocks.hsfs.metadata.HopsworksClient.PROJECT_PATH;
-
 public class TagsApi {
 
   public static final String ENTITY_ROOT_PATH = "{/entityType}";
   public static final String ENTITY_ID_PATH = ENTITY_ROOT_PATH + "{/entityId}";
   public static final String TAGS_PATH = ENTITY_ID_PATH + "/tags{/name}{?value}";
-  public static final String FV_TAGS_PATH = PROJECT_PATH + FeatureStoreApi.FEATURE_STORE_PATH
+  public static final String FV_TAGS_PATH = HopsworksClient.PROJECT_PATH + FeatureStoreApi.FEATURE_STORE_PATH
       + "/featureview{/fvName}/version{/fvVersion}/tags{/name}";
-  public static final String FV_TD_TAGS_PATH = PROJECT_PATH + FeatureStoreApi.FEATURE_STORE_PATH
+  public static final String FV_TD_TAGS_PATH = HopsworksClient.PROJECT_PATH + FeatureStoreApi.FEATURE_STORE_PATH
       + "/featureview{/fvName}/version{/fvVersion}"
       + "/trainingdatasets/version{/tdVersion}/tags{/name}";
 
@@ -62,7 +60,7 @@ public TagsApi(@NonNull EntityEndpointType entityType) {
   private void add(Integer projectId, Integer featurestoreId, Integer entityId, String name, Object value)
       throws FeatureStoreException, IOException {
 
-    String pathTemplate = PROJECT_PATH
+    String pathTemplate = HopsworksClient.PROJECT_PATH
         + FeatureStoreApi.FEATURE_STORE_PATH
         + TAGS_PATH;
 
@@ -147,7 +145,7 @@ private UriTemplate getFvTdUriTemplate(FeatureViewBase featureViewBase, Integer
 
   private Map<String, Object> get(Integer projectId, Integer featurestoreId, Integer entityId, Optional<String> name)
       throws FeatureStoreException, IOException {
-    String pathTemplate = PROJECT_PATH
+    String pathTemplate = HopsworksClient.PROJECT_PATH
         + FeatureStoreApi.FEATURE_STORE_PATH
         + TAGS_PATH;
 
@@ -238,7 +236,7 @@ public Object parseTagValue(ObjectMapper objectMapper, Object value) throws IOEx
 
   private void deleteTag(Integer projectId, Integer featurestoreId, Integer entityId, String name)
       throws FeatureStoreException, IOException {
-    String pathTemplate = PROJECT_PATH
+    String pathTemplate = HopsworksClient.PROJECT_PATH
         + FeatureStoreApi.FEATURE_STORE_PATH
         + TAGS_PATH;
 

java/src/main/java/com/logicalclocks/hsfs/FeatureGroupCommit.java renamed to java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureGroupCommit.java

@@ -14,10 +14,9 @@
  *  See the License for the specific language governing permissions and limitations under the License.
  *
  */
-package com.logicalclocks;
+package com.logicalclocks.hsfs;
 
 import com.logicalclocks.hsfs.metadata.HopsworksExternalClient;
-import com.logicalclocks.hsfs.FeatureStoreException;
 import com.logicalclocks.hsfs.metadata.Credentials;
 import io.specto.hoverfly.junit.core.SimulationSource;
 import io.specto.hoverfly.junit.dsl.HttpBodyConverter;

java/src/main/java/com/logicalclocks/hsfs/FeatureStoreBase.java renamed to java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureStoreBase.java

@@ -42,7 +42,7 @@ public void testStringEntitySerialization() throws IOException {
     Assertions.assertEquals("Content-Type: application/json; charset=UTF-8",
         stringEntity.getContentType().toString());
 
-    String json = IOUtils.toString(stringEntity.getContent(), StandardCharsets.UTF_8);
+    String json = IOUtils.toString(stringEntity.getContent(), String.valueOf(StandardCharsets.UTF_8));
     Assertions.assertEquals("{\"email\":\"[email protected]\",\"firstName\":\"test\",\"lastName\":\"de la Rúa Martínez\"}",
         json);
   }