Merge branch 'main' into feature/94-comply-with-latest-delta-sharing-protocol

Author: agilelab-tmnd1991

.github/workflows/build_doc.yaml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         with:
           cache: 'gradle'
           distribution: temurin
@@ -18,13 +18,13 @@ jobs:
       - name: Execute npm run build
         run:  |
               cp -r protocol docsite/static/protocol
-              ./gradlew docsite:npm_run_build --no-daemon
+              ./gradlew docsite:npm_install docsite:npm_run_build --no-daemon
               chmod -c -R +rX "docsite/build" | while read line; do
                 echo "::warning title=Invalid file permissions automatically fixed::$line"
               done
         shell: bash
       - name: Upload gh-pages artifact
-        uses: actions/upload-pages-artifact@v2
+        uses: actions/upload-pages-artifact@v3
         with:
           path: docsite/build
   deploy:
@@ -43,5 +43,5 @@ jobs:
     steps:
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@v2
+        uses: actions/deploy-pages@v4
 

.github/workflows/compile.yaml

@@ -16,7 +16,7 @@ jobs:
         with:
           fetch-depth: 0
           ref: ${{ github.event.pull_request.head.sha }}
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         with:
           cache: 'gradle'
           distribution: temurin

.gitignore

@@ -12,4 +12,8 @@ build/
 .gradle
 .env
 client/bin
-server/bin
+server/bin
+.terraform
+.terraform.*
+*.tfstate
+*.tfstate.backup

build.gradle.kts

@@ -1,4 +1,4 @@
 group = "io.whitefox"
 plugins {
-    id("co.uzzu.dotenv.gradle").version("3.0.0")
+    id("co.uzzu.dotenv.gradle").version("4.0.0")
 }

buildSrc/build.gradle.kts

@@ -8,6 +8,6 @@ repositories {
 
 dependencies {
     implementation("org.openapi.generator:org.openapi.generator.gradle.plugin:6.6.0")
-    implementation("com.diffplug.spotless:spotless-plugin-gradle:6.23.1")
+    implementation("com.diffplug.spotless:spotless-plugin-gradle:6.23.3")
     implementation("com.palantir.gradle.gitversion:gradle-git-version:3.0.0")
 }

client-spark/build.gradle.kts

@@ -19,15 +19,14 @@ dependencies {
 
     // DELTA
     testImplementation("org.apache.hadoop:hadoop-common:3.3.6")
-    testImplementation("io.delta:delta-sharing-spark_2.12:1.0.2")
+    testImplementation("io.delta:delta-sharing-spark_2.13:1.0.3")
 
     //SPARK
-    testImplementation("org.apache.spark:spark-core_2.12:3.3.2")
-    testImplementation("org.apache.spark:spark-sql_2.12:3.3.2")
-    testImplementation("com.github.mrpowers:spark-fast-tests_2.12:1.3.0")
+    testImplementation("org.apache.spark:spark-sql_2.13:3.5.0")
+    testImplementation("com.github.mrpowers:spark-fast-tests_2.13:1.3.0")
 
     //JUNIT
-    testImplementation("org.junit.jupiter:junit-jupiter:5.8.1")
+    testImplementation("org.junit.jupiter:junit-jupiter:5.10.1")
 }
 
 

client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java

@@ -4,34 +4,31 @@
 
 import com.github.mrpowers.spark.fast.tests.DatasetComparer;
 import io.whitefox.api.client.model.CreateMetastore;
-import io.whitefox.api.client.model.Metastore;
+import io.whitefox.api.client.model.Provider;
 import io.whitefox.api.models.MrFoxDeltaTableSchema;
-import io.whitefox.api.utils.SparkUtil;
+import io.whitefox.api.utils.ScalaUtils;
 import io.whitefox.api.utils.StorageManagerInitializer;
 import io.whitefox.api.utils.TablePath;
+import io.whitefox.api.utils.TestSparkSession;
 import java.util.List;
-import org.apache.spark.sql.SparkSession;
 import org.apache.spark.sql.types.DataType;
 import org.apache.spark.sql.types.Metadata;
 import org.apache.spark.sql.types.StructField;
 import org.apache.spark.sql.types.StructType;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
-import scala.collection.GenMap;
 
 @Tag("clientSparkTest")
-public class ITDeltaSharingClient implements DatasetComparer, SparkUtil {
+public class ITDeltaSharingClient implements DatasetComparer, ScalaUtils {
 
   private final StorageManagerInitializer storageManagerInitializer;
   private final String deltaTablePath;
-  private final SparkSession spark;
 
   public ITDeltaSharingClient() {
     this.storageManagerInitializer = new StorageManagerInitializer();
     this.deltaTablePath =
         TablePath.getDeltaTablePath(getClass().getClassLoader().getResource("MrFoxProfile.json"));
-    this.spark = newSparkSession();
   }
 
   @BeforeAll
@@ -41,10 +38,11 @@ static void initStorageManager() {
 
   @Test
   void showS3Table1withQueryTableApi() {
+    var spark = TestSparkSession.newSparkSession();
     storageManagerInitializer.createS3DeltaTable();
     var ds = spark.read().format("deltaSharing").load(deltaTablePath);
     var expectedSchema = new StructType(new StructField[] {
-      new StructField("id", DataType.fromDDL("long"), true, new Metadata(GenMap.empty()))
+      new StructField("id", DataType.fromDDL("long"), true, new Metadata(emptyScalaMap()))
     });
     var expectedData = spark
         .createDataFrame(
@@ -57,15 +55,16 @@ void showS3Table1withQueryTableApi() {
             MrFoxDeltaTableSchema.class)
         .toDF();
 
-    assertEquals(expectedSchema.json(), ds.schema().json());
+    assertEquals(expectedSchema, ds.schema());
     assertEquals(5, ds.count());
     assertSmallDatasetEquality(ds, expectedData, true, false, false, 500);
   }
 
   @Test
-  void createGlueMetastore() {
-    Metastore metastore = storageManagerInitializer.createGlueMetastore();
-    assertEquals(metastore.getName(), "MrFoxMetastore");
-    assertEquals(metastore.getType(), CreateMetastore.TypeEnum.GLUE.getValue());
+  void createProviderWithGlueMetastore() {
+    Provider provider = storageManagerInitializer.createProviderWithGlueMetastore();
+    assertEquals(provider.getStorage().getName(), "MrFoxStorage");
+    assertEquals(provider.getMetastore().getName(), "MrFoxMetastore");
+    assertEquals(provider.getMetastore().getType(), CreateMetastore.TypeEnum.GLUE.getValue());
   }
 }

client-spark/src/test/java/io/whitefox/api/utils/ScalaUtils.java

@@ -0,0 +1,7 @@
+package io.whitefox.api.utils;
+
+public interface ScalaUtils {
+  default <K, V> scala.collection.immutable.Map<K, V> emptyScalaMap() {
+    return scala.collection.immutable.Map$.MODULE$.empty();
+  }
+}

client-spark/src/test/java/io/whitefox/api/utils/SparkUtil.java

@@ -49,11 +49,15 @@ public void createS3DeltaTable() {
         addTableToSchemaRequest(providerRequest.getName(), createTableRequest.getName())));
   }
 
-  public Metastore createGlueMetastore() {
+  public Provider createProviderWithGlueMetastore() {
     var metastoreRequest = createMetastoreRequest(s3TestConfig, CreateMetastore.TypeEnum.GLUE);
-    return ApiUtils.recoverConflictLazy(
+    var metastore = ApiUtils.recoverConflictLazy(
         () -> metastoreV1Api.createMetastore(metastoreRequest),
         () -> metastoreV1Api.describeMetastore(metastoreRequest.getName()));
+    var providerRequest = addProviderRequest(Optional.of(metastore.getName()), TableFormat.iceberg);
+    return ApiUtils.recoverConflictLazy(
+        () -> providerV1Api.addProvider(providerRequest),
+        () -> providerV1Api.getProvider(providerRequest.getName()));
   }
 
   private String createSchemaRequest(TableFormat tableFormat) {