Initial commit

1. Test Automation Framework skeleton
2. Gradle config with most likely deps
3. Template Test
4. Logging props
5. Base classes for testing and other utility needs
6. Code style accordingly google-formatter-plugin

dumper-integration-tests/redshift-tests/build.gradle

@@ -0,0 +1,49 @@
+buildscript {
+    repositories {
+        mavenCentral()
+    }
+    dependencies {
+        classpath 'com.google.protobuf:protobuf-gradle-plugin:0.8.18'
+    }
+}
+
+plugins {
+    id 'java'
+    id "com.google.protobuf" version "0.8.18"
+}
+
+sourceCompatibility = 1.8
+
+repositories {
+    mavenCentral()
+    maven {
+        url "https://redshift-maven-repository.s3-website-us-east-1.amazonaws.com/release"
+    }
+}
+
+dependencies {
+    implementation 'org.codehaus.groovy:groovy-all:3.0.5'
+    implementation 'org.testng:testng:7.5'
+    implementation 'org.slf4j:slf4j-api:2.0.0-alpha5'
+    implementation 'org.slf4j:slf4j-jdk14:2.0.0-alpha5'
+    implementation 'com.amazon.redshift:redshift-jdbc42:2.1.0.6'
+    implementation 'commons-io:commons-io:2.11.0'
+    implementation 'org.apache.commons:commons-collections4:4.4'
+    implementation 'com.google.guava:guava:31.0.1-jre'
+    implementation 'com.opencsv:opencsv:5.6'
+    implementation 'com.google.protobuf:protobuf-gradle-plugin:0.8.18'
+    implementation 'com.google.protobuf:protobuf-java:3.20.1'
+}
+
+test {
+    useTestNG {
+        preserveOrder true
+        systemProperty 'java.util.logging.config.file', 'src/main/resources/logging.properties'
+    }
+}
+
+protobuf {
+    protoc {
+        artifact = 'com.google.protobuf:protoc:3.20.1'
+    }
+}

dumper-integration-tests/redshift-tests/src/main/java/com/google/base/TestBase.java

@@ -0,0 +1,59 @@
+/*
+ * Copyright 2022 Google LLC
+ * Copyright 2013-2021 CompilerWorks
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.base;
+
+import static java.lang.String.format;
+import static java.lang.System.lineSeparator;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.LinkedHashMultiset;
+import org.junit.Assert;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Base class with general values for all Junit test suites
+ */
+public abstract class TestBase {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(TestBase.class);
+
+  /**
+   * @param dbList List of extracted from DB items
+   * @param outputList List of uploaded from Avro items
+   */
+  public static void assertListsEqual(final LinkedHashMultiset dbList,
+      final LinkedHashMultiset outputList) {
+    String dbListOutputForLogs = lineSeparator() + Joiner.on("").join(dbList);
+    String outputListForLogs = lineSeparator() + Joiner.on("").join(outputList);
+
+    if (dbList.isEmpty() && outputList.isEmpty()) {
+      LOGGER.info("DB view and Output file are equal");
+    } else if (!dbList.isEmpty() && !outputList.isEmpty()) {
+      Assert.fail(format("DB view and Output file have mutually exclusive row(s)%n"
+              + "DB view '%s' has %d different row(s): %s%n"
+              + "Output file %s has %d different row(s): %s", dbList.size(), dbListOutputForLogs,
+          outputList.size(), outputListForLogs));
+    } else if (!dbList.isEmpty()) {
+      Assert.fail(
+          format("DB view '%s' has %d extra row(s):%n%s", dbList.size(), dbListOutputForLogs));
+    } else if (!outputList.isEmpty()) {
+      Assert.fail(
+          format("Output file %s has %d extra row(s):%n%s", outputList.size(), outputListForLogs));
+    }
+  }
+}

dumper-integration-tests/redshift-tests/src/main/java/com/google/base/TestConstants.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2022 Google LLC
+ * Copyright 2013-2021 CompilerWorks
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.base;
+
+import static java.lang.System.getenv;
+
+import java.util.regex.Pattern;
+
+/**
+ * Stores constants common among all tests.
+ */
+public final class TestConstants {
+
+  public static final String URL_DB = getenv("DB_URL");
+  public static final String USERNAME_DB = getenv("USERNAME");
+  public static final String PASSWORD_DB = getenv("PASSWORD");
+
+  public static final String ET_OUTPUT_PATH = getenv("EXPORT_PATH");
+  public static final Pattern TRAILING_SPACES_REGEX = Pattern.compile("\\s+$");
+
+  private TestConstants() {
+  }
+}

dumper-integration-tests/redshift-tests/src/main/java/com/google/sql/SqlUtil.java

@@ -0,0 +1,88 @@
+/*
+ * Copyright 2022 Google LLC
+ * Copyright 2013-2021 CompilerWorks
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.sql;
+
+import static com.google.base.TestConstants.URL_DB;
+import static java.lang.String.format;
+import static org.apache.commons.io.FileUtils.readFileToString;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.List;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A helper class for reading .sql files.
+ */
+public final class SqlUtil {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(SqlUtil.class);
+
+  private SqlUtil() {
+  }
+
+  /**
+   * @param sqlPath Path to an .sql file.
+   * @return File contents, never null.
+   */
+  public static String getSql(String sqlPath) {
+    try {
+      return readFileToString(new File(sqlPath), StandardCharsets.UTF_8);
+    } catch (IOException exception) {
+      throw new UncheckedIOException(format("Error while reading sql file %s", sqlPath), exception);
+    }
+  }
+
+  /**
+   * @param connection DB connection parameter
+   * @param queries List of strings each of the contains a parametrized SQL request
+   */
+  public static void executeQueries(Connection connection, List<String> queries)
+      throws SQLException {
+    for (String query : queries) {
+      try (PreparedStatement preparedStatement = connection.prepareStatement(query)) {
+        preparedStatement.execute();
+      } catch (SQLException e) {
+        LOGGER.error(format("Cannot execute query: %n%s%n", query));
+        throw e;
+      }
+    }
+  }
+
+  /**
+   * @param username DB username
+   * @param password DB password
+   * @param query A single string of a parametrized SQL request
+   */
+  public static void connectAndExecuteQueryAsUser(String username, String password, String query)
+      throws SQLException {
+    Connection connection = DriverManager.getConnection(URL_DB, username, password);
+    try (PreparedStatement preparedStatement = connection.prepareStatement(query)) {
+      preparedStatement.execute();
+    } catch (SQLException e) {
+      LOGGER.error(format("Cannot execute query: %n%s%n", query));
+      throw e;
+    }
+  }
+}

dumper-integration-tests/redshift-tests/src/main/resources/logging.properties

@@ -0,0 +1,21 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+handlers=java.util.logging.ConsoleHandler
+Run.useParentHandlers = false
+Run.handlers = java.util.logging.ConsoleHandler
+
+java.util.logging.ConsoleHandler.level=INFO
+java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter
+java.util.logging.SimpleFormatter.format=[%1$tF %1$tT] [%4$s] %5$s %n
+.level=INFO

dumper-integration-tests/redshift-tests/src/test/java/com/google/integration/RedshiftTest.java

@@ -0,0 +1,80 @@
+/*
+ * Copyright 2022 Google LLC
+ * Copyright 2013-2021 CompilerWorks
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.integration;
+
+import static com.google.base.TestBase.assertListsEqual;
+import static com.google.base.TestConstants.PASSWORD_DB;
+import static com.google.base.TestConstants.URL_DB;
+import static com.google.base.TestConstants.USERNAME_DB;
+
+import com.google.common.collect.LinkedHashMultiset;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+public class RedshiftTest {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(RedshiftTest.class);
+  private static Connection connection;
+
+  @BeforeClass
+  public static void beforeClass() throws SQLException {
+    connection = DriverManager.getConnection(URL_DB, USERNAME_DB, PASSWORD_DB);
+  }
+
+  @Test
+  public void templateTest() throws SQLException {
+    String sql = "select * from customers;";
+
+    LinkedHashMultiset<String> dbList = LinkedHashMultiset.create();
+    LinkedHashMultiset<String> outputList = LinkedHashMultiset.create();
+
+    //Extract from RedShift
+    try (PreparedStatement preparedStatement = connection.prepareStatement(sql)) {
+      ResultSet rs = preparedStatement.executeQuery();
+
+      while (rs.next()) {
+        String customernumber = rs.getString("customernumber");
+        String customername = rs.getString("customername");
+        String phonenumber = rs.getString("phonenumber");
+        String postalcode = rs.getString("postalcode");
+        String locale = rs.getString("locale");
+        String datecreated = rs.getString("datecreated");
+        String email = rs.getString("email");
+
+        LOGGER.info(String.format("%s, %s, %s, %s, %s, %s, %s", customernumber.trim(), customername,
+            phonenumber, postalcode, locale, datecreated, email));
+      }
+    }
+
+    //Extract from output files
+    // TODO
+
+    //Reduce and compare
+    LinkedHashMultiset<String> dbListCopy = LinkedHashMultiset.create(dbList);
+    outputList.forEach(dbList::remove);
+    dbListCopy.forEach(outputList::remove);
+
+    assertListsEqual(dbList, outputList);
+  }
+}

dumper/settings.gradle

@@ -0,0 +1,10 @@
+rootProject.name = 'dwh-migration-dumper'
+include(
+	'lib-common',
+	'lib-dumper-spi',
+	'lib-ext-bigquery',
+	'lib-ext-hive-metastore',
+	'app'
+)
+includeFlat 'integration-tests'
+