From a99dd62d02c7ef59f00c9ee988b16a5f234ebf76 Mon Sep 17 00:00:00 2001 From: Danila Morokov Date: Fri, 13 May 2022 21:36:15 +0000 Subject: [PATCH] Initial commit 1. Test Automation Framework skeleton 2. Gradle config with most likely deps 3. Template Test 4. Logging props 5. Base classes for testing and other utility needs 6. Code style accordingly google-formatter-plugin --- .../redshift-tests/build.gradle | 49 +++++++++++ .../main/java/com/google/base/TestBase.java | 59 +++++++++++++ .../java/com/google/base/TestConstants.java | 37 ++++++++ .../src/main/java/com/google/sql/SqlUtil.java | 88 +++++++++++++++++++ .../src/main/resources/logging.properties | 21 +++++ .../com/google/integration/RedshiftTest.java | 80 +++++++++++++++++ dumper/settings.gradle | 10 +++ 7 files changed, 344 insertions(+) create mode 100644 dumper-integration-tests/redshift-tests/build.gradle create mode 100644 dumper-integration-tests/redshift-tests/src/main/java/com/google/base/TestBase.java create mode 100644 dumper-integration-tests/redshift-tests/src/main/java/com/google/base/TestConstants.java create mode 100644 dumper-integration-tests/redshift-tests/src/main/java/com/google/sql/SqlUtil.java create mode 100644 dumper-integration-tests/redshift-tests/src/main/resources/logging.properties create mode 100644 dumper-integration-tests/redshift-tests/src/test/java/com/google/integration/RedshiftTest.java create mode 100644 dumper/settings.gradle diff --git a/dumper-integration-tests/redshift-tests/build.gradle b/dumper-integration-tests/redshift-tests/build.gradle new file mode 100644 index 000000000..4f3dd5975 --- /dev/null +++ b/dumper-integration-tests/redshift-tests/build.gradle @@ -0,0 +1,49 @@ +buildscript { + repositories { + mavenCentral() + } + dependencies { + classpath 'com.google.protobuf:protobuf-gradle-plugin:0.8.18' + } +} + +plugins { + id 'java' + id "com.google.protobuf" version "0.8.18" +} + +sourceCompatibility = 1.8 + +repositories { + mavenCentral() + maven { + url "https://redshift-maven-repository.s3-website-us-east-1.amazonaws.com/release" + } +} + +dependencies { + implementation 'org.codehaus.groovy:groovy-all:3.0.5' + implementation 'org.testng:testng:7.5' + implementation 'org.slf4j:slf4j-api:2.0.0-alpha5' + implementation 'org.slf4j:slf4j-jdk14:2.0.0-alpha5' + implementation 'com.amazon.redshift:redshift-jdbc42:2.1.0.6' + implementation 'commons-io:commons-io:2.11.0' + implementation 'org.apache.commons:commons-collections4:4.4' + implementation 'com.google.guava:guava:31.0.1-jre' + implementation 'com.opencsv:opencsv:5.6' + implementation 'com.google.protobuf:protobuf-gradle-plugin:0.8.18' + implementation 'com.google.protobuf:protobuf-java:3.20.1' +} + +test { + useTestNG { + preserveOrder true + systemProperty 'java.util.logging.config.file', 'src/main/resources/logging.properties' + } +} + +protobuf { + protoc { + artifact = 'com.google.protobuf:protoc:3.20.1' + } +} \ No newline at end of file diff --git a/dumper-integration-tests/redshift-tests/src/main/java/com/google/base/TestBase.java b/dumper-integration-tests/redshift-tests/src/main/java/com/google/base/TestBase.java new file mode 100644 index 000000000..201a05f51 --- /dev/null +++ b/dumper-integration-tests/redshift-tests/src/main/java/com/google/base/TestBase.java @@ -0,0 +1,59 @@ +/* + * Copyright 2022 Google LLC + * Copyright 2013-2021 CompilerWorks + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.base; + +import static java.lang.String.format; +import static java.lang.System.lineSeparator; + +import com.google.common.base.Joiner; +import com.google.common.collect.LinkedHashMultiset; +import org.junit.Assert; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Base class with general values for all Junit test suites + */ +public abstract class TestBase { + + private static final Logger LOGGER = LoggerFactory.getLogger(TestBase.class); + + /** + * @param dbList List of extracted from DB items + * @param outputList List of uploaded from Avro items + */ + public static void assertListsEqual(final LinkedHashMultiset dbList, + final LinkedHashMultiset outputList) { + String dbListOutputForLogs = lineSeparator() + Joiner.on("").join(dbList); + String outputListForLogs = lineSeparator() + Joiner.on("").join(outputList); + + if (dbList.isEmpty() && outputList.isEmpty()) { + LOGGER.info("DB view and Output file are equal"); + } else if (!dbList.isEmpty() && !outputList.isEmpty()) { + Assert.fail(format("DB view and Output file have mutually exclusive row(s)%n" + + "DB view '%s' has %d different row(s): %s%n" + + "Output file %s has %d different row(s): %s", dbList.size(), dbListOutputForLogs, + outputList.size(), outputListForLogs)); + } else if (!dbList.isEmpty()) { + Assert.fail( + format("DB view '%s' has %d extra row(s):%n%s", dbList.size(), dbListOutputForLogs)); + } else if (!outputList.isEmpty()) { + Assert.fail( + format("Output file %s has %d extra row(s):%n%s", outputList.size(), outputListForLogs)); + } + } +} diff --git a/dumper-integration-tests/redshift-tests/src/main/java/com/google/base/TestConstants.java b/dumper-integration-tests/redshift-tests/src/main/java/com/google/base/TestConstants.java new file mode 100644 index 000000000..16d966dfd --- /dev/null +++ b/dumper-integration-tests/redshift-tests/src/main/java/com/google/base/TestConstants.java @@ -0,0 +1,37 @@ +/* + * Copyright 2022 Google LLC + * Copyright 2013-2021 CompilerWorks + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.base; + +import static java.lang.System.getenv; + +import java.util.regex.Pattern; + +/** + * Stores constants common among all tests. + */ +public final class TestConstants { + + public static final String URL_DB = getenv("DB_URL"); + public static final String USERNAME_DB = getenv("USERNAME"); + public static final String PASSWORD_DB = getenv("PASSWORD"); + + public static final String ET_OUTPUT_PATH = getenv("EXPORT_PATH"); + public static final Pattern TRAILING_SPACES_REGEX = Pattern.compile("\\s+$"); + + private TestConstants() { + } +} diff --git a/dumper-integration-tests/redshift-tests/src/main/java/com/google/sql/SqlUtil.java b/dumper-integration-tests/redshift-tests/src/main/java/com/google/sql/SqlUtil.java new file mode 100644 index 000000000..fdc8daae6 --- /dev/null +++ b/dumper-integration-tests/redshift-tests/src/main/java/com/google/sql/SqlUtil.java @@ -0,0 +1,88 @@ +/* + * Copyright 2022 Google LLC + * Copyright 2013-2021 CompilerWorks + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.sql; + +import static com.google.base.TestConstants.URL_DB; +import static java.lang.String.format; +import static org.apache.commons.io.FileUtils.readFileToString; + +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A helper class for reading .sql files. + */ +public final class SqlUtil { + + private static final Logger LOGGER = LoggerFactory.getLogger(SqlUtil.class); + + private SqlUtil() { + } + + /** + * @param sqlPath Path to an .sql file. + * @return File contents, never null. + */ + public static String getSql(String sqlPath) { + try { + return readFileToString(new File(sqlPath), StandardCharsets.UTF_8); + } catch (IOException exception) { + throw new UncheckedIOException(format("Error while reading sql file %s", sqlPath), exception); + } + } + + /** + * @param connection DB connection parameter + * @param queries List of strings each of the contains a parametrized SQL request + */ + public static void executeQueries(Connection connection, List queries) + throws SQLException { + for (String query : queries) { + try (PreparedStatement preparedStatement = connection.prepareStatement(query)) { + preparedStatement.execute(); + } catch (SQLException e) { + LOGGER.error(format("Cannot execute query: %n%s%n", query)); + throw e; + } + } + } + + /** + * @param username DB username + * @param password DB password + * @param query A single string of a parametrized SQL request + */ + public static void connectAndExecuteQueryAsUser(String username, String password, String query) + throws SQLException { + Connection connection = DriverManager.getConnection(URL_DB, username, password); + try (PreparedStatement preparedStatement = connection.prepareStatement(query)) { + preparedStatement.execute(); + } catch (SQLException e) { + LOGGER.error(format("Cannot execute query: %n%s%n", query)); + throw e; + } + } +} diff --git a/dumper-integration-tests/redshift-tests/src/main/resources/logging.properties b/dumper-integration-tests/redshift-tests/src/main/resources/logging.properties new file mode 100644 index 000000000..763182384 --- /dev/null +++ b/dumper-integration-tests/redshift-tests/src/main/resources/logging.properties @@ -0,0 +1,21 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +handlers=java.util.logging.ConsoleHandler +Run.useParentHandlers = false +Run.handlers = java.util.logging.ConsoleHandler + +java.util.logging.ConsoleHandler.level=INFO +java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter +java.util.logging.SimpleFormatter.format=[%1$tF %1$tT] [%4$s] %5$s %n +.level=INFO \ No newline at end of file diff --git a/dumper-integration-tests/redshift-tests/src/test/java/com/google/integration/RedshiftTest.java b/dumper-integration-tests/redshift-tests/src/test/java/com/google/integration/RedshiftTest.java new file mode 100644 index 000000000..d3cb84edb --- /dev/null +++ b/dumper-integration-tests/redshift-tests/src/test/java/com/google/integration/RedshiftTest.java @@ -0,0 +1,80 @@ +/* + * Copyright 2022 Google LLC + * Copyright 2013-2021 CompilerWorks + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.integration; + +import static com.google.base.TestBase.assertListsEqual; +import static com.google.base.TestConstants.PASSWORD_DB; +import static com.google.base.TestConstants.URL_DB; +import static com.google.base.TestConstants.USERNAME_DB; + +import com.google.common.collect.LinkedHashMultiset; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +public class RedshiftTest { + + private static final Logger LOGGER = LoggerFactory.getLogger(RedshiftTest.class); + private static Connection connection; + + @BeforeClass + public static void beforeClass() throws SQLException { + connection = DriverManager.getConnection(URL_DB, USERNAME_DB, PASSWORD_DB); + } + + @Test + public void templateTest() throws SQLException { + String sql = "select * from customers;"; + + LinkedHashMultiset dbList = LinkedHashMultiset.create(); + LinkedHashMultiset outputList = LinkedHashMultiset.create(); + + //Extract from RedShift + try (PreparedStatement preparedStatement = connection.prepareStatement(sql)) { + ResultSet rs = preparedStatement.executeQuery(); + + while (rs.next()) { + String customernumber = rs.getString("customernumber"); + String customername = rs.getString("customername"); + String phonenumber = rs.getString("phonenumber"); + String postalcode = rs.getString("postalcode"); + String locale = rs.getString("locale"); + String datecreated = rs.getString("datecreated"); + String email = rs.getString("email"); + + LOGGER.info(String.format("%s, %s, %s, %s, %s, %s, %s", customernumber.trim(), customername, + phonenumber, postalcode, locale, datecreated, email)); + } + } + + //Extract from output files + // TODO + + //Reduce and compare + LinkedHashMultiset dbListCopy = LinkedHashMultiset.create(dbList); + outputList.forEach(dbList::remove); + dbListCopy.forEach(outputList::remove); + + assertListsEqual(dbList, outputList); + } +} diff --git a/dumper/settings.gradle b/dumper/settings.gradle new file mode 100644 index 000000000..acd15c588 --- /dev/null +++ b/dumper/settings.gradle @@ -0,0 +1,10 @@ +rootProject.name = 'dwh-migration-dumper' +include( + 'lib-common', + 'lib-dumper-spi', + 'lib-ext-bigquery', + 'lib-ext-hive-metastore', + 'app' +) +includeFlat 'integration-tests' +