diff --git a/dumper-integration-tests/redshift-tests/build.gradle b/dumper-integration-tests/redshift-tests/build.gradle new file mode 100644 index 000000000..4f3dd5975 --- /dev/null +++ b/dumper-integration-tests/redshift-tests/build.gradle @@ -0,0 +1,49 @@ +buildscript { + repositories { + mavenCentral() + } + dependencies { + classpath 'com.google.protobuf:protobuf-gradle-plugin:0.8.18' + } +} + +plugins { + id 'java' + id "com.google.protobuf" version "0.8.18" +} + +sourceCompatibility = 1.8 + +repositories { + mavenCentral() + maven { + url "https://redshift-maven-repository.s3-website-us-east-1.amazonaws.com/release" + } +} + +dependencies { + implementation 'org.codehaus.groovy:groovy-all:3.0.5' + implementation 'org.testng:testng:7.5' + implementation 'org.slf4j:slf4j-api:2.0.0-alpha5' + implementation 'org.slf4j:slf4j-jdk14:2.0.0-alpha5' + implementation 'com.amazon.redshift:redshift-jdbc42:2.1.0.6' + implementation 'commons-io:commons-io:2.11.0' + implementation 'org.apache.commons:commons-collections4:4.4' + implementation 'com.google.guava:guava:31.0.1-jre' + implementation 'com.opencsv:opencsv:5.6' + implementation 'com.google.protobuf:protobuf-gradle-plugin:0.8.18' + implementation 'com.google.protobuf:protobuf-java:3.20.1' +} + +test { + useTestNG { + preserveOrder true + systemProperty 'java.util.logging.config.file', 'src/main/resources/logging.properties' + } +} + +protobuf { + protoc { + artifact = 'com.google.protobuf:protoc:3.20.1' + } +} \ No newline at end of file diff --git a/dumper-integration-tests/redshift-tests/src/main/java/com/google/base/TestBase.java b/dumper-integration-tests/redshift-tests/src/main/java/com/google/base/TestBase.java new file mode 100644 index 000000000..201a05f51 --- /dev/null +++ b/dumper-integration-tests/redshift-tests/src/main/java/com/google/base/TestBase.java @@ -0,0 +1,59 @@ +/* + * Copyright 2022 Google LLC + * Copyright 2013-2021 CompilerWorks + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.base; + +import static java.lang.String.format; +import static java.lang.System.lineSeparator; + +import com.google.common.base.Joiner; +import com.google.common.collect.LinkedHashMultiset; +import org.junit.Assert; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Base class with general values for all Junit test suites + */ +public abstract class TestBase { + + private static final Logger LOGGER = LoggerFactory.getLogger(TestBase.class); + + /** + * @param dbList List of extracted from DB items + * @param outputList List of uploaded from Avro items + */ + public static void assertListsEqual(final LinkedHashMultiset dbList, + final LinkedHashMultiset outputList) { + String dbListOutputForLogs = lineSeparator() + Joiner.on("").join(dbList); + String outputListForLogs = lineSeparator() + Joiner.on("").join(outputList); + + if (dbList.isEmpty() && outputList.isEmpty()) { + LOGGER.info("DB view and Output file are equal"); + } else if (!dbList.isEmpty() && !outputList.isEmpty()) { + Assert.fail(format("DB view and Output file have mutually exclusive row(s)%n" + + "DB view '%s' has %d different row(s): %s%n" + + "Output file %s has %d different row(s): %s", dbList.size(), dbListOutputForLogs, + outputList.size(), outputListForLogs)); + } else if (!dbList.isEmpty()) { + Assert.fail( + format("DB view '%s' has %d extra row(s):%n%s", dbList.size(), dbListOutputForLogs)); + } else if (!outputList.isEmpty()) { + Assert.fail( + format("Output file %s has %d extra row(s):%n%s", outputList.size(), outputListForLogs)); + } + } +} diff --git a/dumper-integration-tests/redshift-tests/src/main/java/com/google/base/TestConstants.java b/dumper-integration-tests/redshift-tests/src/main/java/com/google/base/TestConstants.java new file mode 100644 index 000000000..16d966dfd --- /dev/null +++ b/dumper-integration-tests/redshift-tests/src/main/java/com/google/base/TestConstants.java @@ -0,0 +1,37 @@ +/* + * Copyright 2022 Google LLC + * Copyright 2013-2021 CompilerWorks + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.base; + +import static java.lang.System.getenv; + +import java.util.regex.Pattern; + +/** + * Stores constants common among all tests. + */ +public final class TestConstants { + + public static final String URL_DB = getenv("DB_URL"); + public static final String USERNAME_DB = getenv("USERNAME"); + public static final String PASSWORD_DB = getenv("PASSWORD"); + + public static final String ET_OUTPUT_PATH = getenv("EXPORT_PATH"); + public static final Pattern TRAILING_SPACES_REGEX = Pattern.compile("\\s+$"); + + private TestConstants() { + } +} diff --git a/dumper-integration-tests/redshift-tests/src/main/java/com/google/sql/SqlUtil.java b/dumper-integration-tests/redshift-tests/src/main/java/com/google/sql/SqlUtil.java new file mode 100644 index 000000000..fdc8daae6 --- /dev/null +++ b/dumper-integration-tests/redshift-tests/src/main/java/com/google/sql/SqlUtil.java @@ -0,0 +1,88 @@ +/* + * Copyright 2022 Google LLC + * Copyright 2013-2021 CompilerWorks + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.sql; + +import static com.google.base.TestConstants.URL_DB; +import static java.lang.String.format; +import static org.apache.commons.io.FileUtils.readFileToString; + +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A helper class for reading .sql files. + */ +public final class SqlUtil { + + private static final Logger LOGGER = LoggerFactory.getLogger(SqlUtil.class); + + private SqlUtil() { + } + + /** + * @param sqlPath Path to an .sql file. + * @return File contents, never null. + */ + public static String getSql(String sqlPath) { + try { + return readFileToString(new File(sqlPath), StandardCharsets.UTF_8); + } catch (IOException exception) { + throw new UncheckedIOException(format("Error while reading sql file %s", sqlPath), exception); + } + } + + /** + * @param connection DB connection parameter + * @param queries List of strings each of the contains a parametrized SQL request + */ + public static void executeQueries(Connection connection, List queries) + throws SQLException { + for (String query : queries) { + try (PreparedStatement preparedStatement = connection.prepareStatement(query)) { + preparedStatement.execute(); + } catch (SQLException e) { + LOGGER.error(format("Cannot execute query: %n%s%n", query)); + throw e; + } + } + } + + /** + * @param username DB username + * @param password DB password + * @param query A single string of a parametrized SQL request + */ + public static void connectAndExecuteQueryAsUser(String username, String password, String query) + throws SQLException { + Connection connection = DriverManager.getConnection(URL_DB, username, password); + try (PreparedStatement preparedStatement = connection.prepareStatement(query)) { + preparedStatement.execute(); + } catch (SQLException e) { + LOGGER.error(format("Cannot execute query: %n%s%n", query)); + throw e; + } + } +} diff --git a/dumper-integration-tests/redshift-tests/src/main/resources/logging.properties b/dumper-integration-tests/redshift-tests/src/main/resources/logging.properties new file mode 100644 index 000000000..763182384 --- /dev/null +++ b/dumper-integration-tests/redshift-tests/src/main/resources/logging.properties @@ -0,0 +1,21 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +handlers=java.util.logging.ConsoleHandler +Run.useParentHandlers = false +Run.handlers = java.util.logging.ConsoleHandler + +java.util.logging.ConsoleHandler.level=INFO +java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter +java.util.logging.SimpleFormatter.format=[%1$tF %1$tT] [%4$s] %5$s %n +.level=INFO \ No newline at end of file diff --git a/dumper-integration-tests/redshift-tests/src/test/java/com/google/integration/RedshiftTest.java b/dumper-integration-tests/redshift-tests/src/test/java/com/google/integration/RedshiftTest.java new file mode 100644 index 000000000..d3cb84edb --- /dev/null +++ b/dumper-integration-tests/redshift-tests/src/test/java/com/google/integration/RedshiftTest.java @@ -0,0 +1,80 @@ +/* + * Copyright 2022 Google LLC + * Copyright 2013-2021 CompilerWorks + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.integration; + +import static com.google.base.TestBase.assertListsEqual; +import static com.google.base.TestConstants.PASSWORD_DB; +import static com.google.base.TestConstants.URL_DB; +import static com.google.base.TestConstants.USERNAME_DB; + +import com.google.common.collect.LinkedHashMultiset; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +public class RedshiftTest { + + private static final Logger LOGGER = LoggerFactory.getLogger(RedshiftTest.class); + private static Connection connection; + + @BeforeClass + public static void beforeClass() throws SQLException { + connection = DriverManager.getConnection(URL_DB, USERNAME_DB, PASSWORD_DB); + } + + @Test + public void templateTest() throws SQLException { + String sql = "select * from customers;"; + + LinkedHashMultiset dbList = LinkedHashMultiset.create(); + LinkedHashMultiset outputList = LinkedHashMultiset.create(); + + //Extract from RedShift + try (PreparedStatement preparedStatement = connection.prepareStatement(sql)) { + ResultSet rs = preparedStatement.executeQuery(); + + while (rs.next()) { + String customernumber = rs.getString("customernumber"); + String customername = rs.getString("customername"); + String phonenumber = rs.getString("phonenumber"); + String postalcode = rs.getString("postalcode"); + String locale = rs.getString("locale"); + String datecreated = rs.getString("datecreated"); + String email = rs.getString("email"); + + LOGGER.info(String.format("%s, %s, %s, %s, %s, %s, %s", customernumber.trim(), customername, + phonenumber, postalcode, locale, datecreated, email)); + } + } + + //Extract from output files + // TODO + + //Reduce and compare + LinkedHashMultiset dbListCopy = LinkedHashMultiset.create(dbList); + outputList.forEach(dbList::remove); + dbListCopy.forEach(outputList::remove); + + assertListsEqual(dbList, outputList); + } +} diff --git a/dumper/settings.gradle b/dumper/settings.gradle new file mode 100644 index 000000000..acd15c588 --- /dev/null +++ b/dumper/settings.gradle @@ -0,0 +1,10 @@ +rootProject.name = 'dwh-migration-dumper' +include( + 'lib-common', + 'lib-dumper-spi', + 'lib-ext-bigquery', + 'lib-ext-hive-metastore', + 'app' +) +includeFlat 'integration-tests' +