diff --git a/dumper-integration-tests/redshift-tests/build.gradle b/dumper-integration-tests/redshift-tests/build.gradle index 4f3dd5975..5804241ff 100644 --- a/dumper-integration-tests/redshift-tests/build.gradle +++ b/dumper-integration-tests/redshift-tests/build.gradle @@ -2,14 +2,11 @@ buildscript { repositories { mavenCentral() } - dependencies { - classpath 'com.google.protobuf:protobuf-gradle-plugin:0.8.18' - } } plugins { id 'java' - id "com.google.protobuf" version "0.8.18" + id 'com.adarshr.test-logger' version '3.2.0' } sourceCompatibility = 1.8 @@ -22,28 +19,24 @@ repositories { } dependencies { - implementation 'org.codehaus.groovy:groovy-all:3.0.5' + implementation 'org.codehaus.groovy:groovy-all:3.0.10' + implementation 'com.google.guava:guava:31.1-jre' implementation 'org.testng:testng:7.5' implementation 'org.slf4j:slf4j-api:2.0.0-alpha5' implementation 'org.slf4j:slf4j-jdk14:2.0.0-alpha5' - implementation 'com.amazon.redshift:redshift-jdbc42:2.1.0.6' + implementation 'com.amazon.redshift:redshift-jdbc42:2.1.0.7' implementation 'commons-io:commons-io:2.11.0' implementation 'org.apache.commons:commons-collections4:4.4' - implementation 'com.google.guava:guava:31.0.1-jre' implementation 'com.opencsv:opencsv:5.6' - implementation 'com.google.protobuf:protobuf-gradle-plugin:0.8.18' - implementation 'com.google.protobuf:protobuf-java:3.20.1' + annotationProcessor 'com.google.auto.value:auto-value:1.9' + compileOnly 'com.google.auto.value:auto-value-annotations:1.9' } test { + ignoreFailures = true + useTestNG { preserveOrder true systemProperty 'java.util.logging.config.file', 'src/main/resources/logging.properties' } -} - -protobuf { - protoc { - artifact = 'com.google.protobuf:protoc:3.20.1' - } } \ No newline at end of file diff --git a/dumper-integration-tests/redshift-tests/src/main/java/com/google/base/TestBase.java b/dumper-integration-tests/redshift-tests/src/main/java/com/google/base/TestBase.java deleted file mode 100644 index 201a05f51..000000000 --- a/dumper-integration-tests/redshift-tests/src/main/java/com/google/base/TestBase.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 2022 Google LLC - * Copyright 2013-2021 CompilerWorks - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.base; - -import static java.lang.String.format; -import static java.lang.System.lineSeparator; - -import com.google.common.base.Joiner; -import com.google.common.collect.LinkedHashMultiset; -import org.junit.Assert; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Base class with general values for all Junit test suites - */ -public abstract class TestBase { - - private static final Logger LOGGER = LoggerFactory.getLogger(TestBase.class); - - /** - * @param dbList List of extracted from DB items - * @param outputList List of uploaded from Avro items - */ - public static void assertListsEqual(final LinkedHashMultiset dbList, - final LinkedHashMultiset outputList) { - String dbListOutputForLogs = lineSeparator() + Joiner.on("").join(dbList); - String outputListForLogs = lineSeparator() + Joiner.on("").join(outputList); - - if (dbList.isEmpty() && outputList.isEmpty()) { - LOGGER.info("DB view and Output file are equal"); - } else if (!dbList.isEmpty() && !outputList.isEmpty()) { - Assert.fail(format("DB view and Output file have mutually exclusive row(s)%n" - + "DB view '%s' has %d different row(s): %s%n" - + "Output file %s has %d different row(s): %s", dbList.size(), dbListOutputForLogs, - outputList.size(), outputListForLogs)); - } else if (!dbList.isEmpty()) { - Assert.fail( - format("DB view '%s' has %d extra row(s):%n%s", dbList.size(), dbListOutputForLogs)); - } else if (!outputList.isEmpty()) { - Assert.fail( - format("Output file %s has %d extra row(s):%n%s", outputList.size(), outputListForLogs)); - } - } -} diff --git a/dumper-integration-tests/redshift-tests/src/main/java/com/google/edwmigration/dumper/base/TestBase.java b/dumper-integration-tests/redshift-tests/src/main/java/com/google/edwmigration/dumper/base/TestBase.java new file mode 100644 index 000000000..198183ef0 --- /dev/null +++ b/dumper-integration-tests/redshift-tests/src/main/java/com/google/edwmigration/dumper/base/TestBase.java @@ -0,0 +1,56 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.edwmigration.dumper.base; + +import static java.lang.String.format; +import static java.lang.System.lineSeparator; + +import com.google.common.base.Joiner; +import com.google.common.collect.LinkedHashMultiset; +import org.junit.Assert; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Base class with general values for all TestNG test suites */ +public abstract class TestBase { + + private static final Logger LOGGER = LoggerFactory.getLogger(TestBase.class); + + /** + * @param dbList List of extracted from DB items + * @param csvList List of uploaded from Avro items + */ + public static void assertListsEqual( + final LinkedHashMultiset dbList, final LinkedHashMultiset csvList) { + String dbListForLogs = lineSeparator() + Joiner.on("").join(dbList); + String csvListForLogs = lineSeparator() + Joiner.on("").join(csvList); + + if (dbList.isEmpty() && csvList.isEmpty()) { + LOGGER.info("DB view and CSV file are equal"); + } else if (!dbList.isEmpty() && !csvList.isEmpty()) { + Assert.fail( + format( + "DB view and CSV file have mutually exclusive row(s)%n" + + "DB view has %d different row(s): %s%n" + + "CSV file has %d different row(s): %s", + dbList.size(), dbListForLogs, csvList.size(), csvListForLogs)); + } else if (!dbList.isEmpty()) { + Assert.fail(format("DB view has %d extra row(s):%n%s", dbList.size(), dbListForLogs)); + } else if (!csvList.isEmpty()) { + Assert.fail(format("CSV file has %d extra row(s):%n%s", csvList.size(), csvListForLogs)); + } + } +} diff --git a/dumper-integration-tests/redshift-tests/src/main/java/com/google/base/TestConstants.java b/dumper-integration-tests/redshift-tests/src/main/java/com/google/edwmigration/dumper/base/TestConstants.java similarity index 77% rename from dumper-integration-tests/redshift-tests/src/main/java/com/google/base/TestConstants.java rename to dumper-integration-tests/redshift-tests/src/main/java/com/google/edwmigration/dumper/base/TestConstants.java index 16d966dfd..09135b177 100644 --- a/dumper-integration-tests/redshift-tests/src/main/java/com/google/base/TestConstants.java +++ b/dumper-integration-tests/redshift-tests/src/main/java/com/google/edwmigration/dumper/base/TestConstants.java @@ -1,6 +1,5 @@ /* * Copyright 2022 Google LLC - * Copyright 2013-2021 CompilerWorks * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,24 +13,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.base; +package com.google.edwmigration.dumper.base; import static java.lang.System.getenv; import java.util.regex.Pattern; -/** - * Stores constants common among all tests. - */ +/** Stores constants common among all tests. */ public final class TestConstants { public static final String URL_DB = getenv("DB_URL"); public static final String USERNAME_DB = getenv("USERNAME"); public static final String PASSWORD_DB = getenv("PASSWORD"); - public static final String ET_OUTPUT_PATH = getenv("EXPORT_PATH"); + public static final String EXPORTED_FILES_BASE_PATH = getenv("EXPORT_PATH"); + + public static final String SQL_REQUESTS_BASE_PATH = "src/main/resources/sql/"; + public static final Pattern TRAILING_SPACES_REGEX = Pattern.compile("\\s+$"); - private TestConstants() { - } + private TestConstants() {} } diff --git a/dumper-integration-tests/redshift-tests/src/main/java/com/google/edwmigration/dumper/csv/CsvUtil.java b/dumper-integration-tests/redshift-tests/src/main/java/com/google/edwmigration/dumper/csv/CsvUtil.java new file mode 100644 index 000000000..8397a25c2 --- /dev/null +++ b/dumper-integration-tests/redshift-tests/src/main/java/com/google/edwmigration/dumper/csv/CsvUtil.java @@ -0,0 +1,39 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.edwmigration.dumper.csv; + +import static com.google.edwmigration.dumper.base.TestConstants.TRAILING_SPACES_REGEX; +import static java.lang.Integer.parseInt; + +/** A helper class for reading and extracting data from CSV files. */ +public final class CsvUtil { + + private CsvUtil() {} + + /** + * @return String or an empty string if null. + */ + public static String getStringNotNull(String value) { + return value == null ? "" : TRAILING_SPACES_REGEX.matcher(value).replaceFirst(""); + } + + /** + * @return int or 0 if "". + */ + public static int getIntNotNull(String value) { + return getStringNotNull(value).equals("") ? 0 : parseInt(value); + } +} diff --git a/dumper-integration-tests/redshift-tests/src/main/java/com/google/edwmigration/dumper/jdbc/JdbcUtil.java b/dumper-integration-tests/redshift-tests/src/main/java/com/google/edwmigration/dumper/jdbc/JdbcUtil.java new file mode 100644 index 000000000..d265845fd --- /dev/null +++ b/dumper-integration-tests/redshift-tests/src/main/java/com/google/edwmigration/dumper/jdbc/JdbcUtil.java @@ -0,0 +1,105 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.edwmigration.dumper.jdbc; + +import static com.google.edwmigration.dumper.base.TestConstants.TRAILING_SPACES_REGEX; + +import java.math.BigDecimal; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.time.ZonedDateTime; +import java.util.Calendar; +import java.util.TimeZone; + +/** + * A helper class for checking Null values returned by executing SELECT request against a database. + */ +public final class JdbcUtil { + + private JdbcUtil() {} + + /** + * @param rs A row with SELECT results. + * @param column Database column name. + * @return String or an empty string if null. + */ + public static String getStringNotNull(ResultSet rs, String column) throws SQLException { + String string = rs.getString(column); + return rs.wasNull() ? "" : TRAILING_SPACES_REGEX.matcher(string).replaceFirst(""); + } + + /** + * @param rs A row with SELECT results. + * @param column Database column name. + * @return int or 0 if null. + */ + public static int getIntNotNull(ResultSet rs, String column) throws SQLException { + int intValue = rs.getInt(column); + return rs.wasNull() ? 0 : intValue; + } + + /** + * @param rs A row with SELECT results. + * @param column Database column name. + * @return long or 0L if null. + */ + public static long getLongNotNull(ResultSet rs, String column) throws SQLException { + long longValue = rs.getLong(column); + return rs.wasNull() ? 0L : longValue; + } + + /** + * @param rs A row with SELECT results. + * @param column Database column name. + * @return byte[] or empty byte[] if null. + */ + public static byte[] getBytesNotNull(ResultSet rs, String column) throws SQLException { + try { + byte[] bytesValue = rs.getBytes(column); + return rs.wasNull() ? new byte[0] : bytesValue; + } catch (SQLException e) { + BigDecimal bigDecimal = rs.getBigDecimal(column); + return rs.wasNull() ? new byte[0] : bigDecimal.toBigInteger().toByteArray(); + } + } + + /** + * @param rs A row with SELECT results. + * @param column Database column name. + * @return double or 0.0 if null. + */ + public static double getDoubleNotNull(ResultSet rs, String column) throws SQLException { + double doubleValue = rs.getDouble(column); + return rs.wasNull() ? 0.0 : doubleValue; + } + + /** + * @param rs A row with SELECT results. + * @param column Database column name. + * @return long or 0L if null. + */ + public static long getTimestampNotNull(ResultSet rs, String column) throws SQLException { + Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("UTC")); + Timestamp timestamp = rs.getTimestamp(column, cal); + if (rs.wasNull()) { + return 0L; + } + return Timestamp.from( + ZonedDateTime.of(timestamp.toLocalDateTime(), cal.getTimeZone().toZoneId()).toInstant()) + .getTime(); + } +} diff --git a/dumper-integration-tests/redshift-tests/src/main/java/com/google/edwmigration/dumper/pojo/SvvColumnsRow.java b/dumper-integration-tests/redshift-tests/src/main/java/com/google/edwmigration/dumper/pojo/SvvColumnsRow.java new file mode 100644 index 000000000..94c4a7a1d --- /dev/null +++ b/dumper-integration-tests/redshift-tests/src/main/java/com/google/edwmigration/dumper/pojo/SvvColumnsRow.java @@ -0,0 +1,231 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.edwmigration.dumper.pojo; + +import static com.google.edwmigration.dumper.jdbc.JdbcUtil.getIntNotNull; +import static com.google.edwmigration.dumper.jdbc.JdbcUtil.getStringNotNull; +import static java.lang.System.lineSeparator; + +import com.google.auto.value.AutoValue; +import com.google.edwmigration.dumper.csv.CsvUtil; +import java.sql.ResultSet; +import java.sql.SQLException; + +/** POJO class for serialization data from DB and CSV files. */ +@AutoValue +public abstract class SvvColumnsRow { + + public static SvvColumnsRow create( + String tableCatalog, + String tableSchema, + String tableName, + String columnName, + int ordinalPosition, + String columnDefault, + String isNullable, + String dataType, + int characterMaximumLength, + int numericPrecision, + int numericPrecisionRadix, + int numericScale, + int datetimePrecision, + String intervalType, + String intervalPrecision, + String characterSetCatalog, + String characterSetSchema, + String characterSetName, + String collationCatalog, + String collationSchema, + String collationName, + String domainName, + String remarks) { + return new AutoValue_SvvColumnsRow( + tableCatalog, + tableSchema, + tableName, + columnName, + ordinalPosition, + columnDefault, + isNullable, + dataType, + characterMaximumLength, + numericPrecision, + numericPrecisionRadix, + numericScale, + datetimePrecision, + intervalType, + intervalPrecision, + characterSetCatalog, + characterSetSchema, + characterSetName, + collationCatalog, + collationSchema, + collationName, + domainName, + remarks); + } + + public static SvvColumnsRow create(ResultSet rs) throws SQLException { + return SvvColumnsRow.create( + getStringNotNull(rs, "table_catalog"), + getStringNotNull(rs, "table_schema"), + getStringNotNull(rs, "table_name"), + getStringNotNull(rs, "column_name"), + getIntNotNull(rs, "ordinal_position"), + getStringNotNull(rs, "column_default"), + getStringNotNull(rs, "is_nullable"), + getStringNotNull(rs, "data_type"), + getIntNotNull(rs, "character_maximum_length"), + getIntNotNull(rs, "numeric_precision"), + getIntNotNull(rs, "numeric_precision_radix"), + getIntNotNull(rs, "numeric_scale"), + getIntNotNull(rs, "datetime_precision"), + getStringNotNull(rs, "interval_type"), + getStringNotNull(rs, "interval_precision"), + getStringNotNull(rs, "character_set_catalog"), + getStringNotNull(rs, "character_set_schema"), + getStringNotNull(rs, "character_set_name"), + getStringNotNull(rs, "collation_catalog"), + getStringNotNull(rs, "collation_schema"), + getStringNotNull(rs, "collation_name"), + getStringNotNull(rs, "domain_name"), + getStringNotNull(rs, "remarks")); + } + + public static SvvColumnsRow create(String[] csvLine) { + return new AutoValue_SvvColumnsRow( + CsvUtil.getStringNotNull(csvLine[0]), + CsvUtil.getStringNotNull(csvLine[1]), + CsvUtil.getStringNotNull(csvLine[2]), + CsvUtil.getStringNotNull(csvLine[3]), + CsvUtil.getIntNotNull(csvLine[4]), + CsvUtil.getStringNotNull(csvLine[5]), + CsvUtil.getStringNotNull(csvLine[6]), + CsvUtil.getStringNotNull(csvLine[7]), + CsvUtil.getIntNotNull(csvLine[8]), + CsvUtil.getIntNotNull(csvLine[9]), + CsvUtil.getIntNotNull(csvLine[10]), + CsvUtil.getIntNotNull(csvLine[11]), + CsvUtil.getIntNotNull(csvLine[12]), + CsvUtil.getStringNotNull(csvLine[13]), + CsvUtil.getStringNotNull(csvLine[14]), + CsvUtil.getStringNotNull(csvLine[15]), + CsvUtil.getStringNotNull(csvLine[16]), + CsvUtil.getStringNotNull(csvLine[17]), + CsvUtil.getStringNotNull(csvLine[18]), + CsvUtil.getStringNotNull(csvLine[19]), + CsvUtil.getStringNotNull(csvLine[20]), + CsvUtil.getStringNotNull(csvLine[21]), + CsvUtil.getStringNotNull(csvLine[22])); + } + + public abstract String tableCatalog(); + + public abstract String tableSchema(); + + public abstract String tableName(); + + public abstract String columnName(); + + public abstract int ordinalPosition(); + + public abstract String columnDefault(); + + public abstract String isNullable(); + + public abstract String dataType(); + + public abstract int characterMaximumLength(); + + public abstract int numericPrecision(); + + public abstract int numericPrecisionRadix(); + + public abstract int numericScale(); + + public abstract int datetimePrecision(); + + public abstract String intervalType(); + + public abstract String intervalPrecision(); + + public abstract String characterSetCatalog(); + + public abstract String characterSetSchema(); + + public abstract String characterSetName(); + + public abstract String collationCatalog(); + + public abstract String collationSchema(); + + public abstract String collationName(); + + public abstract String domainName(); + + public abstract String remarks(); + + @Override + public String toString() { + return "tableCatalog=" + + tableCatalog() + + ", tableSchema=" + + tableSchema() + + ", tableName=" + + tableName() + + ", columnName=" + + columnName() + + ", ordinalPosition=" + + ordinalPosition() + + ", columnDefault=" + + columnDefault() + + ", isNullable=" + + isNullable() + + ", dataType=" + + dataType() + + ", characterMaximumLength=" + + characterMaximumLength() + + ", numericPrecision=" + + numericPrecision() + + ", numericPrecisionRadix=" + + numericPrecisionRadix() + + ", numericScale=" + + numericScale() + + ", datetimePrecision=" + + datetimePrecision() + + ", intervalType=" + + intervalType() + + ", intervalPrecision=" + + intervalPrecision() + + ", characterSetCatalog=" + + characterSetCatalog() + + ", characterSetSchema=" + + characterSetSchema() + + ", characterSetName=" + + characterSetName() + + ", collationCatalog=" + + collationCatalog() + + ", collationSchema=" + + collationSchema() + + ", collationName=" + + collationName() + + ", domainName=" + + domainName() + + ", remarks=" + + remarks() + + lineSeparator(); + } +} diff --git a/dumper-integration-tests/redshift-tests/src/main/java/com/google/sql/SqlUtil.java b/dumper-integration-tests/redshift-tests/src/main/java/com/google/edwmigration/dumper/sql/SqlUtil.java similarity index 93% rename from dumper-integration-tests/redshift-tests/src/main/java/com/google/sql/SqlUtil.java rename to dumper-integration-tests/redshift-tests/src/main/java/com/google/edwmigration/dumper/sql/SqlUtil.java index fdc8daae6..2d6db9f25 100644 --- a/dumper-integration-tests/redshift-tests/src/main/java/com/google/sql/SqlUtil.java +++ b/dumper-integration-tests/redshift-tests/src/main/java/com/google/edwmigration/dumper/sql/SqlUtil.java @@ -1,6 +1,5 @@ /* * Copyright 2022 Google LLC - * Copyright 2013-2021 CompilerWorks * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.sql; +package com.google.edwmigration.dumper.sql; -import static com.google.base.TestConstants.URL_DB; +import static com.google.edwmigration.dumper.base.TestConstants.URL_DB; import static java.lang.String.format; import static org.apache.commons.io.FileUtils.readFileToString; @@ -32,15 +31,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * A helper class for reading .sql files. - */ +/** A helper class for reading .sql files. */ public final class SqlUtil { private static final Logger LOGGER = LoggerFactory.getLogger(SqlUtil.class); - private SqlUtil() { - } + private SqlUtil() {} /** * @param sqlPath Path to an .sql file. diff --git a/dumper-integration-tests/redshift-tests/src/main/resources/sql/svv_columns.sql b/dumper-integration-tests/redshift-tests/src/main/resources/sql/svv_columns.sql new file mode 100644 index 000000000..91ee61970 --- /dev/null +++ b/dumper-integration-tests/redshift-tests/src/main/resources/sql/svv_columns.sql @@ -0,0 +1,15 @@ +-- Copyright 2022 Google LLC +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +SELECT * FROM SVV_COLUMNS; \ No newline at end of file diff --git a/dumper-integration-tests/redshift-tests/src/test/java/com/google/integration/RedshiftTest.java b/dumper-integration-tests/redshift-tests/src/test/java/com/google/integration/RedshiftTest.java deleted file mode 100644 index d3cb84edb..000000000 --- a/dumper-integration-tests/redshift-tests/src/test/java/com/google/integration/RedshiftTest.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright 2022 Google LLC - * Copyright 2013-2021 CompilerWorks - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.integration; - -import static com.google.base.TestBase.assertListsEqual; -import static com.google.base.TestConstants.PASSWORD_DB; -import static com.google.base.TestConstants.URL_DB; -import static com.google.base.TestConstants.USERNAME_DB; - -import com.google.common.collect.LinkedHashMultiset; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.Test; - -public class RedshiftTest { - - private static final Logger LOGGER = LoggerFactory.getLogger(RedshiftTest.class); - private static Connection connection; - - @BeforeClass - public static void beforeClass() throws SQLException { - connection = DriverManager.getConnection(URL_DB, USERNAME_DB, PASSWORD_DB); - } - - @Test - public void templateTest() throws SQLException { - String sql = "select * from customers;"; - - LinkedHashMultiset dbList = LinkedHashMultiset.create(); - LinkedHashMultiset outputList = LinkedHashMultiset.create(); - - //Extract from RedShift - try (PreparedStatement preparedStatement = connection.prepareStatement(sql)) { - ResultSet rs = preparedStatement.executeQuery(); - - while (rs.next()) { - String customernumber = rs.getString("customernumber"); - String customername = rs.getString("customername"); - String phonenumber = rs.getString("phonenumber"); - String postalcode = rs.getString("postalcode"); - String locale = rs.getString("locale"); - String datecreated = rs.getString("datecreated"); - String email = rs.getString("email"); - - LOGGER.info(String.format("%s, %s, %s, %s, %s, %s, %s", customernumber.trim(), customername, - phonenumber, postalcode, locale, datecreated, email)); - } - } - - //Extract from output files - // TODO - - //Reduce and compare - LinkedHashMultiset dbListCopy = LinkedHashMultiset.create(dbList); - outputList.forEach(dbList::remove); - dbListCopy.forEach(outputList::remove); - - assertListsEqual(dbList, outputList); - } -} diff --git a/dumper-integration-tests/redshift-tests/src/test/java/com/google/integration/SvvColumnsTest.java b/dumper-integration-tests/redshift-tests/src/test/java/com/google/integration/SvvColumnsTest.java new file mode 100644 index 000000000..ac5a39f17 --- /dev/null +++ b/dumper-integration-tests/redshift-tests/src/test/java/com/google/integration/SvvColumnsTest.java @@ -0,0 +1,81 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.integration; + +import static com.google.edwmigration.dumper.base.TestBase.assertListsEqual; +import static com.google.edwmigration.dumper.base.TestConstants.EXPORTED_FILES_BASE_PATH; +import static com.google.edwmigration.dumper.base.TestConstants.PASSWORD_DB; +import static com.google.edwmigration.dumper.base.TestConstants.SQL_REQUESTS_BASE_PATH; +import static com.google.edwmigration.dumper.base.TestConstants.URL_DB; +import static com.google.edwmigration.dumper.base.TestConstants.USERNAME_DB; + +import com.google.common.collect.LinkedHashMultiset; +import com.google.edwmigration.dumper.pojo.SvvColumnsRow; +import com.google.edwmigration.dumper.sql.SqlUtil; +import com.opencsv.CSVReader; +import com.opencsv.CSVReaderBuilder; +import com.opencsv.exceptions.CsvValidationException; +import java.io.FileReader; +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +public class SvvColumnsTest { + + private static final String SQL_PATH = SQL_REQUESTS_BASE_PATH + "svv_columns.sql"; + private static final String CSV_FILE_PATH = EXPORTED_FILES_BASE_PATH + "svv_columns.csv"; + private static Connection connection; + + @BeforeClass + public static void beforeClass() throws SQLException { + connection = DriverManager.getConnection(URL_DB, USERNAME_DB, PASSWORD_DB); + } + + @Test + public void svvColumnsTest() throws SQLException, IOException, CsvValidationException { + LinkedHashMultiset dbList = LinkedHashMultiset.create(); + LinkedHashMultiset csvList = LinkedHashMultiset.create(); + + try (PreparedStatement preparedStatement = + connection.prepareStatement(SqlUtil.getSql(SQL_PATH))) { + ResultSet rs = preparedStatement.executeQuery(); + + while (rs.next()) { + dbList.add(SvvColumnsRow.create(rs)); + } + } + + FileReader fileReader = new FileReader(CSV_FILE_PATH); + try (CSVReader reader = new CSVReaderBuilder(fileReader).withSkipLines(1).build()) { + String[] line; + while ((line = reader.readNext()) != null) { + SvvColumnsRow csvRow = SvvColumnsRow.create(line); + csvList.add(csvRow); + } + } + + LinkedHashMultiset dbListCopy = LinkedHashMultiset.create(dbList); + csvList.forEach(dbList::remove); + dbListCopy.forEach(csvList::remove); + + assertListsEqual(dbList, csvList); + } +}