diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/MetadataDumper.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/MetadataDumper.java index 01e63acf8..1e50c54de 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/MetadataDumper.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/MetadataDumper.java @@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.io.Closer; import com.google.common.io.Files; +import com.google.edwmigration.dumper.application.dumper.SummaryPrinter.SummaryLinePrinter; import com.google.edwmigration.dumper.application.dumper.connector.Connector; import com.google.edwmigration.dumper.application.dumper.handle.Handle; import com.google.edwmigration.dumper.application.dumper.io.FileSystemOutputHandleFactory; @@ -43,6 +44,9 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.time.Clock; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -61,6 +65,9 @@ public class MetadataDumper { private static final Pattern GCS_PATH_PATTERN = Pattern.compile("gs://(?[^/]+)/(?.*)"); + private static DateTimeFormatter OUTPUT_DATE_FORMAT = + DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm").withZone(ZoneOffset.UTC); + public boolean run(String... args) throws Exception { ConnectorArguments arguments = new ConnectorArguments(JsonResponseFile.addResponseFiles(args)); try { @@ -194,6 +201,7 @@ protected boolean run(@Nonnull Connector connector, @Nonnull ConnectorArguments logFinalSummary( summaryPrinter, state, + connector, outputFileLength, stopwatch, outputFileLocation, @@ -265,9 +273,36 @@ private boolean checkRequiredTaskSuccess( return true; } + private void outputFirstAndLastQueryLogEnries(SummaryLinePrinter linePrinter) { + + if (QueryLogSharedState.sizeOfQueryLogEntries() == 0) { + return; + } + + ZonedDateTime queryLogFirstEntry = + QueryLogSharedState.getQueryLogEntry( + QueryLogSharedState.QueryLogEntry.QUERY_LOG_FIRST_ENTRY); + + ZonedDateTime queryLogLastEntry = + QueryLogSharedState.getQueryLogEntry( + QueryLogSharedState.QueryLogEntry.QUERY_LOG_LAST_ENTRY); + + if (queryLogFirstEntry != null && queryLogLastEntry != null) { + linePrinter.println( + "The first query log entry is '%s' UTC and the last query log entry is '%s' UTC", + QueryLogSharedState.getQueryLogEntry( + QueryLogSharedState.QueryLogEntry.QUERY_LOG_FIRST_ENTRY) + .format(OUTPUT_DATE_FORMAT), + QueryLogSharedState.getQueryLogEntry( + QueryLogSharedState.QueryLogEntry.QUERY_LOG_LAST_ENTRY) + .format(OUTPUT_DATE_FORMAT)); + } + } + private void logFinalSummary( SummaryPrinter summaryPrinter, TaskSetState state, + Connector connector, long outputFileLength, Stopwatch stopwatch, String outputFileLocation, @@ -281,6 +316,10 @@ private void logFinalSummary( + state.getTasksReports().stream() .map(taskReport -> taskReport.count() + " " + taskReport.state()) .collect(joining(", "))); + // For now, it will return true only for TeradataLogsConnector and Terada14LogsConnector + if (connector.shouldOutputFirstAndLastQueryLog()) { + outputFirstAndLastQueryLogEnries(linePrinter); + } if (requiredTaskSucceeded) { linePrinter.println("Output saved to '%s'", outputFileLocation); } else { diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedState.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedState.java new file mode 100644 index 000000000..a8dda1bd9 --- /dev/null +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedState.java @@ -0,0 +1,66 @@ +/* + * Copyright 2022-2024 Google LLC + * Copyright 2013-2021 CompilerWorks + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.edwmigration.dumper.application.dumper; + +import com.google.common.annotations.VisibleForTesting; +import java.time.ZonedDateTime; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/* + * Shared state class for calculating first and last query log entries. + * Class could be used inside different tasks, which run concurrently, + * so class designed to be thread-safe, by using ConcurrentMap. + */ +public class QueryLogSharedState { + private static final ConcurrentMap queryLogEntries = + new ConcurrentHashMap<>(); + + public enum QueryLogEntry { + QUERY_LOG_FIRST_ENTRY, + QUERY_LOG_LAST_ENTRY + } + + public static ZonedDateTime getQueryLogEntry(QueryLogEntry queryLogEntry) { + return queryLogEntries.getOrDefault(queryLogEntry, null); + } + + public static int sizeOfQueryLogEntries() { + return queryLogEntries.size(); + } + + /* + * Calculates first and last query log entries, by applying 'min' and 'max' logic. + */ + public static void updateQueryLogEntries(QueryLogEntry logEntry, ZonedDateTime newDateTime) { + ZonedDateTime currentDateTime = QueryLogSharedState.queryLogEntries.get(logEntry); + if (currentDateTime == null) { + QueryLogSharedState.queryLogEntries.put(logEntry, newDateTime); + } else { + if ((logEntry == QueryLogEntry.QUERY_LOG_FIRST_ENTRY && newDateTime.isBefore(currentDateTime)) + || (logEntry == QueryLogEntry.QUERY_LOG_LAST_ENTRY + && newDateTime.isAfter(currentDateTime))) { + QueryLogSharedState.queryLogEntries.put(logEntry, newDateTime); + } + } + } + + @VisibleForTesting + static void clearQueryLogEntries() { + queryLogEntries.clear(); + } +} diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/Connector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/Connector.java index 9b1ef9f37..2ee61016f 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/Connector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/Connector.java @@ -45,4 +45,8 @@ public void addTasksTo(@Nonnull List> out, @Nonnull ConnectorArg public default Class> getConnectorProperties() { return DefaultProperties.class; } + + default boolean shouldOutputFirstAndLastQueryLog() { + return false; + } } diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/Teradata14LogsConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/Teradata14LogsConnector.java index a0df5db24..bfb5e1c26 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/Teradata14LogsConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/Teradata14LogsConnector.java @@ -23,6 +23,8 @@ import com.google.common.io.ByteSink; import com.google.edwmigration.dumper.application.dumper.ConnectorArguments; import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException; +import com.google.edwmigration.dumper.application.dumper.QueryLogSharedState; +import com.google.edwmigration.dumper.application.dumper.QueryLogSharedState.QueryLogEntry; import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentQueryLogDays; import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentQueryLogEnd; import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentQueryLogStart; @@ -86,6 +88,15 @@ public Teradata14LogsConnector() { super("teradata14-logs"); } + /* + * Overriding it only for Teradata logs connectors, so in MetadataDumper summary + * section only they can output first and last entries of query logs for now + */ + @Override + public boolean shouldOutputFirstAndLastQueryLog() { + return true; + } + private abstract static class Teradata14LogsJdbcTask extends AbstractJdbcTask { protected static String EXPRESSION_VALIDITY_QUERY = "SELECT TOP 1 %s FROM %s"; @@ -133,7 +144,14 @@ protected Summary doInConnection( throws SQLException { String sql = getSql(jdbcHandle); ResultSetExtractor rse = newCsvResultSetExtractor(sink); - return doSelect(connection, withInterval(rse, interval), sql); + Summary summary = doSelect(connection, withInterval(rse, interval), sql); + if (summary != null && summary.rowCount() > 0) { + QueryLogSharedState.updateQueryLogEntries( + QueryLogEntry.QUERY_LOG_FIRST_ENTRY, interval.getStart()); + QueryLogSharedState.updateQueryLogEntries( + QueryLogEntry.QUERY_LOG_LAST_ENTRY, interval.getEndExclusive()); + } + return summary; } @Nonnull diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsConnector.java index 0395147d1..081555c71 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsConnector.java @@ -175,6 +175,15 @@ public TeradataLogsConnector() { super("teradata-logs"); } + /* + * Overriding it only for Teradata logs connectors, so in MetadataDumper summary section + * only they can output first and last entries of query logs for now + */ + @Override + public boolean shouldOutputFirstAndLastQueryLog() { + return true; + } + private ImmutableList createTimeSeriesTasks( ZonedInterval interval, @Nonnull ConnectorArguments arguments) { return TIME_SERIES_PROPERTY_TO_FILENAME_PREFIX_MAP.keySet().stream() diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsJdbcTask.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsJdbcTask.java index 71d0b6b08..d91fcdd93 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsJdbcTask.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsJdbcTask.java @@ -31,6 +31,8 @@ import com.google.common.collect.ImmutableSet; import com.google.common.io.ByteSink; import com.google.common.primitives.Ints; +import com.google.edwmigration.dumper.application.dumper.QueryLogSharedState; +import com.google.edwmigration.dumper.application.dumper.QueryLogSharedState.QueryLogEntry; import com.google.edwmigration.dumper.application.dumper.connector.ZonedInterval; import com.google.edwmigration.dumper.application.dumper.connector.teradata.AbstractTeradataConnector.SharedState; import com.google.edwmigration.dumper.application.dumper.connector.teradata.query.model.Expression; @@ -157,7 +159,14 @@ protected Summary doInConnection( throws SQLException { String sql = getOrCreateSql(jdbcHandle); ResultSetExtractor rse = newCsvResultSetExtractor(sink); - return doSelect(connection, withInterval(rse, interval), sql); + Summary summary = doSelect(connection, withInterval(rse, interval), sql); + if (summary != null && summary.rowCount() > 0) { + QueryLogSharedState.updateQueryLogEntries( + QueryLogEntry.QUERY_LOG_FIRST_ENTRY, interval.getStart()); + QueryLogSharedState.updateQueryLogEntries( + QueryLogEntry.QUERY_LOG_LAST_ENTRY, interval.getEndExclusive()); + } + return summary; } @Nonnull diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedStateTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedStateTest.java new file mode 100644 index 000000000..ec03d25e5 --- /dev/null +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedStateTest.java @@ -0,0 +1,88 @@ +/* + * Copyright 2022-2024 Google LLC + * Copyright 2013-2021 CompilerWorks + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.edwmigration.dumper.application.dumper; + +import static org.junit.Assert.assertEquals; + +import com.google.edwmigration.dumper.application.dumper.QueryLogSharedState.QueryLogEntry; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class QueryLogSharedStateTest { + + @Before + public void beforeEachTest() { + QueryLogSharedState.clearQueryLogEntries(); + } + + @Test + public void queryLogFirstEntryUpdatedSuccessfully() { + ZonedDateTime newQueryLogDate = ZonedDateTime.now(); + + // Act + QueryLogSharedState.updateQueryLogEntries(QueryLogEntry.QUERY_LOG_FIRST_ENTRY, newQueryLogDate); + + // Assert + assertEquals( + newQueryLogDate, QueryLogSharedState.getQueryLogEntry(QueryLogEntry.QUERY_LOG_FIRST_ENTRY)); + } + + @Test + public void queryLogLastEntryUpdatedSuccessfully() { + ZonedDateTime newQueryLogDate = ZonedDateTime.now(); + + // Act + QueryLogSharedState.updateQueryLogEntries(QueryLogEntry.QUERY_LOG_LAST_ENTRY, newQueryLogDate); + + // Assert + assertEquals( + newQueryLogDate, QueryLogSharedState.getQueryLogEntry(QueryLogEntry.QUERY_LOG_LAST_ENTRY)); + } + + @Test + public void queryLogFirstEntryUpdatedSuccessfullyForEarlierDate() { + ZonedDateTime now = ZonedDateTime.now(); + ZonedDateTime earlierDate = ZonedDateTime.of(1970, 1, 1, 1, 1, 1, 1, ZoneId.of("UTC")); + + // Act + QueryLogSharedState.updateQueryLogEntries(QueryLogEntry.QUERY_LOG_FIRST_ENTRY, now); + QueryLogSharedState.updateQueryLogEntries(QueryLogEntry.QUERY_LOG_FIRST_ENTRY, earlierDate); + + // Assert + assertEquals( + earlierDate, QueryLogSharedState.getQueryLogEntry(QueryLogEntry.QUERY_LOG_FIRST_ENTRY)); + } + + @Test + public void queryLogLastEntryUpdatedSuccessfullyForLaterDate() { + ZonedDateTime date = ZonedDateTime.of(2000, 1, 1, 1, 1, 1, 1, ZoneId.of("UTC")); + ZonedDateTime laterDate = ZonedDateTime.now(); + + // Act + QueryLogSharedState.updateQueryLogEntries(QueryLogEntry.QUERY_LOG_LAST_ENTRY, date); + QueryLogSharedState.updateQueryLogEntries(QueryLogEntry.QUERY_LOG_LAST_ENTRY, laterDate); + + // Assert + assertEquals( + laterDate, QueryLogSharedState.getQueryLogEntry(QueryLogEntry.QUERY_LOG_LAST_ENTRY)); + } +}