From ce8fa5a9328f2e8eb50c2dc1a456fd7358b3b44d Mon Sep 17 00:00:00 2001 From: Sanan Yuzbashiyev Date: Wed, 2 Oct 2024 13:48:32 +0000 Subject: [PATCH 01/12] Add conccurent map to store first and last entries of query logs --- .../application/dumper/MetadataDumper.java | 12 +++++++++ .../dumper/connector/Connector.java | 4 +++ .../dumper/connector/LogsConnector.java | 5 ++++ .../teradata/AbstractTeradataConnector.java | 11 ++++++++ .../teradata/TeradataLogsJdbcTask.java | 27 ++++++++++++++++++- 5 files changed, 58 insertions(+), 1 deletion(-) diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/MetadataDumper.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/MetadataDumper.java index 01e63acf8..db927e1ca 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/MetadataDumper.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/MetadataDumper.java @@ -25,6 +25,8 @@ import com.google.common.io.Closer; import com.google.common.io.Files; import com.google.edwmigration.dumper.application.dumper.connector.Connector; +import com.google.edwmigration.dumper.application.dumper.connector.teradata.AbstractTeradataConnector.TeradataExternalSharedState; +import com.google.edwmigration.dumper.application.dumper.connector.teradata.AbstractTeradataConnector.TeradataQueryLogEntries; import com.google.edwmigration.dumper.application.dumper.handle.Handle; import com.google.edwmigration.dumper.application.dumper.io.FileSystemOutputHandleFactory; import com.google.edwmigration.dumper.application.dumper.io.OutputHandleFactory; @@ -194,6 +196,7 @@ protected boolean run(@Nonnull Connector connector, @Nonnull ConnectorArguments logFinalSummary( summaryPrinter, state, + connector, outputFileLength, stopwatch, outputFileLocation, @@ -268,6 +271,7 @@ private boolean checkRequiredTaskSuccess( private void logFinalSummary( SummaryPrinter summaryPrinter, TaskSetState state, + Connector connector, long outputFileLength, Stopwatch stopwatch, String outputFileLocation, @@ -281,6 +285,14 @@ private void logFinalSummary( + state.getTasksReports().stream() .map(taskReport -> taskReport.count() + " " + taskReport.state()) .collect(joining(", "))); + if (connector.isLogConnector()) { + linePrinter.println( + "First query log entry is '%s' and last query log entry is '%s'", + TeradataExternalSharedState.queryLogEntries.get( + TeradataQueryLogEntries.QUERY_LOG_FIRST_ENTRY), + TeradataExternalSharedState.queryLogEntries.get( + TeradataQueryLogEntries.QUERY_LOG_LAST_ENTRY)); + } if (requiredTaskSucceeded) { linePrinter.println("Output saved to '%s'", outputFileLocation); } else { diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/Connector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/Connector.java index 9b1ef9f37..793a345b8 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/Connector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/Connector.java @@ -45,4 +45,8 @@ public void addTasksTo(@Nonnull List> out, @Nonnull ConnectorArg public default Class> getConnectorProperties() { return DefaultProperties.class; } + + default boolean isLogConnector() { + return false; + } } diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/LogsConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/LogsConnector.java index 176730bd2..50e60b1c7 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/LogsConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/LogsConnector.java @@ -32,4 +32,9 @@ default String getDefaultFileName(boolean isAssessment, Clock clock) { return ArchiveNameUtil.getFileName(getName()); } } + + @Override + default boolean isLogConnector() { + return true; + } } diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/AbstractTeradataConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/AbstractTeradataConnector.java index eef7e6aa2..6c9d57112 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/AbstractTeradataConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/AbstractTeradataConnector.java @@ -45,6 +45,7 @@ import java.sql.PreparedStatement; import java.sql.SQLException; import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.List; import java.util.Optional; @@ -118,6 +119,11 @@ public String getDescription() { } } + public enum TeradataQueryLogEntries { + QUERY_LOG_FIRST_ENTRY, + QUERY_LOG_LAST_ENTRY + } + protected static class TeradataJdbcSelectTask extends JdbcSelectTask { private final String sqlCount; @@ -175,6 +181,11 @@ protected static class SharedState { protected final ConcurrentMap expressionValidity = new ConcurrentHashMap<>(); } + public static class TeradataExternalSharedState { + public static final ConcurrentMap queryLogEntries = + new ConcurrentHashMap<>(); + } + protected static boolean isQueryTable(@Nonnull String expression) { return expression.startsWith("ST."); } diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsJdbcTask.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsJdbcTask.java index 71d0b6b08..ef426b452 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsJdbcTask.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsJdbcTask.java @@ -33,6 +33,8 @@ import com.google.common.primitives.Ints; import com.google.edwmigration.dumper.application.dumper.connector.ZonedInterval; import com.google.edwmigration.dumper.application.dumper.connector.teradata.AbstractTeradataConnector.SharedState; +import com.google.edwmigration.dumper.application.dumper.connector.teradata.AbstractTeradataConnector.TeradataExternalSharedState; +import com.google.edwmigration.dumper.application.dumper.connector.teradata.AbstractTeradataConnector.TeradataQueryLogEntries; import com.google.edwmigration.dumper.application.dumper.connector.teradata.query.model.Expression; import com.google.edwmigration.dumper.application.dumper.handle.JdbcHandle; import com.google.edwmigration.dumper.application.dumper.task.AbstractJdbcTask; @@ -42,6 +44,7 @@ import java.sql.Connection; import java.sql.SQLException; import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.List; import java.util.Optional; @@ -157,7 +160,14 @@ protected Summary doInConnection( throws SQLException { String sql = getOrCreateSql(jdbcHandle); ResultSetExtractor rse = newCsvResultSetExtractor(sink); - return doSelect(connection, withInterval(rse, interval), sql); + Summary summary = doSelect(connection, withInterval(rse, interval), sql); + if (summary != null && summary.rowCount() > 0) { + updateTeradaQueryLogEntries( + TeradataQueryLogEntries.QUERY_LOG_FIRST_ENTRY, interval.getStart()); + updateTeradaQueryLogEntries( + TeradataQueryLogEntries.QUERY_LOG_LAST_ENTRY, interval.getEndExclusive()); + } + return summary; } @Nonnull @@ -314,4 +324,19 @@ public String describeSourceData() { "from tables '%s' and '%s'", tableNames.queryLogsTableName(), tableNames.sqlLogsTableName())); } + + private void updateTeradaQueryLogEntries( + TeradataQueryLogEntries logEntry, ZonedDateTime newDateTime) { + ZonedDateTime currentDateTime = TeradataExternalSharedState.queryLogEntries.get(logEntry); + if (currentDateTime == null) { + TeradataExternalSharedState.queryLogEntries.put(logEntry, newDateTime); + } else { + if (logEntry == TeradataQueryLogEntries.QUERY_LOG_FIRST_ENTRY + && newDateTime.isBefore(currentDateTime) + || logEntry == TeradataQueryLogEntries.QUERY_LOG_LAST_ENTRY + && newDateTime.isAfter(currentDateTime)) { + TeradataExternalSharedState.queryLogEntries.put(logEntry, newDateTime); + } + } + } } From b3ea1c1eebc3fa56cc0dd514a3fa1b98d3152f09 Mon Sep 17 00:00:00 2001 From: Sanan Yuzbashiyev Date: Thu, 3 Oct 2024 10:27:27 +0000 Subject: [PATCH 02/12] Separate method to output in summary section --- .../application/dumper/MetadataDumper.java | 22 ++++++++++++++----- .../teradata/TeradataLogsJdbcTask.java | 3 +++ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/MetadataDumper.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/MetadataDumper.java index db927e1ca..e285cfad6 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/MetadataDumper.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/MetadataDumper.java @@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.io.Closer; import com.google.common.io.Files; +import com.google.edwmigration.dumper.application.dumper.SummaryPrinter.SummaryLinePrinter; import com.google.edwmigration.dumper.application.dumper.connector.Connector; import com.google.edwmigration.dumper.application.dumper.connector.teradata.AbstractTeradataConnector.TeradataExternalSharedState; import com.google.edwmigration.dumper.application.dumper.connector.teradata.AbstractTeradataConnector.TeradataQueryLogEntries; @@ -268,6 +269,20 @@ private boolean checkRequiredTaskSuccess( return true; } + private void outputFirstAndLastQueryLogEnries(SummaryLinePrinter linePrinter) { + + if (TeradataExternalSharedState.queryLogEntries.size() == 0) { + return; + } + + linePrinter.println( + "First query log entry is '%s' and last query log entry is '%s'", + TeradataExternalSharedState.queryLogEntries.get( + TeradataQueryLogEntries.QUERY_LOG_FIRST_ENTRY), + TeradataExternalSharedState.queryLogEntries.get( + TeradataQueryLogEntries.QUERY_LOG_LAST_ENTRY)); + } + private void logFinalSummary( SummaryPrinter summaryPrinter, TaskSetState state, @@ -286,12 +301,7 @@ private void logFinalSummary( .map(taskReport -> taskReport.count() + " " + taskReport.state()) .collect(joining(", "))); if (connector.isLogConnector()) { - linePrinter.println( - "First query log entry is '%s' and last query log entry is '%s'", - TeradataExternalSharedState.queryLogEntries.get( - TeradataQueryLogEntries.QUERY_LOG_FIRST_ENTRY), - TeradataExternalSharedState.queryLogEntries.get( - TeradataQueryLogEntries.QUERY_LOG_LAST_ENTRY)); + outputFirstAndLastQueryLogEnries(linePrinter); } if (requiredTaskSucceeded) { linePrinter.println("Output saved to '%s'", outputFileLocation); diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsJdbcTask.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsJdbcTask.java index ef426b452..24aec671e 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsJdbcTask.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsJdbcTask.java @@ -325,6 +325,9 @@ public String describeSourceData() { tableNames.queryLogsTableName(), tableNames.sqlLogsTableName())); } + /* + * Calculates first and last query log entries, by applying 'min' and 'max' logic. + */ private void updateTeradaQueryLogEntries( TeradataQueryLogEntries logEntry, ZonedDateTime newDateTime) { ZonedDateTime currentDateTime = TeradataExternalSharedState.queryLogEntries.get(logEntry); From b4325582b7971fdd5da8e49c33c39040b76f28f2 Mon Sep 17 00:00:00 2001 From: Sanan Yuzbashiyev Date: Thu, 3 Oct 2024 11:06:03 +0000 Subject: [PATCH 03/12] Implement in general way, but apply for only Teradata for now --- .../application/dumper/MetadataDumper.java | 16 +++--- .../dumper/QueryLogSharedState.java | 49 +++++++++++++++++++ .../dumper/connector/LogsConnector.java | 5 -- .../teradata/AbstractTeradataConnector.java | 11 ----- .../teradata/Teradata14LogsConnector.java | 21 +++++++- .../teradata/TeradataLogsConnector.java | 9 ++++ .../teradata/TeradataLogsJdbcTask.java | 31 +++--------- 7 files changed, 93 insertions(+), 49 deletions(-) create mode 100644 dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedState.java diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/MetadataDumper.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/MetadataDumper.java index e285cfad6..ec067639a 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/MetadataDumper.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/MetadataDumper.java @@ -26,8 +26,6 @@ import com.google.common.io.Files; import com.google.edwmigration.dumper.application.dumper.SummaryPrinter.SummaryLinePrinter; import com.google.edwmigration.dumper.application.dumper.connector.Connector; -import com.google.edwmigration.dumper.application.dumper.connector.teradata.AbstractTeradataConnector.TeradataExternalSharedState; -import com.google.edwmigration.dumper.application.dumper.connector.teradata.AbstractTeradataConnector.TeradataQueryLogEntries; import com.google.edwmigration.dumper.application.dumper.handle.Handle; import com.google.edwmigration.dumper.application.dumper.io.FileSystemOutputHandleFactory; import com.google.edwmigration.dumper.application.dumper.io.OutputHandleFactory; @@ -269,18 +267,21 @@ private boolean checkRequiredTaskSuccess( return true; } + /* + * This logic works for only Teradata logs connector for now + */ private void outputFirstAndLastQueryLogEnries(SummaryLinePrinter linePrinter) { - if (TeradataExternalSharedState.queryLogEntries.size() == 0) { + if (QueryLogSharedState.queryLogEntries.size() == 0) { return; } linePrinter.println( "First query log entry is '%s' and last query log entry is '%s'", - TeradataExternalSharedState.queryLogEntries.get( - TeradataQueryLogEntries.QUERY_LOG_FIRST_ENTRY), - TeradataExternalSharedState.queryLogEntries.get( - TeradataQueryLogEntries.QUERY_LOG_LAST_ENTRY)); + QueryLogSharedState.queryLogEntries.get( + QueryLogSharedState.QueryLogEntries.QUERY_LOG_FIRST_ENTRY), + QueryLogSharedState.queryLogEntries.get( + QueryLogSharedState.QueryLogEntries.QUERY_LOG_LAST_ENTRY)); } private void logFinalSummary( @@ -300,6 +301,7 @@ private void logFinalSummary( + state.getTasksReports().stream() .map(taskReport -> taskReport.count() + " " + taskReport.state()) .collect(joining(", "))); + // For now, it will return true only for TeradataLogsConnector and Terada14LogsConnector if (connector.isLogConnector()) { outputFirstAndLastQueryLogEnries(linePrinter); } diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedState.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedState.java new file mode 100644 index 000000000..b329f0284 --- /dev/null +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedState.java @@ -0,0 +1,49 @@ +/* + * Copyright 2022-2024 Google LLC + * Copyright 2013-2021 CompilerWorks + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.edwmigration.dumper.application.dumper; + +import java.time.ZonedDateTime; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +public class QueryLogSharedState { + public static final ConcurrentMap queryLogEntries = + new ConcurrentHashMap<>(); + + public enum QueryLogEntries { + QUERY_LOG_FIRST_ENTRY, + QUERY_LOG_LAST_ENTRY + } + + /* + * Calculates first and last query log entries, by applying 'min' and 'max' logic. + */ + public static void updateQueryLogEntries( + QueryLogSharedState.QueryLogEntries logEntry, ZonedDateTime newDateTime) { + ZonedDateTime currentDateTime = QueryLogSharedState.queryLogEntries.get(logEntry); + if (currentDateTime == null) { + QueryLogSharedState.queryLogEntries.put(logEntry, newDateTime); + } else { + if (logEntry == QueryLogSharedState.QueryLogEntries.QUERY_LOG_FIRST_ENTRY + && newDateTime.isBefore(currentDateTime) + || logEntry == QueryLogSharedState.QueryLogEntries.QUERY_LOG_LAST_ENTRY + && newDateTime.isAfter(currentDateTime)) { + QueryLogSharedState.queryLogEntries.put(logEntry, newDateTime); + } + } + } +} diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/LogsConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/LogsConnector.java index 50e60b1c7..176730bd2 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/LogsConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/LogsConnector.java @@ -32,9 +32,4 @@ default String getDefaultFileName(boolean isAssessment, Clock clock) { return ArchiveNameUtil.getFileName(getName()); } } - - @Override - default boolean isLogConnector() { - return true; - } } diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/AbstractTeradataConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/AbstractTeradataConnector.java index 6c9d57112..eef7e6aa2 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/AbstractTeradataConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/AbstractTeradataConnector.java @@ -45,7 +45,6 @@ import java.sql.PreparedStatement; import java.sql.SQLException; import java.time.ZoneOffset; -import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.List; import java.util.Optional; @@ -119,11 +118,6 @@ public String getDescription() { } } - public enum TeradataQueryLogEntries { - QUERY_LOG_FIRST_ENTRY, - QUERY_LOG_LAST_ENTRY - } - protected static class TeradataJdbcSelectTask extends JdbcSelectTask { private final String sqlCount; @@ -181,11 +175,6 @@ protected static class SharedState { protected final ConcurrentMap expressionValidity = new ConcurrentHashMap<>(); } - public static class TeradataExternalSharedState { - public static final ConcurrentMap queryLogEntries = - new ConcurrentHashMap<>(); - } - protected static boolean isQueryTable(@Nonnull String expression) { return expression.startsWith("ST."); } diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/Teradata14LogsConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/Teradata14LogsConnector.java index a0df5db24..d851d6a2a 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/Teradata14LogsConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/Teradata14LogsConnector.java @@ -23,6 +23,8 @@ import com.google.common.io.ByteSink; import com.google.edwmigration.dumper.application.dumper.ConnectorArguments; import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException; +import com.google.edwmigration.dumper.application.dumper.QueryLogSharedState; +import com.google.edwmigration.dumper.application.dumper.QueryLogSharedState.QueryLogEntries; import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentQueryLogDays; import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentQueryLogEnd; import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentQueryLogStart; @@ -86,6 +88,16 @@ public Teradata14LogsConnector() { super("teradata14-logs"); } + /* + * Overriding it only for Teradata logs connectors, so in MetadataDumper summary + * section + * only them can output first and last entries of query logs for now + */ + @Override + public boolean isLogConnector() { + return true; + } + private abstract static class Teradata14LogsJdbcTask extends AbstractJdbcTask { protected static String EXPRESSION_VALIDITY_QUERY = "SELECT TOP 1 %s FROM %s"; @@ -133,7 +145,14 @@ protected Summary doInConnection( throws SQLException { String sql = getSql(jdbcHandle); ResultSetExtractor rse = newCsvResultSetExtractor(sink); - return doSelect(connection, withInterval(rse, interval), sql); + Summary summary = doSelect(connection, withInterval(rse, interval), sql); + if (summary != null && summary.rowCount() > 0) { + QueryLogSharedState.updateQueryLogEntries( + QueryLogEntries.QUERY_LOG_FIRST_ENTRY, interval.getStart()); + QueryLogSharedState.updateQueryLogEntries( + QueryLogEntries.QUERY_LOG_LAST_ENTRY, interval.getEndExclusive()); + } + return summary; } @Nonnull diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsConnector.java index 0395147d1..7735819c2 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsConnector.java @@ -175,6 +175,15 @@ public TeradataLogsConnector() { super("teradata-logs"); } + /* + * Overriding it only for Teradata logs connectors, so in MetadataDumper summary section + * only them can output first and last entries of query logs for now + */ + @Override + public boolean isLogConnector() { + return true; + } + private ImmutableList createTimeSeriesTasks( ZonedInterval interval, @Nonnull ConnectorArguments arguments) { return TIME_SERIES_PROPERTY_TO_FILENAME_PREFIX_MAP.keySet().stream() diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsJdbcTask.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsJdbcTask.java index 24aec671e..1f1b599c6 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsJdbcTask.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsJdbcTask.java @@ -31,10 +31,10 @@ import com.google.common.collect.ImmutableSet; import com.google.common.io.ByteSink; import com.google.common.primitives.Ints; +import com.google.edwmigration.dumper.application.dumper.QueryLogSharedState; +import com.google.edwmigration.dumper.application.dumper.QueryLogSharedState.QueryLogEntries; import com.google.edwmigration.dumper.application.dumper.connector.ZonedInterval; import com.google.edwmigration.dumper.application.dumper.connector.teradata.AbstractTeradataConnector.SharedState; -import com.google.edwmigration.dumper.application.dumper.connector.teradata.AbstractTeradataConnector.TeradataExternalSharedState; -import com.google.edwmigration.dumper.application.dumper.connector.teradata.AbstractTeradataConnector.TeradataQueryLogEntries; import com.google.edwmigration.dumper.application.dumper.connector.teradata.query.model.Expression; import com.google.edwmigration.dumper.application.dumper.handle.JdbcHandle; import com.google.edwmigration.dumper.application.dumper.task.AbstractJdbcTask; @@ -44,7 +44,6 @@ import java.sql.Connection; import java.sql.SQLException; import java.time.ZoneOffset; -import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.List; import java.util.Optional; @@ -162,10 +161,10 @@ protected Summary doInConnection( ResultSetExtractor rse = newCsvResultSetExtractor(sink); Summary summary = doSelect(connection, withInterval(rse, interval), sql); if (summary != null && summary.rowCount() > 0) { - updateTeradaQueryLogEntries( - TeradataQueryLogEntries.QUERY_LOG_FIRST_ENTRY, interval.getStart()); - updateTeradaQueryLogEntries( - TeradataQueryLogEntries.QUERY_LOG_LAST_ENTRY, interval.getEndExclusive()); + QueryLogSharedState.updateQueryLogEntries( + QueryLogEntries.QUERY_LOG_FIRST_ENTRY, interval.getStart()); + QueryLogSharedState.updateQueryLogEntries( + QueryLogEntries.QUERY_LOG_LAST_ENTRY, interval.getEndExclusive()); } return summary; } @@ -324,22 +323,4 @@ public String describeSourceData() { "from tables '%s' and '%s'", tableNames.queryLogsTableName(), tableNames.sqlLogsTableName())); } - - /* - * Calculates first and last query log entries, by applying 'min' and 'max' logic. - */ - private void updateTeradaQueryLogEntries( - TeradataQueryLogEntries logEntry, ZonedDateTime newDateTime) { - ZonedDateTime currentDateTime = TeradataExternalSharedState.queryLogEntries.get(logEntry); - if (currentDateTime == null) { - TeradataExternalSharedState.queryLogEntries.put(logEntry, newDateTime); - } else { - if (logEntry == TeradataQueryLogEntries.QUERY_LOG_FIRST_ENTRY - && newDateTime.isBefore(currentDateTime) - || logEntry == TeradataQueryLogEntries.QUERY_LOG_LAST_ENTRY - && newDateTime.isAfter(currentDateTime)) { - TeradataExternalSharedState.queryLogEntries.put(logEntry, newDateTime); - } - } - } } From e05908b9c106fb06ce2dc0ca7cc2936d3c39f976 Mon Sep 17 00:00:00 2001 From: Sanan Yuzbashiyev Date: Thu, 3 Oct 2024 11:20:13 +0000 Subject: [PATCH 04/12] Refactoring --- .../dumper/application/dumper/MetadataDumper.java | 7 ++----- .../application/dumper/QueryLogSharedState.java | 12 +++++------- .../connector/teradata/Teradata14LogsConnector.java | 6 +++--- .../connector/teradata/TeradataLogsJdbcTask.java | 6 +++--- 4 files changed, 13 insertions(+), 18 deletions(-) diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/MetadataDumper.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/MetadataDumper.java index ec067639a..a7d5cfb68 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/MetadataDumper.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/MetadataDumper.java @@ -267,9 +267,6 @@ private boolean checkRequiredTaskSuccess( return true; } - /* - * This logic works for only Teradata logs connector for now - */ private void outputFirstAndLastQueryLogEnries(SummaryLinePrinter linePrinter) { if (QueryLogSharedState.queryLogEntries.size() == 0) { @@ -279,9 +276,9 @@ private void outputFirstAndLastQueryLogEnries(SummaryLinePrinter linePrinter) { linePrinter.println( "First query log entry is '%s' and last query log entry is '%s'", QueryLogSharedState.queryLogEntries.get( - QueryLogSharedState.QueryLogEntries.QUERY_LOG_FIRST_ENTRY), + QueryLogSharedState.QueryLogEntry.QUERY_LOG_FIRST_ENTRY), QueryLogSharedState.queryLogEntries.get( - QueryLogSharedState.QueryLogEntries.QUERY_LOG_LAST_ENTRY)); + QueryLogSharedState.QueryLogEntry.QUERY_LOG_LAST_ENTRY)); } private void logFinalSummary( diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedState.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedState.java index b329f0284..0ba43e570 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedState.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedState.java @@ -21,10 +21,10 @@ import java.util.concurrent.ConcurrentMap; public class QueryLogSharedState { - public static final ConcurrentMap queryLogEntries = + public static final ConcurrentMap queryLogEntries = new ConcurrentHashMap<>(); - public enum QueryLogEntries { + public enum QueryLogEntry { QUERY_LOG_FIRST_ENTRY, QUERY_LOG_LAST_ENTRY } @@ -32,15 +32,13 @@ public enum QueryLogEntries { /* * Calculates first and last query log entries, by applying 'min' and 'max' logic. */ - public static void updateQueryLogEntries( - QueryLogSharedState.QueryLogEntries logEntry, ZonedDateTime newDateTime) { + public static void updateQueryLogEntries(QueryLogEntry logEntry, ZonedDateTime newDateTime) { ZonedDateTime currentDateTime = QueryLogSharedState.queryLogEntries.get(logEntry); if (currentDateTime == null) { QueryLogSharedState.queryLogEntries.put(logEntry, newDateTime); } else { - if (logEntry == QueryLogSharedState.QueryLogEntries.QUERY_LOG_FIRST_ENTRY - && newDateTime.isBefore(currentDateTime) - || logEntry == QueryLogSharedState.QueryLogEntries.QUERY_LOG_LAST_ENTRY + if (logEntry == QueryLogEntry.QUERY_LOG_FIRST_ENTRY && newDateTime.isBefore(currentDateTime) + || logEntry == QueryLogEntry.QUERY_LOG_LAST_ENTRY && newDateTime.isAfter(currentDateTime)) { QueryLogSharedState.queryLogEntries.put(logEntry, newDateTime); } diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/Teradata14LogsConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/Teradata14LogsConnector.java index d851d6a2a..540481455 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/Teradata14LogsConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/Teradata14LogsConnector.java @@ -24,7 +24,7 @@ import com.google.edwmigration.dumper.application.dumper.ConnectorArguments; import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException; import com.google.edwmigration.dumper.application.dumper.QueryLogSharedState; -import com.google.edwmigration.dumper.application.dumper.QueryLogSharedState.QueryLogEntries; +import com.google.edwmigration.dumper.application.dumper.QueryLogSharedState.QueryLogEntry; import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentQueryLogDays; import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentQueryLogEnd; import com.google.edwmigration.dumper.application.dumper.annotations.RespectsArgumentQueryLogStart; @@ -148,9 +148,9 @@ protected Summary doInConnection( Summary summary = doSelect(connection, withInterval(rse, interval), sql); if (summary != null && summary.rowCount() > 0) { QueryLogSharedState.updateQueryLogEntries( - QueryLogEntries.QUERY_LOG_FIRST_ENTRY, interval.getStart()); + QueryLogEntry.QUERY_LOG_FIRST_ENTRY, interval.getStart()); QueryLogSharedState.updateQueryLogEntries( - QueryLogEntries.QUERY_LOG_LAST_ENTRY, interval.getEndExclusive()); + QueryLogEntry.QUERY_LOG_LAST_ENTRY, interval.getEndExclusive()); } return summary; } diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsJdbcTask.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsJdbcTask.java index 1f1b599c6..d91fcdd93 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsJdbcTask.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsJdbcTask.java @@ -32,7 +32,7 @@ import com.google.common.io.ByteSink; import com.google.common.primitives.Ints; import com.google.edwmigration.dumper.application.dumper.QueryLogSharedState; -import com.google.edwmigration.dumper.application.dumper.QueryLogSharedState.QueryLogEntries; +import com.google.edwmigration.dumper.application.dumper.QueryLogSharedState.QueryLogEntry; import com.google.edwmigration.dumper.application.dumper.connector.ZonedInterval; import com.google.edwmigration.dumper.application.dumper.connector.teradata.AbstractTeradataConnector.SharedState; import com.google.edwmigration.dumper.application.dumper.connector.teradata.query.model.Expression; @@ -162,9 +162,9 @@ protected Summary doInConnection( Summary summary = doSelect(connection, withInterval(rse, interval), sql); if (summary != null && summary.rowCount() > 0) { QueryLogSharedState.updateQueryLogEntries( - QueryLogEntries.QUERY_LOG_FIRST_ENTRY, interval.getStart()); + QueryLogEntry.QUERY_LOG_FIRST_ENTRY, interval.getStart()); QueryLogSharedState.updateQueryLogEntries( - QueryLogEntries.QUERY_LOG_LAST_ENTRY, interval.getEndExclusive()); + QueryLogEntry.QUERY_LOG_LAST_ENTRY, interval.getEndExclusive()); } return summary; } From 8b5fbcc355d4d098ca42784d029f5a93b75dbdd7 Mon Sep 17 00:00:00 2001 From: Sanan Yuzbashiyev Date: Thu, 3 Oct 2024 11:25:47 +0000 Subject: [PATCH 05/12] Small refactoring --- .../dumper/connector/teradata/Teradata14LogsConnector.java | 3 +-- .../dumper/connector/teradata/TeradataLogsConnector.java | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/Teradata14LogsConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/Teradata14LogsConnector.java index 540481455..ab08769a0 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/Teradata14LogsConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/Teradata14LogsConnector.java @@ -90,8 +90,7 @@ public Teradata14LogsConnector() { /* * Overriding it only for Teradata logs connectors, so in MetadataDumper summary - * section - * only them can output first and last entries of query logs for now + * section only they can output first and last entries of query logs for now */ @Override public boolean isLogConnector() { diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsConnector.java index 7735819c2..a8402c91f 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsConnector.java @@ -177,7 +177,7 @@ public TeradataLogsConnector() { /* * Overriding it only for Teradata logs connectors, so in MetadataDumper summary section - * only them can output first and last entries of query logs for now + * only they can output first and last entries of query logs for now */ @Override public boolean isLogConnector() { From d8f2027e3b130868cd433093417f3996cf5a5572 Mon Sep 17 00:00:00 2001 From: Sanan Yuzbashiyev Date: Thu, 3 Oct 2024 14:06:35 +0000 Subject: [PATCH 06/12] Add tests for QueryLogSharedState --- .../application/dumper/MetadataDumper.java | 17 +++- .../dumper/QueryLogSharedState.java | 6 ++ .../dumper/QueryLogSharedStateTest.java | 94 +++++++++++++++++++ 3 files changed, 112 insertions(+), 5 deletions(-) create mode 100644 dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedStateTest.java diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/MetadataDumper.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/MetadataDumper.java index a7d5cfb68..f6e482d6b 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/MetadataDumper.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/MetadataDumper.java @@ -44,6 +44,8 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.time.Clock; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -62,6 +64,9 @@ public class MetadataDumper { private static final Pattern GCS_PATH_PATTERN = Pattern.compile("gs://(?[^/]+)/(?.*)"); + private static DateTimeFormatter OUTPUT_DATE_FORMAT = + DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm").withZone(ZoneOffset.UTC); + public boolean run(String... args) throws Exception { ConnectorArguments arguments = new ConnectorArguments(JsonResponseFile.addResponseFiles(args)); try { @@ -274,11 +279,13 @@ private void outputFirstAndLastQueryLogEnries(SummaryLinePrinter linePrinter) { } linePrinter.println( - "First query log entry is '%s' and last query log entry is '%s'", - QueryLogSharedState.queryLogEntries.get( - QueryLogSharedState.QueryLogEntry.QUERY_LOG_FIRST_ENTRY), - QueryLogSharedState.queryLogEntries.get( - QueryLogSharedState.QueryLogEntry.QUERY_LOG_LAST_ENTRY)); + "The first query log entry is '%s' and the last query log entry is '%s'", + QueryLogSharedState.queryLogEntries + .get(QueryLogSharedState.QueryLogEntry.QUERY_LOG_FIRST_ENTRY) + .format(OUTPUT_DATE_FORMAT), + QueryLogSharedState.queryLogEntries + .get(QueryLogSharedState.QueryLogEntry.QUERY_LOG_LAST_ENTRY) + .format(OUTPUT_DATE_FORMAT)); } private void logFinalSummary( diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedState.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedState.java index 0ba43e570..7f009f11a 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedState.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedState.java @@ -16,6 +16,7 @@ */ package com.google.edwmigration.dumper.application.dumper; +import com.google.common.annotations.VisibleForTesting; import java.time.ZonedDateTime; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -44,4 +45,9 @@ public static void updateQueryLogEntries(QueryLogEntry logEntry, ZonedDateTime n } } } + + @VisibleForTesting + static void clearQueryLogEntries() { + queryLogEntries.clear(); + } } diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedStateTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedStateTest.java new file mode 100644 index 000000000..b623dc0ef --- /dev/null +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedStateTest.java @@ -0,0 +1,94 @@ +/* + * Copyright 2022-2024 Google LLC + * Copyright 2013-2021 CompilerWorks + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.edwmigration.dumper.application.dumper; + +import static org.junit.Assert.assertEquals; + +import com.google.edwmigration.dumper.application.dumper.QueryLogSharedState.QueryLogEntry; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class QueryLogSharedStateTest { + + @Before + public void beforeEachTest() { + QueryLogSharedState.clearQueryLogEntries(); + } + + @Test + public void queryLogFirstEntryUpdatedSuccessfully() { + // Arrange + ZonedDateTime newQueryLogDate = ZonedDateTime.now(); + + // Act + QueryLogSharedState.updateQueryLogEntries(QueryLogEntry.QUERY_LOG_FIRST_ENTRY, newQueryLogDate); + + // Assert + assertEquals( + newQueryLogDate, + QueryLogSharedState.queryLogEntries.get(QueryLogEntry.QUERY_LOG_FIRST_ENTRY)); + } + + @Test + public void queryLogLastEntryUpdatedSuccessfully() { + // Arrange + ZonedDateTime newQueryLogDate = ZonedDateTime.now(); + + // Act + QueryLogSharedState.updateQueryLogEntries(QueryLogEntry.QUERY_LOG_LAST_ENTRY, newQueryLogDate); + + // Assert + assertEquals( + newQueryLogDate, + QueryLogSharedState.queryLogEntries.get(QueryLogEntry.QUERY_LOG_LAST_ENTRY)); + } + + @Test + public void queryLogFirstEntryUpdatedSuccessfullyForEarlierDate() { + // Arrange + ZonedDateTime now = ZonedDateTime.now(); + ZonedDateTime olderDate = ZonedDateTime.of(1970, 1, 1, 1, 1, 1, 1, ZoneId.of("UTC")); + + // Act + QueryLogSharedState.updateQueryLogEntries(QueryLogEntry.QUERY_LOG_FIRST_ENTRY, now); + QueryLogSharedState.updateQueryLogEntries(QueryLogEntry.QUERY_LOG_FIRST_ENTRY, olderDate); + + // Assert + assertEquals( + olderDate, QueryLogSharedState.queryLogEntries.get(QueryLogEntry.QUERY_LOG_FIRST_ENTRY)); + } + + @Test + public void queryLogFirstEntryUpdatedSuccessfullyForLaterDate() { + // Arrange + ZonedDateTime date = ZonedDateTime.of(2000, 1, 1, 1, 1, 1, 1, ZoneId.of("UTC")); + ZonedDateTime laterDate = ZonedDateTime.now(); + + // Act + QueryLogSharedState.updateQueryLogEntries(QueryLogEntry.QUERY_LOG_LAST_ENTRY, date); + QueryLogSharedState.updateQueryLogEntries(QueryLogEntry.QUERY_LOG_LAST_ENTRY, laterDate); + + // Assert + assertEquals( + laterDate, QueryLogSharedState.queryLogEntries.get(QueryLogEntry.QUERY_LOG_LAST_ENTRY)); + } +} From def13ebc1573e198ddecb8a0139757121146ce64 Mon Sep 17 00:00:00 2001 From: Sanan Yuzbashiyev Date: Thu, 3 Oct 2024 14:13:52 +0000 Subject: [PATCH 07/12] Fix test name --- .../dumper/application/dumper/QueryLogSharedStateTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedStateTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedStateTest.java index b623dc0ef..bf036b938 100644 --- a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedStateTest.java +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedStateTest.java @@ -78,7 +78,7 @@ public void queryLogFirstEntryUpdatedSuccessfullyForEarlierDate() { } @Test - public void queryLogFirstEntryUpdatedSuccessfullyForLaterDate() { + public void queryLogLastEntryUpdatedSuccessfullyForLaterDate() { // Arrange ZonedDateTime date = ZonedDateTime.of(2000, 1, 1, 1, 1, 1, 1, ZoneId.of("UTC")); ZonedDateTime laterDate = ZonedDateTime.now(); From fc6524d57edda33dc6e574e4fd9fb4a221ec1782 Mon Sep 17 00:00:00 2001 From: Sanan Yuzbashiyev Date: Fri, 4 Oct 2024 12:45:01 +0000 Subject: [PATCH 08/12] Small refactoring --- .../dumper/application/dumper/QueryLogSharedState.java | 6 +++--- .../dumper/application/dumper/QueryLogSharedStateTest.java | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedState.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedState.java index 7f009f11a..aa2fbf950 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedState.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedState.java @@ -38,9 +38,9 @@ public static void updateQueryLogEntries(QueryLogEntry logEntry, ZonedDateTime n if (currentDateTime == null) { QueryLogSharedState.queryLogEntries.put(logEntry, newDateTime); } else { - if (logEntry == QueryLogEntry.QUERY_LOG_FIRST_ENTRY && newDateTime.isBefore(currentDateTime) - || logEntry == QueryLogEntry.QUERY_LOG_LAST_ENTRY - && newDateTime.isAfter(currentDateTime)) { + if ((logEntry == QueryLogEntry.QUERY_LOG_FIRST_ENTRY && newDateTime.isBefore(currentDateTime)) + || (logEntry == QueryLogEntry.QUERY_LOG_LAST_ENTRY + && newDateTime.isAfter(currentDateTime))) { QueryLogSharedState.queryLogEntries.put(logEntry, newDateTime); } } diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedStateTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedStateTest.java index bf036b938..651795eae 100644 --- a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedStateTest.java +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedStateTest.java @@ -66,15 +66,15 @@ public void queryLogLastEntryUpdatedSuccessfully() { public void queryLogFirstEntryUpdatedSuccessfullyForEarlierDate() { // Arrange ZonedDateTime now = ZonedDateTime.now(); - ZonedDateTime olderDate = ZonedDateTime.of(1970, 1, 1, 1, 1, 1, 1, ZoneId.of("UTC")); + ZonedDateTime earlierDate = ZonedDateTime.of(1970, 1, 1, 1, 1, 1, 1, ZoneId.of("UTC")); // Act QueryLogSharedState.updateQueryLogEntries(QueryLogEntry.QUERY_LOG_FIRST_ENTRY, now); - QueryLogSharedState.updateQueryLogEntries(QueryLogEntry.QUERY_LOG_FIRST_ENTRY, olderDate); + QueryLogSharedState.updateQueryLogEntries(QueryLogEntry.QUERY_LOG_FIRST_ENTRY, earlierDate); // Assert assertEquals( - olderDate, QueryLogSharedState.queryLogEntries.get(QueryLogEntry.QUERY_LOG_FIRST_ENTRY)); + earlierDate, QueryLogSharedState.queryLogEntries.get(QueryLogEntry.QUERY_LOG_FIRST_ENTRY)); } @Test From f152bcda98529eeccdeb6087ff6b4244e6bc7ec3 Mon Sep 17 00:00:00 2001 From: Sanan Yuzbashiyev Date: Sat, 5 Oct 2024 11:09:38 +0000 Subject: [PATCH 09/12] Add UTC in ouput message --- .../edwmigration/dumper/application/dumper/MetadataDumper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/MetadataDumper.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/MetadataDumper.java index f6e482d6b..fa366ffbe 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/MetadataDumper.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/MetadataDumper.java @@ -279,7 +279,7 @@ private void outputFirstAndLastQueryLogEnries(SummaryLinePrinter linePrinter) { } linePrinter.println( - "The first query log entry is '%s' and the last query log entry is '%s'", + "The first query log entry is '%s' UTC and the last query log entry is '%s' UTC", QueryLogSharedState.queryLogEntries .get(QueryLogSharedState.QueryLogEntry.QUERY_LOG_FIRST_ENTRY) .format(OUTPUT_DATE_FORMAT), From fbb91aafc5dd655253deca58030d7d136dcce073 Mon Sep 17 00:00:00 2001 From: Sanan Yuzbashiyev Date: Mon, 7 Oct 2024 15:58:47 +0000 Subject: [PATCH 10/12] Remove Arrange comment --- .../dumper/application/dumper/QueryLogSharedStateTest.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedStateTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedStateTest.java index 651795eae..299ecf34a 100644 --- a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedStateTest.java +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedStateTest.java @@ -36,7 +36,6 @@ public void beforeEachTest() { @Test public void queryLogFirstEntryUpdatedSuccessfully() { - // Arrange ZonedDateTime newQueryLogDate = ZonedDateTime.now(); // Act @@ -50,7 +49,6 @@ public void queryLogFirstEntryUpdatedSuccessfully() { @Test public void queryLogLastEntryUpdatedSuccessfully() { - // Arrange ZonedDateTime newQueryLogDate = ZonedDateTime.now(); // Act @@ -64,7 +62,6 @@ public void queryLogLastEntryUpdatedSuccessfully() { @Test public void queryLogFirstEntryUpdatedSuccessfullyForEarlierDate() { - // Arrange ZonedDateTime now = ZonedDateTime.now(); ZonedDateTime earlierDate = ZonedDateTime.of(1970, 1, 1, 1, 1, 1, 1, ZoneId.of("UTC")); @@ -79,7 +76,6 @@ public void queryLogFirstEntryUpdatedSuccessfullyForEarlierDate() { @Test public void queryLogLastEntryUpdatedSuccessfullyForLaterDate() { - // Arrange ZonedDateTime date = ZonedDateTime.of(2000, 1, 1, 1, 1, 1, 1, ZoneId.of("UTC")); ZonedDateTime laterDate = ZonedDateTime.now(); From 3113913bffdc93497f83f7e45643f3ab98f2b3ea Mon Sep 17 00:00:00 2001 From: Sanan Yuzbashiyev Date: Tue, 8 Oct 2024 14:55:55 +0000 Subject: [PATCH 11/12] Incapsulating hash map methods. Renaming logs connector method --- .../application/dumper/MetadataDumper.java | 31 +++++++++++++------ .../dumper/QueryLogSharedState.java | 10 +++++- .../dumper/connector/Connector.java | 2 +- .../teradata/Teradata14LogsConnector.java | 2 +- .../teradata/TeradataLogsConnector.java | 2 +- .../dumper/QueryLogSharedStateTest.java | 10 +++--- 6 files changed, 37 insertions(+), 20 deletions(-) diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/MetadataDumper.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/MetadataDumper.java index fa366ffbe..1e50c54de 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/MetadataDumper.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/MetadataDumper.java @@ -45,6 +45,7 @@ import java.nio.file.Paths; import java.time.Clock; import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.List; @@ -274,18 +275,28 @@ private boolean checkRequiredTaskSuccess( private void outputFirstAndLastQueryLogEnries(SummaryLinePrinter linePrinter) { - if (QueryLogSharedState.queryLogEntries.size() == 0) { + if (QueryLogSharedState.sizeOfQueryLogEntries() == 0) { return; } - linePrinter.println( - "The first query log entry is '%s' UTC and the last query log entry is '%s' UTC", - QueryLogSharedState.queryLogEntries - .get(QueryLogSharedState.QueryLogEntry.QUERY_LOG_FIRST_ENTRY) - .format(OUTPUT_DATE_FORMAT), - QueryLogSharedState.queryLogEntries - .get(QueryLogSharedState.QueryLogEntry.QUERY_LOG_LAST_ENTRY) - .format(OUTPUT_DATE_FORMAT)); + ZonedDateTime queryLogFirstEntry = + QueryLogSharedState.getQueryLogEntry( + QueryLogSharedState.QueryLogEntry.QUERY_LOG_FIRST_ENTRY); + + ZonedDateTime queryLogLastEntry = + QueryLogSharedState.getQueryLogEntry( + QueryLogSharedState.QueryLogEntry.QUERY_LOG_LAST_ENTRY); + + if (queryLogFirstEntry != null && queryLogLastEntry != null) { + linePrinter.println( + "The first query log entry is '%s' UTC and the last query log entry is '%s' UTC", + QueryLogSharedState.getQueryLogEntry( + QueryLogSharedState.QueryLogEntry.QUERY_LOG_FIRST_ENTRY) + .format(OUTPUT_DATE_FORMAT), + QueryLogSharedState.getQueryLogEntry( + QueryLogSharedState.QueryLogEntry.QUERY_LOG_LAST_ENTRY) + .format(OUTPUT_DATE_FORMAT)); + } } private void logFinalSummary( @@ -306,7 +317,7 @@ private void logFinalSummary( .map(taskReport -> taskReport.count() + " " + taskReport.state()) .collect(joining(", "))); // For now, it will return true only for TeradataLogsConnector and Terada14LogsConnector - if (connector.isLogConnector()) { + if (connector.shouldOutputFirstAndLastQueryLog()) { outputFirstAndLastQueryLogEnries(linePrinter); } if (requiredTaskSucceeded) { diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedState.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedState.java index aa2fbf950..18accb3d7 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedState.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedState.java @@ -22,7 +22,7 @@ import java.util.concurrent.ConcurrentMap; public class QueryLogSharedState { - public static final ConcurrentMap queryLogEntries = + private static final ConcurrentMap queryLogEntries = new ConcurrentHashMap<>(); public enum QueryLogEntry { @@ -30,6 +30,14 @@ public enum QueryLogEntry { QUERY_LOG_LAST_ENTRY } + public static ZonedDateTime getQueryLogEntry(QueryLogEntry queryLogEntry) { + return queryLogEntries.getOrDefault(queryLogEntry, null); + } + + public static int sizeOfQueryLogEntries() { + return queryLogEntries.size(); + } + /* * Calculates first and last query log entries, by applying 'min' and 'max' logic. */ diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/Connector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/Connector.java index 793a345b8..2ee61016f 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/Connector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/Connector.java @@ -46,7 +46,7 @@ public default Class> getConnectorPr return DefaultProperties.class; } - default boolean isLogConnector() { + default boolean shouldOutputFirstAndLastQueryLog() { return false; } } diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/Teradata14LogsConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/Teradata14LogsConnector.java index ab08769a0..bfb5e1c26 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/Teradata14LogsConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/Teradata14LogsConnector.java @@ -93,7 +93,7 @@ public Teradata14LogsConnector() { * section only they can output first and last entries of query logs for now */ @Override - public boolean isLogConnector() { + public boolean shouldOutputFirstAndLastQueryLog() { return true; } diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsConnector.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsConnector.java index a8402c91f..081555c71 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsConnector.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/teradata/TeradataLogsConnector.java @@ -180,7 +180,7 @@ public TeradataLogsConnector() { * only they can output first and last entries of query logs for now */ @Override - public boolean isLogConnector() { + public boolean shouldOutputFirstAndLastQueryLog() { return true; } diff --git a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedStateTest.java b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedStateTest.java index 299ecf34a..ec03d25e5 100644 --- a/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedStateTest.java +++ b/dumper/app/src/test/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedStateTest.java @@ -43,8 +43,7 @@ public void queryLogFirstEntryUpdatedSuccessfully() { // Assert assertEquals( - newQueryLogDate, - QueryLogSharedState.queryLogEntries.get(QueryLogEntry.QUERY_LOG_FIRST_ENTRY)); + newQueryLogDate, QueryLogSharedState.getQueryLogEntry(QueryLogEntry.QUERY_LOG_FIRST_ENTRY)); } @Test @@ -56,8 +55,7 @@ public void queryLogLastEntryUpdatedSuccessfully() { // Assert assertEquals( - newQueryLogDate, - QueryLogSharedState.queryLogEntries.get(QueryLogEntry.QUERY_LOG_LAST_ENTRY)); + newQueryLogDate, QueryLogSharedState.getQueryLogEntry(QueryLogEntry.QUERY_LOG_LAST_ENTRY)); } @Test @@ -71,7 +69,7 @@ public void queryLogFirstEntryUpdatedSuccessfullyForEarlierDate() { // Assert assertEquals( - earlierDate, QueryLogSharedState.queryLogEntries.get(QueryLogEntry.QUERY_LOG_FIRST_ENTRY)); + earlierDate, QueryLogSharedState.getQueryLogEntry(QueryLogEntry.QUERY_LOG_FIRST_ENTRY)); } @Test @@ -85,6 +83,6 @@ public void queryLogLastEntryUpdatedSuccessfullyForLaterDate() { // Assert assertEquals( - laterDate, QueryLogSharedState.queryLogEntries.get(QueryLogEntry.QUERY_LOG_LAST_ENTRY)); + laterDate, QueryLogSharedState.getQueryLogEntry(QueryLogEntry.QUERY_LOG_LAST_ENTRY)); } } From 283fb248196e96c32f2c69ef676fc4ca5ab7c586 Mon Sep 17 00:00:00 2001 From: Sanan Yuzbashiyev Date: Tue, 8 Oct 2024 15:06:59 +0000 Subject: [PATCH 12/12] Add comment explaining the usage of shared class. --- .../dumper/application/dumper/QueryLogSharedState.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedState.java b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedState.java index 18accb3d7..a8dda1bd9 100644 --- a/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedState.java +++ b/dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/QueryLogSharedState.java @@ -21,6 +21,11 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +/* + * Shared state class for calculating first and last query log entries. + * Class could be used inside different tasks, which run concurrently, + * so class designed to be thread-safe, by using ConcurrentMap. + */ public class QueryLogSharedState { private static final ConcurrentMap queryLogEntries = new ConcurrentHashMap<>();