From 26225c1eb43dcd986949adf3ae3abdbf8dedd321 Mon Sep 17 00:00:00 2001 From: PatrickHSI Date: Thu, 4 Apr 2024 12:46:30 -0400 Subject: [PATCH 1/6] testing data regen strategies --- common/src/main/resources/reference.conf | 54 +++--- .../repository/ModifiedLarRepository.scala | 14 +- .../src/main/resources/application.conf | 158 ++++++------------ .../src/main/resources/application.conf | 4 +- .../src/main/resources/application.conf | 8 +- 5 files changed, 86 insertions(+), 152 deletions(-) diff --git a/common/src/main/resources/reference.conf b/common/src/main/resources/reference.conf index ebbedb3709..a70f3c6d39 100644 --- a/common/src/main/resources/reference.conf +++ b/common/src/main/resources/reference.conf @@ -36,13 +36,11 @@ hmda { rules { yearly-filing { - years-allowed = "2018,2019,2020,2021,2022,2023,2024" - years-allowed = ${?RULES_YEARLY_FILING_YEARS_ALLOWED} - } + years-allowed = "2020,2021,2022,2023" + } quarterly-filing { - years-allowed = "2020,2021,2022,2023,2024" - years-allowed = ${?RULES_QF_FILING_YEARS_ALLOWED} + years-allowed = "2020,2021,2022,2023" q1 { start = "April 01" @@ -255,37 +253,23 @@ hmda { endpoint = ${?KAFKA_SSL_ENDPOINT} } topics { - institutionTopic = "institution" - institutionTopic = ${?KAFKA_INSTITUTIONS_TOPIC} - signTopic = "hmda-sign" - signTopic = ${?KAFKA_SIGN_TOPIC} - modifiedLarTopic = "hmda-modified-lar" - modifiedLarTopic = ${?KAFKA_MODIFIED_LAR_TOPIC} - irsTopic = "hmda-irs" - irsTopic = ${?KAFKA_IRS_TOPIC} - analyticsTopic = "hmda-analytics" - analyticsTopic = ${?KAFKA_ANALYTICS_TOPIC} - disclosureTopic = "hmda-spark-disclosure" - disclosureTopic = ${?KAFKA_DISCLOSURE_TOPIC} - adTopic = "hmda-spark-ad" - adTopic = ${?KAFKA_AD_TOPIC} - emailTopic = "hmda-email" - emailTopic = ${?KAFKA_EMAIL_TOPIC} - } + institutionTopic = "institution-regen" + signTopic = "hmda-sign-regen" + modifiedLarTopic = "hmda-modified-lar-regen" + irsTopic = "hmda-irs-regen" + analyticsTopic = "hmda-analytics-regen" + disclosureTopic = "hmda-spark-disclosure-regen" + adTopic = "hmda-spark-ad-regen" + emailTopic = "hmda-email-regen" + } groups { - emailGroup = "email-group" - emailGroup = ${?KAFKA_EMAIL_GROUP} - modifiedLarGroup = "modified-lar-group" - modifiedLarGroup = ${?KAFKA_MODIFIED_LAR_GROUP} - analyticsGroup = "analytics-group" - analyticsGroup = ${?KAFKA_ANALYTICS_GROUP} - irsGroup = "irs-group" - irsGroup = ${?KAFKA_IRS_GROUP} - institutionsGroup = "institutions-group" - institutionsGroup = ${?KAFKA_INSTITUTIONS_GROUP} - submissionErrorsGroup = "triggered-quality-edits-group" - submissionErrorsGroup = ${?TRIGGERED_QUALITY_EDITS_GROUP} - } + emailGroup = "email-group-regen" + modifiedLarGroup = "modified-lar-group-regen" + analyticsGroup = "analytics-group-regen" + irsGroup = "irs-group-regen" + institutionsGroup = "institutions-group-regen" + submissionErrorsGroup = "triggered-quality-edits-group-regen" + } } } diff --git a/common/src/main/scala/hmda/query/repository/ModifiedLarRepository.scala b/common/src/main/scala/hmda/query/repository/ModifiedLarRepository.scala index ef6300db62..ecd84e3376 100644 --- a/common/src/main/scala/hmda/query/repository/ModifiedLarRepository.scala +++ b/common/src/main/scala/hmda/query/repository/ModifiedLarRepository.scala @@ -18,14 +18,14 @@ class ModifiedLarRepository(databaseConfig: DatabaseConfig[JdbcProfile]) { def fetchYearTable(year: Int): String = year match { - case 2018 => "modifiedlar2018" - case 2019 => "modifiedlar2019" - case 2020 => "modifiedlar2020" - case 2021 => "modifiedlar2021" - case 2022 => "modifiedlar2022" - case 2023 => "modifiedlar2023" + case 2018 => "modifiedlar2018_regen_2024" + case 2019 => "modifiedlar2019_regen_2024" + case 2020 => "modifiedlar2020_regen_2024" + case 2021 => "modifiedlar2021_regen_2024" + case 2022 => "modifiedlar2022_regen_2024" + case 2023 => "modifiedlar2023_regen_2024" - case _ => "modifiedlar2021" + case _ => "modifiedlar2021_regen_2024" } /** diff --git a/hmda-analytics/src/main/resources/application.conf b/hmda-analytics/src/main/resources/application.conf index 3cc1f5c774..a4be47c161 100644 --- a/hmda-analytics/src/main/resources/application.conf +++ b/hmda-analytics/src/main/resources/application.conf @@ -63,115 +63,67 @@ hmda { historyInsertion = true historyInsertion = ${?HMDA-HISTORY-INSERT} yearsAvailable = "2018,2019,2020,2021,2022,2023,2024" - yearsAvailable = ${?ANALYTICS_YEARS_AVAILABLE} - 2024 { - tsTableName = "transmittalsheet2024" - tsTableName = ${?TS_TABLE_NAME_2024} - tsTableNameQ1 = "ts2024_q1" - tsTableNameQ1 = ${?TS_TABLE_NAME_2024_Q1} - tsTableNameQ2 = "ts2024_q2" - tsTableNameQ2 = ${?TS_TABLE_NAME_2024_Q2} - tsTableNameQ3 = "ts2024_q3" - tsTableNameQ3 = ${?TS_TABLE_NAME_2024_Q1} - larTableName = "loanapplicationregister2024" - larTableName = ${?LAR_TABLE_NAME_2024} - larTableNameQ1 = "lar2024_q1" - larTableNameQ1 = ${?LAR_TABLE_NAME_2024_Q1} - larTableNameQ2 = "lar2024_q2" - larTableNameQ2 = ${?LAR_TABLE_NAME_2024_Q2} - larTableNameQ3 = "lar2024_q3" - larTableNameQ3 = ${?LAR_TABLE_NAME_2024_Q3} - } + 2024 { + tsTableName = "transmittalsheet2024_regen_2024" + tsTableNameQ1 = "ts2024_q1_regen_2024" + tsTableNameQ2 = "ts2024_q2_regen_2024" + tsTableNameQ3 = "ts2024_q3_regen_2024" + larTableName = "loanapplicationregister2024_regen_2024" + larTableNameQ1 = "lar2024_q1_regen_2024" + larTableNameQ2 = "lar2024_q2_regen_2024" + larTableNameQ3 = "lar2024_q3_regen_2024" + } 2023 { - tsTableName = "transmittalsheet2023" - tsTableName = ${?TS_TABLE_NAME_2023} - tsTableNameQ1 = "ts2023_q1" - tsTableNameQ1 = ${?TS_TABLE_NAME_2023_Q1} - tsTableNameQ2 = "ts2023_q2" - tsTableNameQ2 = ${?TS_TABLE_NAME_2023_Q2} - tsTableNameQ3 = "ts2023_q3" - tsTableNameQ3 = ${?TS_TABLE_NAME_2023_Q1} - larTableName = "loanapplicationregister2023" - larTableName = ${?LAR_TABLE_NAME_2023} - larTableNameQ1 = "lar2023_q1" - larTableNameQ1 = ${?LAR_TABLE_NAME_2023_Q1} - larTableNameQ2 = "lar2023_q2" - larTableNameQ2 = ${?LAR_TABLE_NAME_2023_Q2} - larTableNameQ3 = "lar2023_q3" - larTableNameQ3 = ${?LAR_TABLE_NAME_2023_Q3} - } + tsTableName = "transmittalsheet2023_regen_2024" + tsTableNameQ1 = "ts2023_q1_regen_2024" + tsTableNameQ2 = "ts2023_q2_regen_2024" + tsTableNameQ3 = "ts2023_q3_regen_2024" + larTableName = "loanapplicationregister2023_regen_2024" + larTableNameQ1 = "lar2023_q1_regen_2024" + larTableNameQ2 = "lar2023_q2_regen_2024" + larTableNameQ3 = "lar2023_q3_regen_2024" + } 2022 { - tsTableName = "transmittalsheet2022" - tsTableName = ${?TS_TABLE_NAME_2022} - larTableName = "loanapplicationregister2022" - larTableName = ${?LAR_TABLE_NAME_2022} - tsTableNameQ1 = "ts2022_q1" - tsTableNameQ1 = ${?TS_TABLE_NAME_2022_Q1} - tsTableNameQ2 = "ts2022_q2" - tsTableNameQ2 = ${?TS_TABLE_NAME_2022_Q2} - tsTableNameQ3 = "ts2022_q3" - tsTableNameQ3 = ${?TS_TABLE_NAME_2022_Q1} - larTableNameQ1 = "lar2022_q1" - larTableNameQ1 = ${?LAR_TABLE_NAME_2022_Q1} - larTableNameQ2 = "lar2022_q2" - larTableNameQ2 = ${?LAR_TABLE_NAME_2022_Q2} - larTableNameQ3 = "lar2022_q3" - larTableNameQ3 = ${?LAR_TABLE_NAME_2022_Q3} - } + tsTableName = "transmittalsheet2022_regen_2024" + larTableName = "loanapplicationregister2022_regen_2024" + tsTableNameQ1 = "ts2022_q1_regen_2024" + tsTableNameQ2 = "ts2022_q2_regen_2024" + tsTableNameQ3 = "ts2022_q3_regen_2024" + larTableNameQ1 = "lar2022_q1_regen_2024" + larTableNameQ2 = "lar2022_q2_regen_2024" + larTableNameQ3 = "lar2022_q3_regen_2024" + ® } 2021 { - tsTableName = "transmittalsheet2021" - tsTableName = ${?TS_TABLE_NAME_2021} - tsTableNameQ1 = "ts2021_q1" - tsTableNameQ1 = ${?TS_TABLE_NAME_2021_Q1} - tsTableNameQ2 = "ts2021_q2" - tsTableNameQ2 = ${?TS_TABLE_NAME_2021_Q2} - tsTableNameQ3 = "ts2021_q3" - tsTableNameQ3 = ${?TS_TABLE_NAME_2021_Q1} - larTableName = "loanapplicationregister2021" - larTableName = ${?LAR_TABLE_NAME_2021} - larTableNameQ1 = "lar2021_q1" - larTableNameQ1 = ${?LAR_TABLE_NAME_2021_Q1} - larTableNameQ2 = "lar2021_q2" - larTableNameQ2 = ${?LAR_TABLE_NAME_2021_Q2} - larTableNameQ3 = "lar2021_q3" - larTableNameQ3 = ${?LAR_TABLE_NAME_2021_Q3} - } + tsTableName = "transmittalsheet2021_regen_2024" + tsTableNameQ1 = "ts2021_q1_regen_2024" + tsTableNameQ2 = "ts2021_q2_regen_2024" + tsTableNameQ3 = "ts2021_q3_regen_2024" + larTableName = "loanapplicationregister2021_regen_2024" + larTableNameQ1 = "lar2021_q1_regen_2024" + larTableNameQ2 = "lar2021_q2_regen_2024" + larTableNameQ3 = "lar2021_q3_regen_2024" + } 2020 { - tsTableName = "transmittalsheet2020" - tsTableName = ${?TS_TABLE_NAME_2020} - tsTableNameQ1 = "ts2020_q1" - tsTableNameQ1 = ${?TS_TABLE_NAME_2020_Q1} - tsTableNameQ2 = "ts2020_q2" - tsTableNameQ2 = ${?TS_TABLE_NAME_2020_Q2} - tsTableNameQ3 = "ts2020_q3" - tsTableNameQ3 = ${?TS_TABLE_NAME_2020_Q3} - larTableName = "loanapplicationregister2020" - larTableName = ${?LAR_TABLE_NAME_2020} - larTableNameQ1 = "lar2020_q1" - larTableNameQ1 = ${?LAR_TABLE_NAME_2020_Q1} - larTableNameQ2 = "lar2020_q2" - larTableNameQ2 = ${?LAR_TABLE_NAME_2020_Q2} - larTableNameQ3 = "lar2020_q3" - larTableNameQ3 = ${?LAR_TABLE_NAME_2020_Q3} - historyTableName = "submission_history" - historyTableName = ${?HIST_TABLE_NAME} - } + tsTableName = "transmittalsheet2020_regen_2024" + tsTableNameQ1 = "ts2020_q1_regen_2024" + tsTableNameQ2 = "ts2020_q2_regen_2024" + tsTableNameQ3 = "ts2020_q3_regen_2024" + larTableName = "loanapplicationregister2020_regen_2024" + larTableNameQ1 = "lar2020_q1_regen_2024" + larTableNameQ2 = "lar2020_q2_regen_2024" + larTableNameQ3 = "lar2020_q3_regen_2024" + historyTableName = "submission_history_regen_2024" + } 2019 { - tsTableName = "transmittalsheet2019" - tsTableName = ${?TS_TABLE_NAME_2019} - larTableName = "loanapplicationregister2019" - larTableName = ${?LAR_TABLE_NAME_2019} - historyTableName = "submission_history" - historyTableName = ${?HIST_TABLE_NAME} - } + tsTableName = "transmittalsheet2019_regen_2024" + larTableName = "loanapplicationregister2019_regen_2024" + historyTableName = "submission_history" + } 2018 { - tsTableName = "transmittalsheet2018" - tsTableName = ${?TS_TABLE_NAME_2018} - larTableName = "loanapplicationregister2018" - larTableName = ${?LAR_TABLE_NAME_2018} - historyTableName = "submission_history" - historyTableName = ${?HIST_TABLE_NAME} - } + tsTableName = "transmittalsheet2018_regen_2024" + larTableName = "loanapplicationregister2018_regen_2024" + historyTableName = "submission_history" + } } } diff --git a/irs-publisher/src/main/resources/application.conf b/irs-publisher/src/main/resources/application.conf index 3970bb53e6..0c210e5d95 100644 --- a/irs-publisher/src/main/resources/application.conf +++ b/irs-publisher/src/main/resources/application.conf @@ -69,9 +69,9 @@ aws { access-key-id = ${?AWS_ACCESS_KEY_ID} secret-access-key = "" secret-access-key = ${?AWS_SECRET_ACCESS_KEY} - environment = "dev" //change to "prod" for production S3 bucket + environment = "dev/regen" //change to "prod" for production S3 bucket environment = ${?AWS_ENVIRONMENT} - public-bucket = "cfpb-hmda-public" + public-bucket = "cfpb-hmda-export" public-bucket = ${?S3_PUBLIC_BUCKET} region = "us-east-1" region = ${?AWS_REGION} diff --git a/modified-lar/src/main/resources/application.conf b/modified-lar/src/main/resources/application.conf index 10fbceb1a1..bbce29cb97 100644 --- a/modified-lar/src/main/resources/application.conf +++ b/modified-lar/src/main/resources/application.conf @@ -64,11 +64,9 @@ aws { access-key-id = ${?AWS_ACCESS_KEY_ID} secret-access-key = "" secret-access-key = ${?AWS_SECRET_ACCESS_KEY} - environment = "dev" //change to "prod" for production S3 bucket - environment = ${?AWS_ENVIRONMENT} - public-bucket = "cfpb-hmda-public" - public-bucket = ${?S3_PUBLIC_BUCKET} - region = "us-east-1" + environment = "dev/regen" //change to "prod" for production S3 bucket + public-bucket = "cfpb-hmda-export" + region = "us-east-1" region = ${?AWS_REGION} } From 10f410e203420758638de871439d30a9058333da Mon Sep 17 00:00:00 2001 From: PatrickHSI Date: Thu, 4 Apr 2024 13:05:20 -0400 Subject: [PATCH 2/6] testing data regen strategies --- hmda-analytics/src/main/resources/application.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hmda-analytics/src/main/resources/application.conf b/hmda-analytics/src/main/resources/application.conf index a4be47c161..37e68f10a2 100644 --- a/hmda-analytics/src/main/resources/application.conf +++ b/hmda-analytics/src/main/resources/application.conf @@ -92,7 +92,7 @@ hmda { larTableNameQ1 = "lar2022_q1_regen_2024" larTableNameQ2 = "lar2022_q2_regen_2024" larTableNameQ3 = "lar2022_q3_regen_2024" - ® } + } 2021 { tsTableName = "transmittalsheet2021_regen_2024" tsTableNameQ1 = "ts2021_q1_regen_2024" From 916aadcf3c189fe26911d6a63135fa214a022d92 Mon Sep 17 00:00:00 2001 From: PatrickHSI Date: Thu, 4 Apr 2024 13:51:19 -0400 Subject: [PATCH 3/6] testing data regen strategies --- modified-lar/src/test/resources/modifiedlar.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modified-lar/src/test/resources/modifiedlar.sql b/modified-lar/src/test/resources/modifiedlar.sql index e825fed215..1b91ed297b 100644 --- a/modified-lar/src/test/resources/modifiedlar.sql +++ b/modified-lar/src/test/resources/modifiedlar.sql @@ -1,4 +1,4 @@ -CREATE TABLE modifiedlar2019 ( +CREATE TABLE modifiedlar2019_regen_2024 ( id integer NOT NULL, uli character varying not null, lei character varying NOT NULL, @@ -99,7 +99,7 @@ CREATE TABLE modifiedlar2019 ( conforming_loan_limit character varying, median_age integer, median_age_calculated character varying, - median_income_percentage integer, + median_income_percentage double precision, race_categorization character varying, sex_categorization character varying, ethnicity_categorization character varying, @@ -110,7 +110,7 @@ CREATE TABLE modifiedlar2019 ( checksum character varying ); -CREATE TABLE modifiedlar2018 ( +CREATE TABLE modifiedlar2018_regen_2024 ( id integer NOT NULL, uli character varying not null, lei character varying NOT NULL, @@ -211,7 +211,7 @@ CREATE TABLE modifiedlar2018 ( conforming_loan_limit character varying, median_age integer, median_age_calculated character varying, - median_income_percentage integer, + median_income_percentage double precision, race_categorization character varying, sex_categorization character varying, ethnicity_categorization character varying, From 9af064242f9690338466a65206f240968b7ef51c Mon Sep 17 00:00:00 2001 From: PatrickHSI Date: Thu, 4 Apr 2024 14:03:02 -0400 Subject: [PATCH 4/6] testing data regen strategies --- irs-publisher/src/main/resources/application.conf | 2 -- 1 file changed, 2 deletions(-) diff --git a/irs-publisher/src/main/resources/application.conf b/irs-publisher/src/main/resources/application.conf index 0c210e5d95..0d6d897d5f 100644 --- a/irs-publisher/src/main/resources/application.conf +++ b/irs-publisher/src/main/resources/application.conf @@ -70,9 +70,7 @@ aws { secret-access-key = "" secret-access-key = ${?AWS_SECRET_ACCESS_KEY} environment = "dev/regen" //change to "prod" for production S3 bucket - environment = ${?AWS_ENVIRONMENT} public-bucket = "cfpb-hmda-export" - public-bucket = ${?S3_PUBLIC_BUCKET} region = "us-east-1" region = ${?AWS_REGION} } From 41ba7b4259e08a6b8a6ddebc0d569e10676260bd Mon Sep 17 00:00:00 2001 From: PatrickHSI Date: Thu, 4 Apr 2024 20:46:18 -0400 Subject: [PATCH 5/6] testing data regen strategies --- hmda-analytics/src/main/resources/application.conf | 9 ++++----- irs-publisher/src/main/resources/application.conf | 4 ++-- modified-lar/src/main/resources/application.conf | 7 +++---- modified-lar/src/main/resources/logback.xml | 8 ++++---- .../lar/publication/ModifiedLarPublisher.scala | 12 +++--------- 5 files changed, 16 insertions(+), 24 deletions(-) diff --git a/hmda-analytics/src/main/resources/application.conf b/hmda-analytics/src/main/resources/application.conf index 37e68f10a2..dbf36aba8a 100644 --- a/hmda-analytics/src/main/resources/application.conf +++ b/hmda-analytics/src/main/resources/application.conf @@ -56,13 +56,12 @@ kafka { hmda { analytics { parallelism = 1 - tsDeletion = true + tsDeletion = false tsDeletion = ${?HMDA-TS-DELETE} - larDeletion = true + larDeletion = false larDeletion = ${?HMDA-LAR-DELETE} - historyInsertion = true - historyInsertion = ${?HMDA-HISTORY-INSERT} - yearsAvailable = "2018,2019,2020,2021,2022,2023,2024" + historyInsertion = false + yearsAvailable = "2018,2019,2020,2021,2022,2023,2024" 2024 { tsTableName = "transmittalsheet2024_regen_2024" tsTableNameQ1 = "ts2024_q1_regen_2024" diff --git a/irs-publisher/src/main/resources/application.conf b/irs-publisher/src/main/resources/application.conf index 0d6d897d5f..04ab16937c 100644 --- a/irs-publisher/src/main/resources/application.conf +++ b/irs-publisher/src/main/resources/application.conf @@ -67,10 +67,10 @@ kafka { aws { access-key-id = "" access-key-id = ${?AWS_ACCESS_KEY_ID} - secret-access-key = "" + secret-ΩΩΩΩaccess-key = "" secret-access-key = ${?AWS_SECRET_ACCESS_KEY} environment = "dev/regen" //change to "prod" for production S3 bucket - public-bucket = "cfpb-hmda-export" + public-bucket = "cfpb-hmda-public" region = "us-east-1" region = ${?AWS_REGION} } diff --git a/modified-lar/src/main/resources/application.conf b/modified-lar/src/main/resources/application.conf index bbce29cb97..78beeb5d6c 100644 --- a/modified-lar/src/main/resources/application.conf +++ b/modified-lar/src/main/resources/application.conf @@ -65,8 +65,8 @@ aws { secret-access-key = "" secret-access-key = ${?AWS_SECRET_ACCESS_KEY} environment = "dev/regen" //change to "prod" for production S3 bucket - public-bucket = "cfpb-hmda-export" - region = "us-east-1" + public-bucket = "cfpb-hmda-public" + region = "us-east-1" region = ${?AWS_REGION} } @@ -74,8 +74,7 @@ hmda { lar { modified { parallelism = 1 - regenerateMlar = false - regenerateMlar = ${IS_REGENERATE_MLAR} + regenerateMlar = true generateS3Files = true generateS3Files = ${?IS_GENERATE_MLAR_S3_FIES} creteDispositionRecord = false diff --git a/modified-lar/src/main/resources/logback.xml b/modified-lar/src/main/resources/logback.xml index 52ce71e541..611d69421f 100644 --- a/modified-lar/src/main/resources/logback.xml +++ b/modified-lar/src/main/resources/logback.xml @@ -10,10 +10,10 @@ - + - - - + + + \ No newline at end of file diff --git a/modified-lar/src/main/scala/hmda/publication/lar/publication/ModifiedLarPublisher.scala b/modified-lar/src/main/scala/hmda/publication/lar/publication/ModifiedLarPublisher.scala index 17b3767a96..feb33a9ed8 100644 --- a/modified-lar/src/main/scala/hmda/publication/lar/publication/ModifiedLarPublisher.scala +++ b/modified-lar/src/main/scala/hmda/publication/lar/publication/ModifiedLarPublisher.scala @@ -176,17 +176,11 @@ object ModifiedLarPublisher { val graphWithJustS3WithHeader = mlarSource.via(serializeMlar).prepend(mlarHeader).toMat(s3SinkWithHeader)(Keep.right) val finalResult: Future[Unit] = for { - _ <- if (regenerateMlar) + _ <- if (true) graphWithS3AndPG.run() - else if (isGenerateBothS3Files) { - removeLei - graphWithS3AndPG.run() - } else if (isJustGenerateS3File) - graphWithJustS3NoHeader.run() - else if (isJustGenerateS3FileHeader) - graphWithJustS3WithHeader.run() + else { //everything - removeLei + Future.sequence(List(graphWithJustS3NoHeader.run(), graphWithJustS3WithHeader.run(), graphWithJustPG.run())) } _ <- produceRecord(disclosureTopic, submissionId.lei, submissionId.toString, kafkaProducer) From c2f95d5026b707daa388d515862d90cfcf78c3a0 Mon Sep 17 00:00:00 2001 From: PatrickHSI Date: Thu, 11 Apr 2024 11:51:39 -0400 Subject: [PATCH 6/6] testing data regen strategies --- .../main/scala/hmda/parser/filing/lar/LarFormatValidator.scala | 2 +- .../src/main/scala/hmda/publisher/helper/SnapshotCheck.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common/src/main/scala/hmda/parser/filing/lar/LarFormatValidator.scala b/common/src/main/scala/hmda/parser/filing/lar/LarFormatValidator.scala index d3a9c7e6b7..c64b301b8f 100644 --- a/common/src/main/scala/hmda/parser/filing/lar/LarFormatValidator.scala +++ b/common/src/main/scala/hmda/parser/filing/lar/LarFormatValidator.scala @@ -36,7 +36,7 @@ sealed trait LarFormatValidator extends LarParser { val occupancy = values(8) val loanAmount = values(9) val actionTaken = values(10) - val actionTakenDate = values(11) + val actionTakenDate = values(11).replaceFirst("^0+(?!$)", "") val street = values(12) val city = values(13) val state = values(14) diff --git a/hmda-data-publisher/src/main/scala/hmda/publisher/helper/SnapshotCheck.scala b/hmda-data-publisher/src/main/scala/hmda/publisher/helper/SnapshotCheck.scala index 332e7414ef..89c46f8ac0 100644 --- a/hmda-data-publisher/src/main/scala/hmda/publisher/helper/SnapshotCheck.scala +++ b/hmda-data-publisher/src/main/scala/hmda/publisher/helper/SnapshotCheck.scala @@ -13,7 +13,7 @@ object SnapshotCheck { def pathSelector(s3Path: String,fileName:String): String = { if(snapshotActive){ if (fileName.takeRight(3) == "zip"){ - val snapshotFile=fileName.replace(".zip","_snapshot.zip") + val snapsƒhotFile=fileName.replace(".zip","_snapshot.zip") snapshotPath+snapshotFile } else{