From e95896011b7ddd831b7bab239fe0a812025b79f2 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Fri, 24 Mar 2023 15:24:48 -0400 Subject: [PATCH 1/3] fix(MobilityDataValidation): slow on large gtfs files --- .../com/conveyal/datatools/manager/models/FeedVersion.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java index ab81f4c31..ec53c251a 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java @@ -393,8 +393,8 @@ public void validateMobility(MonitorableJob.Status status) { status.update("MobilityData Analysis...", 11); // Wait for the file to be entirely copied into the directory. - // TODO: base this on the file being completely saved rather than a fixed amount of time. - Thread.sleep(5000); + // 5 seconds + ~1 second per 10mb + Thread.sleep(5000 + (this.fileSize / 10000)); File gtfsZip = this.retrieveGtfsFile(); // Namespace based folders avoid clash for validation being run on multiple versions of a feed. // TODO: do we know that there will always be a namespace? From 0ebf2227078653ede1d54a77676b22bb4a89a726 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Wed, 24 May 2023 09:33:48 -0400 Subject: [PATCH 2/3] refactor: correct comment --- .../java/com/conveyal/datatools/manager/models/FeedVersion.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java index ec53c251a..a2c358167 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java @@ -393,7 +393,7 @@ public void validateMobility(MonitorableJob.Status status) { status.update("MobilityData Analysis...", 11); // Wait for the file to be entirely copied into the directory. - // 5 seconds + ~1 second per 10mb + // 5 seconds + ~1 second per 100kb Thread.sleep(5000 + (this.fileSize / 10000)); File gtfsZip = this.retrieveGtfsFile(); // Namespace based folders avoid clash for validation being run on multiple versions of a feed. From 638c7a80d73a976d0ad00a5842ecd70504d2146e Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Wed, 24 May 2023 10:22:33 -0400 Subject: [PATCH 3/3] Revert "refactor: correct comment" This reverts commit 0ebf2227078653ede1d54a77676b22bb4a89a726. --- .../java/com/conveyal/datatools/manager/models/FeedVersion.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java index 6fa211f1f..523ee181a 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java @@ -393,7 +393,7 @@ public void validateMobility(MonitorableJob.Status status) { status.update("MobilityData Analysis...", 11); // Wait for the file to be entirely copied into the directory. - // 5 seconds + ~1 second per 100kb + // 5 seconds + ~1 second per 10mb Thread.sleep(5000 + (this.fileSize / 10000)); File gtfsZip = this.retrieveGtfsFile(); // Namespace based folders avoid clash for validation being run on multiple versions of a feed.