+
+
+ {handleChange(e);}}
+ >
+ {Object.values(DisableChecksType).map(disableChecks => {
+ return (
+
+ );
+ })}
+
+
+
{/* TODO This feature is not working perfectly so disabling from web page but allowing in curl.
Rest of changes left so easy to add back in.
@@ -680,7 +807,7 @@ export default function ReadingsCSVUploadComponent() {
originally added to the web page input of import but decided to not allow
it at this time. Thus, the code was commented out. As of now
there is no plan to make this generally available due to its limitations.*/}
- {/*
+ {/**
*/}
- {/*
+ {/**
*/}
@@ -720,11 +847,11 @@ export default function ReadingsCSVUploadComponent() {
-
-
-
- )}
-
-
- );
-}
+
+
+
+ )}
+
+
+ );
+ }
diff --git a/src/client/app/types/csvUploadForm.ts b/src/client/app/types/csvUploadForm.ts
index 5cf4f9c33f..f752f0d1fb 100644
--- a/src/client/app/types/csvUploadForm.ts
+++ b/src/client/app/types/csvUploadForm.ts
@@ -3,12 +3,20 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
import { MeterTimeSortType } from '../types/redux/meters';
+import { DisableChecksType } from './redux/units';
export interface CSVUploadPreferences {
meterIdentifier: string;
gzip: boolean;
headerRow: boolean;
update: boolean;
+ timeZone?: string;
+ minVal?: string;
+ maxVal?: string;
+ minDate?: string;
+ maxDate?: string;
+ maxError?: string;
+ disableChecks?: DisableChecksType;
}
export interface ReadingsCSVUploadPreferences extends CSVUploadPreferences {
diff --git a/src/client/app/utils/api/UploadCSVApi.ts b/src/client/app/utils/api/UploadCSVApi.ts
index e515c70fd9..2fa34be55a 100644
--- a/src/client/app/utils/api/UploadCSVApi.ts
+++ b/src/client/app/utils/api/UploadCSVApi.ts
@@ -35,7 +35,9 @@ export const submitReadings = async (uploadPreferences: ReadingsCSVUploadPrefere
warnOnCumulativeReset: uploadPreferences.warnOnCumulativeReset
};
for (const [preference, value] of Object.entries(uploadPreferencesForm)) {
- formData.append(preference, value.toString());
+ if (value !== undefined && value !== null) {
+ formData.append(preference, value.toString());
+ }
}
formData.append('csvfile', readingsFile); // It is important for the server that the file is attached last.
@@ -61,7 +63,9 @@ export const submitMeters = async (uploadPreferences: MetersCSVUploadPreferences
update: uploadPreferences.update
};
for (const [preference, value] of Object.entries(uploadPreferencesForm)) {
- formData.append(preference, value.toString());
+ if (value !== undefined && value !== null) {
+ formData.append(preference, value.toString());
+ }
}
formData.append('csvfile', metersFile); // It is important for the server that the file is attached last.
diff --git a/src/client/app/utils/csvUploadDefaults.ts b/src/client/app/utils/csvUploadDefaults.ts
index 43d485a0cf..f8d5bcfd2d 100644
--- a/src/client/app/utils/csvUploadDefaults.ts
+++ b/src/client/app/utils/csvUploadDefaults.ts
@@ -2,6 +2,7 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+import { DisableChecksType } from '../types/redux/units';
import { ReadingsCSVUploadPreferences, MetersCSVUploadPreferences } from '../types/csvUploadForm';
import { MeterTimeSortType } from '../types/redux/meters';
@@ -25,7 +26,14 @@ export const ReadingsCSVUploadDefaults: ReadingsCSVUploadPreferences = {
timeSort: MeterTimeSortType.increasing,
update: false,
useMeterZone: false,
- warnOnCumulativeReset: false
+ warnOnCumulativeReset: false,
+ timeZone: '',
+ minVal: '',
+ maxVal: '',
+ minDate: '',
+ maxDate: '',
+ maxError: '',
+ disableChecks: DisableChecksType.reject_all
};
export const MetersCSVUploadDefaults: MetersCSVUploadPreferences = {
diff --git a/src/scripts/installOED.sh b/src/scripts/installOED.sh
index 1bef1b7a5b..be7f0d6421 100755
--- a/src/scripts/installOED.sh
+++ b/src/scripts/installOED.sh
@@ -52,10 +52,20 @@ while test $# -gt 0; do
esac
done
-# Load .env if it exists
-
+# Load .env if it exists, but do not overwrite values already provided by the
+# runtime environment (for example from Docker Compose).
if [ -f ".env" ]; then
- source .env
+ while IFS='=' read -r name value; do
+ case "$name" in
+ ''|'#'*)
+ continue
+ ;;
+ esac
+
+ if [ -z "${!name+x}" ]; then
+ export "$name=$value"
+ fi
+ done < .env
fi
# Skip the install if the node_modules were installed before the package files.
diff --git a/src/server/db.js b/src/server/db.js
index 9f356e386b..db5631dc97 100644
--- a/src/server/db.js
+++ b/src/server/db.js
@@ -67,7 +67,7 @@ function swapConnection(newConfig, newConnection) {
if (newConnection !== null) {
connmanager.connection = newConnection;
} else {
- connmanager = getDB(connmanager.config);
+ connmanager.connection = getDB(connmanager.config);
}
}
diff --git a/src/server/log.js b/src/server/log.js
index 627c168d86..a55f906ab2 100644
--- a/src/server/log.js
+++ b/src/server/log.js
@@ -4,11 +4,22 @@
const fs = require('fs');
const logFile = require('./config').logFile;
-const LogEmail = require('./models/LogEmail');
-const LogMsg = require('./models/LogMsg');
-const { getConnection } = require('./db');
const moment = require('moment');
+/**
+ * Get a database connection if the database module is available.
+ * Logging should still work even if the database is unavailable, so this
+ * intentionally falls back to null instead of throwing.
+ * @returns {object|null}
+ */
+function getConnection() {
+ try {
+ return require('./db').getConnection();
+ } catch (err) {
+ return null;
+ }
+}
+
/**
* Represents the importance of a message to be logged
*/
@@ -55,6 +66,8 @@ class Logger {
let messageToLog = `[${level.name}@${logTime.format('YYYY-MM-DDTHH:mm:ss.SSSZ')}] ${message}\n`;
const conn = getConnection();
+ const LogEmail = require('./models/LogEmail');
+ const LogMsg = require('./models/LogMsg');
// Add a stacktrace to the message if one was provided.
if (error !== null) {
@@ -169,25 +182,17 @@ const defaultLogger = new Logger(logFile);
* Wherever logging is available, the Node.js runtime will call this function to log unhandled rejections.
* This helps with debugging, especially in tests.
*/
-process.on('unhandledRejection', (reason, p) => {
- p.catch(e => {
- // Include both rejection reason and catch error text then pass a real Error object for stack logging
- const reasonText = reason instanceof Error ? reason.message : String(reason);
- const errorText = e instanceof Error ? e.message : String(e);
- const loggedError = e instanceof Error ? e : (reason instanceof Error ? reason : null);
- defaultLogger.error(`Unhandled Promise Rejection (reason: ${reasonText}; error (e): ${errorText})`, loggedError);
- });
-});
-// Log uncaught exceptions
-process.on('uncaughtException', (error) => {
- defaultLogger.error('Unhandled Exception:', error);
+process.on('unhandledRejection', (reason) => {
+ const message = reason instanceof Error ? reason.message : String(reason);
+ defaultLogger.error(`Unhandled Promise Rejection: ${message}`, reason instanceof Error ? reason : null);
});
defaultLogger.logToDb = true;
defaultLogger.logToConsole = true;
defaultLogger.level = LogLevel.DEBUG;
+
/**
* @type {{log: Logger, LogLevel: LogLevel}}
*/
diff --git a/src/server/services/csvPipeline/uploadReadings.js b/src/server/services/csvPipeline/uploadReadings.js
index 5247ac7c77..105e83ee79 100644
--- a/src/server/services/csvPipeline/uploadReadings.js
+++ b/src/server/services/csvPipeline/uploadReadings.js
@@ -18,7 +18,8 @@ const Meter = require('../../models/Meter');
*/
async function uploadReadings(req, res, filepath, conn) {
// extract query parameters
- const { meterIdentifier, meterName, headerRow, update, honorDst, relaxedParsing, useMeterZone, warnOnCumulativeReset } = req.body;
+ const { meterIdentifier, meterName, headerRow, update, honorDst, relaxedParsing, useMeterZone, warnOnCumulativeReset,
+ timeZone, minVal, maxVal, minDate, maxDate, maxError, disableChecks } = req.body;
// The next few have no value in the DB for a meter so always use the value passed.
const hasHeaderRow = normalizeBoolean(headerRow);
const shouldUpdate = normalizeBoolean(update);
@@ -188,16 +189,25 @@ async function uploadReadings(req, res, filepath, conn) {
}
const areReadingsEndOnly = readingEndOnly;
+ const isMissingParam = param => param === undefined || param === '';
+ const resolvedTimeZone = isMissingParam(timeZone) ? meter.meterTimezone : timeZone;
+ const resolvedMinVal = isMissingParam(minVal) ? meter.minVal : parseFloat(minVal);
+ const resolvedMaxVal = isMissingParam(maxVal) ? meter.maxVal : parseFloat(maxVal);
+ const resolvedMinDate = isMissingParam(minDate) ? meter.minDate : new Date(minDate);
+ const resolvedMaxDate = isMissingParam(maxDate) ? meter.maxDate : new Date(maxDate);
+ const resolvedMaxError = isMissingParam(maxError) ? meter.maxError : parseInt(maxError, 10);
+ const resolvedDisableChecks = isMissingParam(disableChecks) ? meter.disableChecks : disableChecks;
+
const mapRowToModel = row => { return row; }; // STUB function to satisfy the parameter of loadCsvInput.
const conditionSet = {
- minVal: meter.minVal,
- maxVal: meter.maxVal,
- minDate: meter.minDate,
- maxDate: meter.maxDate,
- threshold: meter.readingGap,
- maxError: meter.maxError,
- disableChecks: meter.disableChecks
+ minVal: resolvedMinVal,
+ maxVal: resolvedMaxVal,
+ minDate: resolvedMinDate,
+ maxDate: resolvedMaxDate,
+ threshold: readingGap,
+ maxError: resolvedMaxError,
+ disableChecks: resolvedDisableChecks
}
return await loadCsvInput(
@@ -220,7 +230,8 @@ async function uploadReadings(req, res, filepath, conn) {
shouldHonorDst,
shouldRelaxedParsing,
shouldUseMeterZone,
- shouldWarnOnCumulativeReset
+ shouldWarnOnCumulativeReset,
+ resolvedTimeZone
); // load csv data
}
diff --git a/src/server/services/csvPipeline/validateCsvUploadParams.js b/src/server/services/csvPipeline/validateCsvUploadParams.js
index b3b3f92fca..0daa593d46 100644
--- a/src/server/services/csvPipeline/validateCsvUploadParams.js
+++ b/src/server/services/csvPipeline/validateCsvUploadParams.js
@@ -7,6 +7,7 @@ const { CSVPipelineError } = require('./CustomErrors');
const { Param, EnumParam, BooleanParam, StringParam } = require('./ValidationSchemas');
const failure = require('./failure');
const validate = require('jsonschema').validate;
+const Unit = require('../../models/Unit');
// This is only used for meter page inputs but put here so next one above that related to.
/**
@@ -77,7 +78,14 @@ const DEFAULTS = {
relaxedParsing: false,
timeSort: undefined,
useMeterZone: false,
- warnOnCumulativeReset: false
+ warnOnCumulativeReset: false,
+ timeZone: undefined,
+ minVal: undefined,
+ maxVal: undefined,
+ minDate: undefined,
+ maxDate: undefined,
+ maxError: undefined,
+ disableChecks: undefined
}
}
@@ -129,6 +137,13 @@ const VALIDATION = {
timeSort: new EnumParam('timeSort', [MeterTimeSortTypesJS.increasing, MeterTimeSortTypesJS.decreasing]),
useMeterZone: new EnumParam('useMeterZone', BooleanCheckArray),
warnOnCumulativeReset: new EnumParam('warnOnCumulativeReset', BooleanCheckArray),
+ timeZone: new StringParam('timeZone', undefined, undefined),
+ minVal: new StringParam('minVal', undefined, undefined),
+ maxVal: new StringParam('maxVal', undefined, undefined),
+ minDate: new StringParam('minDate', undefined, undefined),
+ maxDate: new StringParam('maxDate', undefined, undefined),
+ maxError: new StringParam('maxError', undefined, undefined),
+ disableChecks: new EnumParam('disableChecks', Object.values(Unit.disableChecksType))
},
anyOf: [
{ required: ['meterIdentifier'] },
@@ -183,7 +198,8 @@ function validateReadingsCsvUploadParams(req, res, next) {
// extract query parameters
const { cumulative, cumulativeReset, duplications, gzip, headerRow, timeSort, update, honorDst,
- refreshReadings, relaxedParsing, useMeterZone, warnOnCumulativeReset } = req.body;
+ refreshReadings, relaxedParsing, useMeterZone, warnOnCumulativeReset, timeZone, minVal,
+ maxVal, minDate, maxDate, maxError, disableChecks } = req.body;
// Set default values of not supplied parameters.
if (cumulative === undefined) {
@@ -222,6 +238,27 @@ function validateReadingsCsvUploadParams(req, res, next) {
if (warnOnCumulativeReset === undefined) {
req.body.warnOnCumulativeReset = DEFAULTS.readings.warnOnCumulativeReset;
}
+ if (timeZone === undefined) {
+ req.body.timeZone = DEFAULTS.readings.timeZone;
+ }
+ if (minVal === undefined) {
+ req.body.minVal = DEFAULTS.readings.minVal;
+ }
+ if (maxVal === undefined) {
+ req.body.maxVal = DEFAULTS.readings.maxVal;
+ }
+ if (minDate === undefined) {
+ req.body.minDate = DEFAULTS.readings.minDate;
+ }
+ if (maxDate === undefined) {
+ req.body.maxDate = DEFAULTS.readings.maxDate;
+ }
+ if (maxError === undefined) {
+ req.body.maxError = DEFAULTS.readings.maxError;
+ }
+ if (disableChecks === undefined) {
+ req.body.disableChecks = DEFAULTS.readings.disableChecks;
+ }
next();
}
diff --git a/src/server/services/pipeline-in-progress/loadArrayInput.js b/src/server/services/pipeline-in-progress/loadArrayInput.js
index cb81787937..1bb8981494 100644
--- a/src/server/services/pipeline-in-progress/loadArrayInput.js
+++ b/src/server/services/pipeline-in-progress/loadArrayInput.js
@@ -30,16 +30,27 @@ const processData = require('./processData');
* @param {boolean} useMeterZone true if the readings are switched to the time zone (meter then site then server)), default if false.
* Should only be true if honorDST is true and reading does not have proper time zone information.
* @param {boolean} warnOnCumulativeReset true if a warning is shown for each reset with cumulative data. cumulative must be true. default is false.
+ * @param {string} timeZone timezone to use while processing data, default is undefined.
* @returns {object[]} {whether readings were all process (true) or false, all the messages from processing the readings as a string}
*/
+// NOTE (follow-up): Callers of this function sometimes omit optional trailing
+// parameters (for example `timeZone`). This works because JS allows omitted
+// trailing args, but it makes call sites inconsistent and harder to maintain.
+// Suggested follow-up: migrate to a single `options` object (e.g.
+// `loadArrayInput(dataRows, meterID, mapRowToModel, opts)`) or mandate
+// explicitly passing all arguments. Do NOT remove `timeZone` or other
+// parameters here in this PR — perform a backward-compatible refactor in a
+// separate change to avoid regressions.
+
async function loadArrayInput(dataRows, meterID, mapRowToModel, timeSort, readingRepetition, isCumulative,
cumulativeReset, cumulativeResetStart, cumulativeResetEnd, readingGap, readingLengthVariation, isEndOnly,
- shouldUpdate, conditionSet, conn, honorDst = false, relaxedParsing = false, useMeterZone = false, warnOnCumulativeReset = false) {
+ shouldUpdate, conditionSet, conn, honorDst = false, relaxedParsing = false, useMeterZone = false,
+ warnOnCumulativeReset = false, timeZone = undefined) {
// Get the reading, then process them for acceptance and finally insert into the DB.
readingsArray = dataRows.map(mapRowToModel);
let { result: readingsToInsert, isAllReadingsOk, msgTotal } = await processData(readingsArray, meterID, timeSort, readingRepetition,
isCumulative, cumulativeReset, cumulativeResetStart, cumulativeResetEnd, readingGap, readingLengthVariation, isEndOnly,
- conditionSet, conn, honorDst, relaxedParsing, useMeterZone, warnOnCumulativeReset);
+ conditionSet, conn, honorDst, relaxedParsing, useMeterZone, warnOnCumulativeReset, timeZone);
if (shouldUpdate) {
// New readings should replace old ones.
await Reading.insertOrUpdateAll(readingsToInsert, conn)
diff --git a/src/server/services/pipeline-in-progress/loadCsvInput.js b/src/server/services/pipeline-in-progress/loadCsvInput.js
index 27a08d7f83..f94986e896 100644
--- a/src/server/services/pipeline-in-progress/loadCsvInput.js
+++ b/src/server/services/pipeline-in-progress/loadCsvInput.js
@@ -32,6 +32,7 @@ const { log } = require('../../log');
* @param {boolean} useMeterZone true if the readings are switched to the time zone (meter then site then server)), default if false.
* Should only be true if honorDST is true and reading does not have proper time zone information.
* @param {boolean} warnOnCumulativeReset true if a warning is shown for each reset with cumulative data. cumulative must be true. default is false.
+ * @param {string} timeZone timezone to use while processing data, default is undefined.
*/
async function loadCsvInput(
filePath,
@@ -53,14 +54,15 @@ async function loadCsvInput(
honorDst = false,
relaxedParsing = false,
useMeterZone = false,
- warnOnCumulativeReset = false
+ warnOnCumulativeReset = false,
+ timeZone = undefined
) {
try {
const dataRows = await readCsv(filePath, headerRow);
return loadArrayInput(dataRows, meterID, mapRowToModel, timeSort, readingRepetition,
isCumulative, cumulativeReset, cumulativeResetStart, cumulativeResetEnd,
readingGap, readingLengthVariation, isEndOnly, shouldUpdate, conditionSet, conn,
- honorDst, relaxedParsing, useMeterZone, warnOnCumulativeReset);
+ honorDst, relaxedParsing, useMeterZone, warnOnCumulativeReset, timeZone);
} catch (err) {
log.error(`Error updating meter ${meterID} with data from ${filePath}: ${err}`, err);
}
diff --git a/src/server/services/pipeline-in-progress/processData.js b/src/server/services/pipeline-in-progress/processData.js
index d919812136..dc58a4e370 100644
--- a/src/server/services/pipeline-in-progress/processData.js
+++ b/src/server/services/pipeline-in-progress/processData.js
@@ -53,6 +53,7 @@ const E0 = moment(0).utc()
* Should only be true if honorDST is true and reading does not have proper time zone information. This feature is not great and should
* be avoided except in special circumstances.
* @param {boolean} warnOnCumulativeReset true if each cumulative reset generates a warning message and false if not. Default is false.
+ * @param {string} timeZone timezone to use while processing data, default is undefined.
* @param {boolean} useMeterFrequency true if isEndTime is true then any reading found with a different reading length that is longer than the meter
* frequency will make the start time by the end time minus the meter reading frequency. The idea is that a change in the length represents
* missing reading(s) then it will have a longer time but that is not what is desired for this meter. This only happens if the length of the reading
@@ -65,7 +66,8 @@ const E0 = moment(0).utc()
*/
async function processData(rows, meterID, timeSort = MeterTimeSortTypesJS.increasing, readingRepetition, isCumulative, cumulativeReset,
resetStart = '00:00:00.000', resetEnd = '23:59:99.999', readingGap = 0, readingLengthVariation = 0, isEndTime = false,
- conditionSet, conn, honorDst = false, relaxedParsing = false, useMeterZone = false, warnOnCumulativeReset = false, useMeterFrequency = false, useMeterFrequencyVariation = 0) {
+ conditionSet, conn, honorDst = false, relaxedParsing = false, useMeterZone = false, warnOnCumulativeReset = false,
+ timeZone = undefined, useMeterFrequency = false, useMeterFrequencyVariation = 0) {
// Holds all the warning message to pass back to inform user.
// Note they use basic HTML because the messages can be long/complex and it was felt it would be easy to put it into a web browser
// to make them easier to read.
@@ -125,7 +127,7 @@ async function processData(rows, meterID, timeSort = MeterTimeSortTypesJS.increa
// These only happen if worried about DST.
if (honorDst) {
// Get the meter timezone since the same while processing this data.
- meterZone = await meterTimezone(meter);
+ meterZone = (timeZone !== undefined && timeZone !== '') ? timeZone : await meterTimezone(meter);
// See if were processing a shift from DST (inDst) when last batch of readings ended so need to continue.
prevEndTimestamp = moment.parseZone(meter.previousEnd, true);
if (!isFirst(prevEndTimestamp)) {