resolving comments

src/client/app/components/meters/EditMeterModalComponent.tsx

@@ -95,10 +95,7 @@ export default function EditMeterModalComponent(props: EditMeterModalComponentPr
 		if (selectedUnit) {
 			setLocalMeterEdits({
 				...localMeterEdits,
-				unitId: selectedUnitId,
-				minVal: selectedUnit.minVal,
-				maxVal: selectedUnit.maxVal,
-				disableChecks: selectedUnit.disableChecks
+				unitId: selectedUnitId
 			});
 		}
 	};

src/client/app/components/unit/CreateUnitModalComponent.tsx

@@ -45,7 +45,7 @@ export default function CreateUnitModalComponent() {
 		id: -99,
 		minVal: MIN_VAL,
 		maxVal: MAX_VAL,
-		disableChecks: DisableChecksType.reject_none
+		disableChecks: DisableChecksType.reject_all
 	};
 
 	/* State */
@@ -133,15 +133,14 @@ export default function CreateUnitModalComponent() {
 
 	// Keeps canSave state up to date. Checks if valid and if edit made.
 	useEffect(() => {
-
 		// This checks:
 		// - Name cannot be blank
 		// - If type of unit is suffix there must be a suffix
 		// - The rate is set so not the custom input value. This happens if select custom value but don't input with enter.
 		// - The custom rate is a positive integer
 		const validUnit = state.name !== '' &&
 			(state.typeOfUnit !== UnitType.suffix || state.suffix !== '') && state.secInRate !== Number(CUSTOM_INPUT)
-			&& (state?.minVal < MIN_VAL || state?.minVal > state?.maxVal)
+			&& state?.minVal >= MIN_VAL && state?.maxVal <= MAX_VAL && state?.minVal <= state?.maxVal
 			&& customRateValid(Number(state.secInRate));
 		setCanSave(validUnit);
 	}, [state]);
@@ -436,7 +435,7 @@ export default function CreateUnitModalComponent() {
 						<Row xs='1' lg='2'>
 							{/* minVal input */}
 							<Col><FormGroup>
-								<Label for='minVal'>{translate('min.val')}</Label>
+								<Label for='minVal'>{translate('min.value')}</Label>
 								<Input id='minVal' name='minVal' type='number'
 									onChange={e => handleNumberChange(e)}
 									min={MIN_VAL}
@@ -482,10 +481,11 @@ export default function CreateUnitModalComponent() {
 								id='note'
 								name='note'
 								type='textarea'
-								onChange={e => handleStringChange(e)}
-								value={state.note} />
+								value={state.note}
+								onChange={e => handleStringChange(e)} />
 						</FormGroup>
-					</Container></ModalBody>
+					</Container>
+				</ModalBody>
 				<ModalFooter>
 					{/* Hides the modal */}
 					<Button color="secondary" onClick={handleClose}>

src/client/app/components/unit/EditUnitModalComponent.tsx

@@ -203,6 +203,7 @@ export default function EditUnitModalComponent(props: EditUnitModalComponentProp
 		// - The custom rate is a positive integer
 		const validUnit = state.name !== '' &&
 			(state.typeOfUnit !== UnitType.suffix || state.suffix !== '') && state.secInRate !== Number(CUSTOM_INPUT)
+			&& state?.minVal >= MIN_VAL && state?.maxVal <= MAX_VAL && state?.minVal <= state?.maxVal
 			&& customRateValid(Number(state.secInRate));
 		// Compare original props to state to see if edit made. Check above avoids thinking edit happened if
 		// custom edit started without enter hit.

src/client/app/redux/selectors/adminSelectors.ts

@@ -312,7 +312,7 @@ export const selectDefaultCreateMeterValues = createAppSelector(
 			minDate: adminPreferences.defaultMeterMinimumDate,
 			maxDate: adminPreferences.defaultMeterMaximumDate,
 			maxError: adminPreferences.defaultMeterMaximumErrors,
-			disableChecks: DisableChecksType.reject_none
+			disableChecks: DisableChecksType.reject_all
 		};
 		return defaultValues;
 	}

src/client/app/redux/slices/adminSlice.ts

@@ -35,7 +35,7 @@ export const defaultAdminState: AdminState = {
 	defaultMeterMaximumDate: moment(0).utc().add(5000, 'years').format('YYYY-MM-DD HH:mm:ssZ'),
 	defaultMeterReadingGap: 0,
 	defaultMeterMaximumErrors: 75,
-	defaultMeterDisableChecks: DisableChecksType.reject_none,
+	defaultMeterDisableChecks: DisableChecksType.reject_all,
 	defaultHelpUrl: ''
 };
 

src/client/app/types/redux/units.ts

@@ -25,6 +25,7 @@ export enum DisableChecksType {
 	reject_all = 'reject_all',
 	reject_none = 'reject_none'
 }
+
 export interface UnitData {
 	id: number;
 	name: string;

src/client/app/utils/input.ts

@@ -86,7 +86,7 @@ export const NoUnit: UnitData = {
 	note: '',
 	minVal: MIN_VAL,
 	maxVal: MAX_VAL,
-	disableChecks: DisableChecksType.reject_none
+	disableChecks: DisableChecksType.reject_all
 };
 
 /**

src/server/migrations/1.0.0-2.0.0/index.js

@@ -20,8 +20,8 @@ module.exports = {
         // It should not matter but first rename cik and then do units.
         await db.none(sqlFile('../migrations/1.0.0-2.0.0/sql/cik/alter_cik_table.sql'));
         await db.none(sqlFile('../migrations/1.0.0-2.0.0/sql/units/alter_units_table.sql'));
-        await db.none(sqlFile('../migrations/1.0.0-2.0.0/sql/units/alter_units_table_add_columns.sql'));
         await db.none(sqlFile('../migrations/1.0.0-2.0.0/sql/units/add_disable_checks_types.sql'));
+        await db.none(sqlFile('../migrations/1.0.0-2.0.0/sql/units/alter_units_table_add_columns.sql'));
         await db.none(sqlFile('../migrations/1.0.0-2.0.0/sql/meter/alter_meter_disable_checks.sql'));
         await db.none(sqlFile('../migrations/1.0.0-2.0.0/sql/preferences/alter_preferences_default_meter_disable_checks.sql'));
     }

src/server/migrations/1.0.0-2.0.0/sql/meter/add_meter_pipeline_checks.sql

@@ -4,8 +4,8 @@
 
 -- Add meter pipeline check columns to the meters table.
 ALTER TABLE meters
-    ADD COLUMN IF NOT EXISTS min_val FLOAT NOT NULL DEFAULT -9007199254740991 CHECK (min_val::FLOAT >= -9007199254740991),
-    ADD COLUMN IF NOT EXISTS max_val FLOAT NOT NULL DEFAULT 9007199254740991 CHECK (max_val::FLOAT <= 9007199254740991),
+    ADD COLUMN IF NOT EXISTS min_val FLOAT NOT NULL DEFAULT -9007199254740991,
+    ADD COLUMN IF NOT EXISTS max_val FLOAT NOT NULL DEFAULT 9007199254740991 CHECK (max_val >= min_val),
     ADD COLUMN IF NOT EXISTS min_date TIMESTAMP NOT NULL DEFAULT '1970-01-01 00:00:00+00:00',
     ADD COLUMN IF NOT EXISTS max_date TIMESTAMP NOT NULL DEFAULT '6970-01-01 00:00:00+00:00',
     ADD COLUMN IF NOT EXISTS max_error INTEGER NOT NULL DEFAULT 75,

src/server/migrations/1.0.0-2.0.0/sql/meter/alter_meter_disable_checks.sql

@@ -4,7 +4,7 @@
 
 -- Add a temporary column with the new type
 ALTER TABLE meters 
-    ADD COLUMN disable_checks_temp disable_checks_type NOT NULL DEFAULT 'reject_none';
+    ADD COLUMN disable_checks_temp disable_checks_type NOT NULL DEFAULT 'reject_all';
 
 -- Update the temporary column based on the current boolean values in disable_checks
 UPDATE meters

src/server/migrations/1.0.0-2.0.0/sql/preferences/add_preferences_pipeline_checks.sql

@@ -12,7 +12,7 @@ ALTER TABLE preferences
     ADD COLUMN IF NOT EXISTS default_meter_maximum_date TIMESTAMP NOT NULL DEFAULT '6970-01-01 00:00:00+00:00',
 	ADD COLUMN IF NOT EXISTS default_meter_reading_gap REAL NOT NULL DEFAULT 0,
     ADD COLUMN IF NOT EXISTS default_meter_maximum_errors INTEGER NOT NULL DEFAULT 75,
-	ADD COLUMN IF NOT EXISTS default_meter_disable_checks disable_checks_type NOT NULL DEFAULT 'reject_none'
+	ADD COLUMN IF NOT EXISTS default_meter_disable_checks disable_checks_type NOT NULL DEFAULT 'reject_all'
 ;
 -- Now remove default since not desired.
 ALTER TABLE preferences

src/server/migrations/1.0.0-2.0.0/sql/preferences/alter_preferences_default_meter_disable_checks.sql

@@ -4,7 +4,7 @@
 
 -- Add a temporary column with the new type
 ALTER TABLE preferences 
-    ADD COLUMN default_meter_disable_checks_temp disable_checks_type NOT NULL DEFAULT 'reject_none';
+    ADD COLUMN default_meter_disable_checks_temp disable_checks_type NOT NULL DEFAULT 'reject_all';
 
 -- Update the temporary column based on the current boolean values in disable_checks
 UPDATE preferences

src/server/migrations/1.0.0-2.0.0/sql/units/alter_units_table_add_columns.sql

@@ -3,7 +3,7 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 ALTER TABLE units
-    ADD COLUMN IF NOT EXISTS min_val FLOAT NOT NULL DEFAULT -9007199254740991 CHECK (min_val::FLOAT >= -9007199254740991),
-    ADD COLUMN IF NOT EXISTS max_val FLOAT NOT NULL DEFAULT 9007199254740991 CHECK (max_val::FLOAT <= 9007199254740991),
-    ADD COLUMN IF NOT EXISTS disable_checks disable_checks_type NOT NULL DEFAULT 'reject_none'
+    ADD COLUMN IF NOT EXISTS min_val FLOAT NOT NULL DEFAULT -9007199254740991,
+    ADD COLUMN IF NOT EXISTS max_val FLOAT NOT NULL DEFAULT 9007199254740991 CHECK (max_val >= min_val),
+    ADD COLUMN IF NOT EXISTS disable_checks disable_checks_type NOT NULL DEFAULT 'reject_all'
 ;

src/server/models/Meter.js

@@ -60,7 +60,7 @@ class Meter {
 		reading = 0.0, startTimestamp = moment(0).utc().format('YYYY-MM-DD HH:mm:ssZ'), endTimestamp = moment(0).utc().format('YYYY-MM-DD HH:mm:ssZ'),
 		previousEnd = moment(0).utc(), unitId = -99, defaultGraphicUnit = -99, areaUnit = Unit.areaUnitType.NONE, readingFrequency = '00:15:00',
 		minVal = Number.MIN_SAFE_INTEGER, maxVal = Number.MAX_SAFE_INTEGER, minDate = moment(0).utc().format('YYYY-MM-DD HH:mm:ssZ'),
-		maxDate = moment(0).utc().add(5000, 'years').format('YYYY-MM-DD HH:mm:ssZ'), maxError = 75, disableChecks = Meter.disableChecksType.REJECT_NONE) {
+		maxDate = moment(0).utc().add(5000, 'years').format('YYYY-MM-DD HH:mm:ssZ'), maxError = 75, disableChecks = Meter.disableChecksType.REJECT_ALL) {
 		// In order for the CSV pipeline to work, the order of the parameters needs to match the order that the fields are declared.
 		// In addition, each new parameter has to be added at the very end.
 		this.id = id;

src/server/models/Unit.js

@@ -20,9 +20,9 @@ class Unit {
 	 * @param {*} note Note about this unit.
 	 * @param minVal inclusive minimum acceptable reading value (won't be rejected)
 	 * @param maxVal inclusive maximum acceptable reading value (won't be rejected)
-	 * @param disableChecks disable checks on meter for conditionSet 
+	 * @param disableChecks disable checks on meter for conditionSet (three possible values)
 	 */
-	constructor(id, name, identifier = name, unitRepresent, secInRate = 3600, typeOfUnit, suffix = '', displayable, preferredDisplay, note, minVal = Number.MIN_SAFE_INTEGER, maxVal = Number.MAX_SAFE_INTEGER, disableChecks = Unit.disableChecksType.REJECT_NONE) {
+	constructor(id, name, identifier = name, unitRepresent, secInRate = 3600, typeOfUnit, suffix = '', displayable, preferredDisplay, note, minVal = Number.MIN_SAFE_INTEGER, maxVal = Number.MAX_SAFE_INTEGER, disableChecks = Unit.disableChecksType.REJECT_ALL) {
 		this.id = id;
 		this.name = name;
 		this.identifier = identifier;

src/server/routes/meters.js

@@ -63,7 +63,7 @@ function formatMeterForResponse(meter, hasFullAccess) {
 		minDate: null,
 		maxDate: null,
 		maxError: null,
-		disableChecks: meter.disableChecks,
+		disableChecks: null,
 	};
 
 	// Only logged in Admins can see url, types, timezones, and internal names
@@ -266,7 +266,6 @@ router.post('/edit', requiredAdmin('edit meters'), async (req, res) => {
 		const conn = getConnection();
 		try {
 			const meter = await Meter.getByID(req.body.id, conn);
-			console.log(req.body);
 			const updatedMeter = new Meter(
 				undefined, // id
 				req.body.name,

src/server/services/pipeline-in-progress/processData.js

@@ -9,7 +9,7 @@ const { log } = require('../../log');
 const Meter = require('../../models/Meter');
 const Reading = require('../../models/Reading');
 const handleCumulativeReset = require('./handleCumulativeReset');
-const { validateReadings } = require('./validateReadings');
+const { validateReadings, validateSingleReading } = require('./validateReadings');
 const { MeterTimeSortTypesJS } = require('../csvPipeline/validateCsvUploadParams');
 const { meterTimezone } = require('../meterTimezone');
 
@@ -659,7 +659,6 @@ async function processData(rows, meterID, timeSort = MeterTimeSortTypesJS.increa
 		// Not used in many cases but just set since easier.
 		prevEndTimestampTz = endTimestampTz;
 	}
-	// Validate data if conditions given
 	// Validate data if conditions are given and disableChecks is not set to 'reject_none'
 	if (conditionSet !== undefined && conditionSet['disableChecks'] !== 'reject_none') {
 		const { validReadings, errMsg: newErrMsg } = validateReadings(result, conditionSet, meterName);
@@ -676,13 +675,12 @@ async function processData(rows, meterID, timeSort = MeterTimeSortTypesJS.increa
 				result = result.filter(reading => validateSingleReading(reading, conditionSet));
 			} else {
 				// Default behavior: reject all readings
-				result.splice(0, result.length); // Empties the result array
+				// Empties the result array
+				result.splice(0, result.length);
 			}
 
-			if (result.length === 0) {
-				isAllReadingsOk = false;
-				return { result, isAllReadingsOk, msgTotal };
-			}
+			// Mark readings as not okay since some were dropped
+			isAllReadingsOk = false;
 		}
 	}
 	// Update the meter to contain information for the last reading in the data file.

src/server/services/pipeline-in-progress/validateReadings.js

@@ -20,26 +20,28 @@ function validateReadings(arrayToValidate, conditionSet, meterIdentifier = undef
 	/* tslint:enable:no-string-literal */
 	const errMsg = errMsgDate + errMsgValue;
 
-	 // Handle the disableChecks logic
-	 if (conditionSet['disableChecks'] === 'reject_none') {
+    const validReadings = validDates && validValues;
+    let rejectedReadings = [];
+
+	// Even when 'reject_none' is set, issues should still be logged
+    if (conditionSet['disableChecks'] === 'reject_none') {
         return {
             validReadings: true,
             rejectedReadings: [],
-            errMsg: '', // No validation errors since checks are disabled
+			// Keep the error messages even if no rejections occur
+            errMsg,
         };
-    }
-
-    let validReadings = validDates && validValues;
-    let rejectedReadings = [];
-
-    if (conditionSet['disableChecks'] === 'reject_bad') {
-        // Remove only invalid readings and return the rejected ones
-        rejectedReadings = [...rejectedDates, ...rejectedValues];
-        arrayToValidate = arrayToValidate.filter(reading => !rejectedReadings.includes(reading));
-    } else if (conditionSet['disableChecks'] === 'reject_all' && !validReadings) {
-        // Reject all readings if validation fails
-        rejectedReadings = [...arrayToValidate];
-        arrayToValidate.length = 0; // Clear the array
+    } else {
+        if (conditionSet['disableChecks'] === 'reject_bad') {
+            // Remove only invalid readings and return the rejected ones
+            rejectedReadings = [...rejectedDates, ...rejectedValues];
+            arrayToValidate = arrayToValidate.filter(reading => !rejectedReadings.includes(reading));
+        } else if (conditionSet['disableChecks'] === 'reject_all' && !validReadings) {
+            // Reject all readings if validation fails
+            rejectedReadings = [...arrayToValidate];
+			// Clear the array
+			arrayToValidate = arrayToValidate.slice(0, 0);
+        }
     }
 
 	return {
@@ -70,9 +72,20 @@ function checkDate(arrayToValidate, minDate, maxDate, maxError, meterIdentifier)
 		if (maxError <= 0) {
 			break;
 		}
-		if (reading.startTimestamp < minDate || reading.endTimestamp > maxDate) {
+		if (reading.startTimestamp < minDate) {
+			const newErrMsg = `error when checking reading time for #${readingNumber} on meter ${meterIdentifier}: ` +
+				`time ${reading.startTimestamp} is earlier than lower bound ${minDate} ` +
+				`with reading ${reading.reading} and endTimestamp ${reading.endTimestamp}`;
+			log.error(newErrMsg);
+			errMsg += '<br>' + newErrMsg + '<br>';
+			--maxError;
+			validDates = false;
+			rejectedDates.push(reading);
+		}
+		if (reading.endTimestamp > maxDate) {
 			const newErrMsg = `error when checking reading time for #${readingNumber} on meter ${meterIdentifier}: ` +
-                `time ${reading.startTimestamp} or ${reading.endTimestamp} is out of bounds (min: ${minDate}, max: ${maxDate}).`;
+			`time ${reading.endTimestamp} is later than upper bound ${maxDate} ` +
+			`with reading ${reading.reading} and startTimestamp ${reading.startTimestamp}`;
 			log.error(newErrMsg);
 			errMsg += '<br>' + newErrMsg + '<br>';
 			--maxError;
@@ -101,9 +114,19 @@ function checkValue(arrayToValidate, minVal, maxVal, maxError, meterIdentifier)
 		if (maxError <= 0) {
 			break;
 		}
-		if (reading.reading < minVal || reading.reading > maxVal) {
+		if (reading.reading < minVal) {
 			const newErrMsg = `error when checking reading value for #${readingNumber} on meter ${meterIdentifier}: ` +
-                `value ${reading.reading} is out of bounds (min: ${minVal}, max: ${maxVal}).`;
+			`value ${reading.reading} is smaller than lower bound ${minVal} ` +
+			`with startTimestamp ${reading.startTimestamp} and endTimestamp ${reading.endTimestamp}`;
+			log.error(newErrMsg);
+			errMsg += '<br>' + newErrMsg + '<br>';
+			--maxError;
+			validValues = false;
+			rejectedValues.push(reading);
+		} else if (reading.reading > maxVal) {
+			const newErrMsg = `error when checking reading value for #${readingNumber} on meter ${meterIdentifier}: ` +
+			`value ${reading.reading} is larger than upper bound ${maxVal} ` +
+			`with startTimestamp ${reading.startTimestamp} and endTimestamp ${reading.endTimestamp}`;
 			log.error(newErrMsg);
 			errMsg += '<br>' + newErrMsg + '<br>';
 			--maxError;
@@ -155,10 +178,30 @@ function checkIntervals(arrayToValidate, threshold, meterIdentifier) {
 	return { validIntervals, errMsg };
 }
 
+/**
+ * Validate a single reading based on the provided conditionSet.
+ * @param {Reading} reading - The reading object to validate.
+ * @param {Object} conditionSet - The validation conditions including min/max values, dates, and thresholds.
+ * @returns {boolean} - Returns true if the reading is valid, false otherwise.
+ */
+function validateSingleReading(reading, conditionSet) {
+    const { minVal, maxVal, minDate, maxDate } = conditionSet;
+
+    // Check if the reading falls within the acceptable date range
+    if ((minDate && reading.startTimestamp < minDate) || (maxDate && reading.endTimestamp > maxDate)) {
+        return false;
+    }
+    // Check if the reading falls within the acceptable value range
+    if ((minVal !== undefined && reading.reading < minVal) || (maxVal !== undefined && reading.reading > maxVal)) {
+        return false;
+    }
+    return true;
+}
 
 module.exports = {
 	validateReadings,
 	checkDate,
 	checkValue,
-	checkIntervals
+	checkIntervals,
+	validateSingleReading
 };

src/server/sql/meter/create_meters_table.sql

@@ -65,10 +65,10 @@ CREATE TABLE IF NOT EXISTS meters (
     default_graphic_unit INTEGER REFERENCES units(id),
     area_unit area_unit_type NOT NULL DEFAULT 'none',
     reading_frequency INTERVAL NOT NULL DEFAULT '00:15:00',
-    min_val FLOAT NOT NULL DEFAULT -9007199254740991 CHECK (min_val::FLOAT >= -9007199254740991),
-    max_val FLOAT NOT NULL DEFAULT 9007199254740991 CHECK (max_val::FLOAT <= 9007199254740991),
+    min_val FLOAT NOT NULL DEFAULT -9007199254740991,
+    max_val FLOAT NOT NULL DEFAULT 9007199254740991 CHECK (max_val >= min_val),
     min_date TIMESTAMP NOT NULL DEFAULT '1970-01-01 00:00:00+00:00',
     max_date TIMESTAMP NOT NULL DEFAULT '6970-01-01 00:00:00+00:00',
     max_error INTEGER NOT NULL DEFAULT 75,
-    disable_checks disable_checks_type NOT NULL DEFAULT 'reject_none'
+    disable_checks disable_checks_type NOT NULL DEFAULT 'reject_all'
 );

src/server/sql/preferences/create_preferences_table.sql

@@ -21,6 +21,6 @@ CREATE TABLE IF NOT EXISTS preferences (
     default_meter_maximum_date TIMESTAMP NOT NULL,
 	default_meter_reading_gap REAL NOT NULL,
     default_meter_maximum_errors INTEGER NOT NULL,
-	default_meter_disable_checks disable_checks_type NOT NULL DEFAULT 'reject_none',
+	default_meter_disable_checks disable_checks_type NOT NULL DEFAULT 'reject_all',
 	default_help_url TEXT DEFAULT NULL
 );

src/server/sql/unit/create_units_table.sql

@@ -13,7 +13,7 @@ CREATE TABLE IF NOT EXISTS units (
 	displayable displayable_type NOT NULL,
 	preferred_display BOOLEAN NOT NULL,
 	note TEXT,
-	min_val FLOAT NOT NULL DEFAULT -9007199254740991 CHECK (min_val::FLOAT >= -9007199254740991),
-    max_val FLOAT NOT NULL DEFAULT 9007199254740991 CHECK (max_val::FLOAT <= 9007199254740991),
+	min_val FLOAT NOT NULL DEFAULT -9007199254740991,
+    max_val FLOAT NOT NULL DEFAULT 9007199254740991 CHECK (max_val >= min_val),
 	disable_checks disable_checks_type NOT NULL DEFAULT 'reject_none'
 );

src/server/sql/unit/insert_new_unit.sql

@@ -2,6 +2,6 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-INSERT INTO units(name, identifier, unit_represent, sec_in_rate, type_of_unit, suffix, displayable, preferred_display, note, max_val, min_val, disable_checks)
-VALUES (${name}, ${identifier}, ${unitRepresent}, ${secInRate}, ${typeOfUnit}, ${suffix}, ${displayable}, ${preferredDisplay}, ${note}, ${maxVal}, ${minVal}, ${disableChecks})
+INSERT INTO units(name, identifier, unit_represent, sec_in_rate, type_of_unit, suffix, displayable, preferred_display, note, min_val, max_val, disable_checks)
+VALUES (${name}, ${identifier}, ${unitRepresent}, ${secInRate}, ${typeOfUnit}, ${suffix}, ${displayable}, ${preferredDisplay}, ${note}, ${minVal}, ${maxVal}, ${disableChecks})
 RETURNING id;