Conditional Restrictions - Add conditional restriction logic to field…

… meta schemas and implement validation logic (#223)

* Remove script restrictions; Move unique property out of restrictions
These are breaking changes to the meta schema that need to be published as a major release. A document should be added to the repo describing these changes.

* Field restrictions can be an array of restriction objects

* Add tests for field restrictions of all forms

* Test validation library uses all restrictions in array

* Add document to detail major version changes

* Adds conditional restrictions to dictionary restriction schema
- performs code list and regex reference replacement recursively through conditional restrictions
- performs reference replacement recursively through meta objects
- applies conditional restriction checks when resolving restriction rules for each field
- WIP: still requires many tests

* Tests for references in meta and regex arrays

* ConditionalRestrictionTest has optional case with proper default in validation

* Single match object per condition, instead of array

* Fix recursive conditional restriction parsing and add tests

* Named container and volume in compose file

* Fix import path

* Empty field restriction validation tests

* Remove development test file

* Remove script restriction from test dictionary

* Restriction schemas directly written for each field type
- Generic conditional restriction function was removed because it could not be interpretted by the json schema generator
- Although there is repeated code, directly writing the typed conditional restriction type schemas is easier to parse and hopefully maintain. They have their types enforced by a generic type even if the schema itself is not generated through a function.

* Adds tests for conditional restriction match rules

* Generated JSON Schema with Conditional Restrictions

* Remove TODO statements that are not needed

* Fix broken table in reference doc

* Code cleanup by removing unused functions and comments

apps/server/docker-compose.yaml

@@ -1,17 +1,19 @@
-version: "2"
+version: '2'
 
 services:
   lecternDb:
-    image: "bitnami/mongodb:4.0"
+    container_name: lectern-mongo
+    image: bitnami/mongodb:4.0
     ports:
-      - "27017:27017"
+      - 27017:27017
     volumes:
-      - "mongodb_data:/bitnami"
+      - mongodb_data:/bitnami
     environment:
       MONGODB_USERNAME: admin
       MONGODB_PASSWORD: password
       MONGODB_DATABASE: lectern
       MONGODB_ROOT_PASSWORD: password123
 volumes:
   mongodb_data:
+    name: lectern-mongo-data
     driver: local

apps/server/src/services/dictionaryService.ts

@@ -29,7 +29,7 @@ import * as immer from 'immer';
 import { omit } from 'lodash';
 import logger from '../config/logger';
 import * as DictionaryRepo from '../db/dictionary';
-import { normalizeSchema, validate } from '../services/schemaService';
+import { validateDictionarySchema } from '../services/schemaService';
 import type { DictionaryDocument, DictionaryDocumentSummary } from '../db/dbTypes';
 
 /**
@@ -127,17 +127,15 @@ export const create = async (newDict: Dictionary): Promise<Dictionary> => {
 
 	// Verify schemas match dictionary
 	newDict.schemas.forEach((e) => {
-		const result = validate(e, newDict.references || {});
+		const result = validateDictionarySchema(e, newDict.references || {});
 		if (!result.valid) throw new BadRequestError(JSON.stringify(result.errors));
 	});
 
-	const normalizedSchemas = newDict.schemas.map((schema) => normalizeSchema(schema));
-
 	// Save new dictionary version
 	const result = await DictionaryRepo.addDictionary({
 		name: newDict.name,
 		version: newDict.version,
-		schemas: normalizedSchemas,
+		schemas: newDict.schemas,
 		references: newDict.references || {},
 	});
 	return result;
@@ -159,18 +157,16 @@ export const addSchema = async (id: string, schema: Schema): Promise<Dictionary>
 		throw new BadRequestError('Dictionary that you are trying to update is not the latest version.');
 	}
 
-	const result = validate(schema, existingDictionary.references || {});
+	const result = validateDictionarySchema(schema, existingDictionary.references || {});
 	if (!result.valid) throw new BadRequestError(JSON.stringify(result.errors));
 
 	if (existingDictionary.schemas.some((s) => s.name === schema.name)) {
 		throw new ConflictError('Schema with this name already exists.');
 	}
 
-	const normalizedSchema = normalizeSchema(schema);
-
 	const updatedDictionary = immer.produce(existingDictionary, (draft) => {
 		draft.version = VersionUtils.incrementMajor(draft.version);
-		draft.schemas = [...draft.schemas, normalizedSchema];
+		draft.schemas = [...draft.schemas, schema];
 	});
 
 	// Save new dictionary version
@@ -191,7 +187,7 @@ export const updateSchema = async (id: string, schema: Schema, major: boolean):
 
 	await checkLatest(existingDictionary);
 
-	const result = validate(schema, existingDictionary.references || {});
+	const result = validateDictionarySchema(schema, existingDictionary.references || {});
 	if (!result.valid) throw new BadRequestError(JSON.stringify(result.errors));
 
 	// Ensure it exists
@@ -202,17 +198,15 @@ export const updateSchema = async (id: string, schema: Schema, major: boolean):
 	// Filter out one to update
 	const schemas = existingDictionary.schemas.filter((s) => !(s['name'] === schema['name']));
 
-	const normalizedSchema = normalizeSchema(schema);
-
-	schemas.push(normalizedSchema);
+	schemas.push(schema);
 
 	// Increment Version
 	const nextVersion = major
 		? VersionUtils.incrementMajor(existingDictionary.version)
 		: VersionUtils.incrementMinor(existingDictionary.version);
 	const updatedDictionary = immer.produce(existingDictionary, (draft) => {
 		const filteredSchemas = draft.schemas.filter((s) => !(s['name'] === schema['name']));
-		draft.schemas = [...filteredSchemas, normalizedSchema];
+		draft.schemas = [...filteredSchemas, schema];
 		draft.version = nextVersion;
 	});
 

apps/server/src/services/schemaService.ts

@@ -18,10 +18,12 @@
  */
 
 import { References, replaceSchemaReferences, Schema } from '@overture-stack/lectern-dictionary';
-import * as immer from 'immer';
 import { ZodError } from 'zod';
 
-export function validate(schema: Schema, references: References): { valid: boolean; errors?: ZodError } {
+export function validateDictionarySchema(
+	schema: Schema,
+	references: References,
+): { valid: boolean; errors?: ZodError } {
 	const schemaWithReplacements = replaceSchemaReferences(schema, references);
 
 	// Ensure schema is still valid after reference replacement
@@ -30,37 +32,3 @@ export function validate(schema: Schema, references: References): { valid: boole
 
 	return parseResult.success ? { valid: true } : { valid: false, errors: parseResult.error };
 }
-
-/**
- * String formatting of values provided as scripts. This will normalize the formatting of newline characters,
- * All instances of `/r/n` will be converted to `/n`
- * @param script
- * @returns
- */
-function normalizeScript(input: string | string[]) {
-	const normalize = (script: string) => script.replace(/\r\n/g, '\n');
-
-	if (typeof input === 'string') {
-		return normalize(input);
-	} else {
-		return input.map(normalize);
-	}
-}
-
-export function normalizeSchema(schema: Schema): Schema {
-	const normalizedFields = schema.fields.map((baseField) =>
-		immer.produce(baseField, (field) => {
-			if (
-				field.valueType !== 'boolean' &&
-				field.restrictions !== undefined &&
-				field.restrictions.script !== undefined
-			) {
-				field.restrictions.script = normalizeScript(field.restrictions.script);
-			}
-		}),
-	);
-
-	return immer.produce(schema, (draft) => {
-		draft.fields = normalizedFields;
-	});
-}

apps/server/test/fixtures/schemas/references.ts

@@ -35,12 +35,5 @@ export default {
 				codeList: '#/listA',
 			},
 		},
-		{
-			name: 'script_as_reference',
-			valueType: 'number',
-			restrictions: {
-				script: '#/scriptA',
-			},
-		},
 	],
 } satisfies Schema;

apps/server/test/integration/fixtures/updateNewFile.json

@@ -30,7 +30,6 @@
 				"units": "days"
 			},
 			"restrictions": {
-				"script": ["validateWithMagic(check dependece on another field)"],
 				"range": {
 					"min": 0,
 					"max": 99999