From 438a04bf9d15727ea95bcd61bcb7351a0634f7f8 Mon Sep 17 00:00:00 2001 From: Yelizar Alturmessov <65674408+yalturmes@users.noreply.github.com> Date: Thu, 8 Feb 2024 11:40:09 -0500 Subject: [PATCH 01/27] update Jenkinsfile for cumulus migration (#196) --- Jenkinsfile | 3 + Jenkinsfile.groovy | 196 --------------------------------------------- 2 files changed, 3 insertions(+), 196 deletions(-) create mode 100644 Jenkinsfile delete mode 100644 Jenkinsfile.groovy diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..db49cc0 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,3 @@ +@Library(value='jenkins-pipeline-library@master', changelog=false) _ +pipelineOVERTURELectern() + diff --git a/Jenkinsfile.groovy b/Jenkinsfile.groovy deleted file mode 100644 index f52993a..0000000 --- a/Jenkinsfile.groovy +++ /dev/null @@ -1,196 +0,0 @@ -String podSpec = ''' -apiVersion: v1 -kind: Pod -spec: - containers: - - name: node - image: node:18.16.1-alpine - tty: true - env: - - name: DOCKER_HOST - value: tcp://localhost:2375 - - name: HOME - value: /home/jenkins/agent - - name: dind-daemon - image: docker:18-dind - securityContext: - privileged: true - runAsUser: 0 - volumeMounts: - - name: docker-graph-storage - mountPath: /var/lib/docker - - name: docker - image: docker:20-git - tty: true - env: - - name: DOCKER_HOST - value: tcp://localhost:2375 - - name: HOME - value: /home/jenkins/agent - securityContext: - runAsUser: 1000 - volumes: - - name: docker-graph-storage - emptyDir: {} -''' - -pipeline { - - agent { - kubernetes { - yaml podSpec - } - } - - environment { - containerRegistry = "ghcr.io" - organization = "overture-stack" - appName = "lectern" - gitHubRepo = "${organization}/${appName}" - containerImageName = "${containerRegistry}/${gitHubRepo}" - - commit = sh(returnStdout: true, script: 'git describe --always').trim() - version = sh(returnStdout: true, script: 'cat package.json | grep version | cut -d \':\' -f2 | sed -e \'s/"//\' -e \'s/",//\'').trim() - - slackNotificationsUrl = credentials('OvertureSlackJenkinsWebhookURL') - } - - options { - timeout(time: 30, unit: 'MINUTES') - timestamps() - } - - stages { - - stage('Prepare') { - steps { - container('node') { - sh "npx --yes pnpm install" - } - } - } - - stage('Test') { - steps { - container('node') { - sh "npx --yes pnpm test:all" - } - } - } - - stage('Build') { - steps { - container('docker') { - // the network=host needed to download dependencies using the host network (since we are inside 'docker' container) - sh "docker build --build-arg=COMMIT=${commit} --network=host -f apps/server/Dockerfile . -t server:${commit}" - } - - } - } - - stage('Git Tags') { - when { - branch 'main' - } - steps { - container('docker') { - withCredentials([usernamePassword( - credentialsId: 'OvertureBioGithub', - passwordVariable: 'GIT_PASSWORD', - usernameVariable: 'GIT_USERNAME' - )]) { - sh "git tag v${version}" - sh "git push https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/${gitHubRepo} --tags" - } - } - } - } - - stage('Publish Image') { - when { - anyOf { - branch "main" - branch "develop" - } - } - steps { - container('docker') { - withCredentials([usernamePassword(credentialsId:'OvertureBioGithub', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { - sh 'docker login ghcr.io -u $USERNAME -p $PASSWORD' - } - script { - if (env.BRANCH_NAME ==~ 'main') { // push latest and version tags - sh "docker tag server:${commit} ${containerImageName}:${version}" - sh "docker push ${containerImageName}:${version}" - - sh "docker tag server:${commit} ${containerImageName}:latest" - sh "docker push ${containerImageName}:latest" - } else { // push commit tag - sh "docker tag server:${commit} ${containerImageName}:${commit}" - sh "docker push ${containerImageName}:${commit}" - } - - if (env.BRANCH_NAME ==~ 'develop') { // push edge tag - sh "docker tag server:${commit} ${containerImageName}:edge" - sh "docker push ${containerImageName}:edge" - } - } - - } - } - } - } - - post { - failure { - container('node') { - script { - if (env.BRANCH_NAME ==~ /(develop|main|\S*[Tt]est\S*)/) { - sh "curl \ - -X POST \ - -H 'Content-type: application/json' \ - --data '{ \ - \"text\":\"Build Failed: ${env.JOB_NAME}#${commit} \ - \n[Build ${env.BUILD_NUMBER}] (${env.BUILD_URL})\" \ - }' \ - ${slackNotificationsUrl}" - } - } - } - } - - fixed { - container('node') { - script { - if (env.BRANCH_NAME ==~ /(develop|main|\S*[Tt]est\S*)/) { - sh "curl \ - -X POST \ - -H 'Content-type: application/json' \ - --data '{ \ - \"text\":\"Build Fixed: ${env.JOB_NAME}#${commit} \ - \n[Build ${env.BUILD_NUMBER}] (${env.BUILD_URL})\" \ - }' \ - ${slackNotificationsUrl}" - } - } - } - } - - success { - container('node') { - script { - if (env.BRANCH_NAME ==~ /(\S*[Tt]est\S*)/) { - sh "curl \ - -X POST \ - -H 'Content-type: application/json' \ - --data '{ \ - \"text\":\"Build tested: ${env.JOB_NAME}#${commit} \ - \n[Build ${env.BUILD_NUMBER}] (${env.BUILD_URL})\" \ - }' \ - ${slackNotificationsUrl}" - } - } - } - } - } -} From 0eaafe2f79542c4019b3161eea0258e022f977a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciar=C3=A1n=20Sch=C3=BCtte?= Date: Fri, 1 Mar 2024 09:13:55 -0500 Subject: [PATCH 02/27] Update README.md (#197) points to lectern-js-client repo --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3673ee9..c1cc124 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ The modules in the monorepo are organized into three categories: | Component | Type | Package Name | Path | Published Location | Description | | ----------------------------------- | ----------- | ------------------------------ | --------------------- | ------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------- | | [Lectern Server](apps/server/README.md) | Application | server | apps/server/ | [GHCR](https://github.com/overture-stack/lectern/pkgs/container/lectern) | Lectern Server web application. | -| [Lectern Client](packages/client/README.md) | Package | @overture-stack/lectern-client | packages/client | [NPM](https://www.npmjs.com/package/@overturebio-stack/lectern-client) | TypeScript Client to interact with Lectern Server and perform data validation. | +| [Lectern Client](https://github.com/overture-stack/js-lectern-client) | Package | @overture-stack/js-lectern-client | | [NPM](https://www.npmjs.com/package/@overturebio-stack/lectern-client) | TypeScript Client to interact with Lectern Server and perform data validation. | | [common](libraries/common/README.md) | Library | common | libraries/common/ | N/A | Non-specific but commonly reusable utilities. Includes shared Error classes. | | [dictionary](libraries/dictionary/README.md) | Library | dictionary | libraries/dictionary/ | N/A | Dictionary meta-schema definition, includes TS types, and Zod schemas. This also exports all utilities for getting the diff of two dictionaries, and for validating data records with a Dictionary. | @@ -98,4 +98,4 @@ We welcome community contributions! Please follow our [code of conduct](./code_o - Filing an [issue](https://github.com/overture-stack/ego/issues) - Connect with us on [Slack](http://slack.overture.bio) -- Add or Upvote a [feature request](https://github.com/overture-stack/ego/issues?q=is%3Aopen+is%3Aissue+label%3Anew-feature) \ No newline at end of file +- Add or Upvote a [feature request](https://github.com/overture-stack/ego/issues?q=is%3Aopen+is%3Aissue+label%3Anew-feature) From cb0ed9646366e1b11c302ae319d21596aaf21a65 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Tue, 26 Mar 2024 11:45:13 -0400 Subject: [PATCH 03/27] `.env` file location updates for monorepo --- apps/server/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/server/README.md b/apps/server/README.md index 9776aee..c39a5b8 100644 --- a/apps/server/README.md +++ b/apps/server/README.md @@ -66,7 +66,7 @@ A template for environment variables is found at [`./.env.example`](./.env.examp ### From Source -When running from source as detailed in [Build & Run](#build--run), the application will look for a file named `.env` in your current working directory. The easiest way to manage this is to copy [`./.env.example`](./.env.example) into a new file named `.env`. +When running from source as detailed in [Build & Run](#build--run), the application will look for a file named `.env` in the `apps/server` directory. The easiest way to manage this is to copy [`apps/server/.env.example`](./.env.example) into a new file named `apps/server/.env`. ```shell cp .env.example .env From 5c2fd7bc5fa0785cab5e7e736a4f1a8b23af3502 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Tue, 21 May 2024 11:30:01 -0400 Subject: [PATCH 04/27] Meta properties can be nested (#201) * DictionaryMeta now recursive to accomodate nested meta data * Rename x->key in type definition * Typo correction in comment Co-authored-by: Anders Richardsson <2107110+justincorrigible@users.noreply.github.com> * Fix Create Dictionary test fixtures to match new meta rules --------- Co-authored-by: Anders Richardsson <2107110+justincorrigible@users.noreply.github.com> --- .../test/integration/dictionaryRoutes.spec.ts | 2 +- .../integration/fixtures/createKeyValue.json | 4 ++- .../fixtures/createKeyValueBad.json | 7 +++- generated/DictionaryMetaSchema.json | 33 ++++++++++++++++--- .../dictionary/src/types/dictionaryTypes.ts | 17 +++++++--- .../dictionary/src/types/referenceTypes.ts | 2 +- .../dictionary/test/dictionaryTypes.spec.ts | 31 +++++++++++++++++ 7 files changed, 84 insertions(+), 12 deletions(-) diff --git a/apps/server/test/integration/dictionaryRoutes.spec.ts b/apps/server/test/integration/dictionaryRoutes.spec.ts index 6c34fba..93563b0 100644 --- a/apps/server/test/integration/dictionaryRoutes.spec.ts +++ b/apps/server/test/integration/dictionaryRoutes.spec.ts @@ -136,7 +136,7 @@ describe('Dictionary Routes', () => { }); }); - it('Should 400 with meta fields that are not string/boolean/integer/number', (done: Mocha.Done) => { + it('Should 400 with meta fields that are arrays of objects', (done: Mocha.Done) => { const dictRequest = require('./fixtures/createKeyValueBad.json'); chai .request(app) diff --git a/apps/server/test/integration/fixtures/createKeyValue.json b/apps/server/test/integration/fixtures/createKeyValue.json index f20e5c7..89b666c 100644 --- a/apps/server/test/integration/fixtures/createKeyValue.json +++ b/apps/server/test/integration/fixtures/createKeyValue.json @@ -17,7 +17,9 @@ "description": "kv_id", "meta": { "key": true, - "displayName": "KEY VALUE ID" + "displayName": "KEY VALUE ID", + "arraysOk": ["arrays", "are", "ok"], + "nested": { "key": "value", "number": 123, "bool": false } }, "restrictions": { "required": true, diff --git a/apps/server/test/integration/fixtures/createKeyValueBad.json b/apps/server/test/integration/fixtures/createKeyValueBad.json index 34f3c10..6b7cb4b 100644 --- a/apps/server/test/integration/fixtures/createKeyValueBad.json +++ b/apps/server/test/integration/fixtures/createKeyValueBad.json @@ -13,7 +13,12 @@ "meta": { "key": true, "displayName": "KEY VALUE ID", - "noArrays": ["lol", "sorry", "no_arrays", "please"] + "arraysOk": ["arrays", "are", "ok"], + "nested": { "key": "value", "number": 123, "bool": false }, + "errorArrayOfObjects": [ + { "a": "b", "c": 4 }, + { "e": "f", "g": 8 } + ] }, "restrictions": { "required": true, diff --git a/generated/DictionaryMetaSchema.json b/generated/DictionaryMetaSchema.json index e17f7ee..a1d6d61 100644 --- a/generated/DictionaryMetaSchema.json +++ b/generated/DictionaryMetaSchema.json @@ -38,10 +38,35 @@ "Meta": { "type": "object", "additionalProperties": { - "type": [ - "string", - "number", - "boolean" + "anyOf": [ + { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + }, + { + "type": "boolean" + }, + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "array", + "items": { + "type": "number" + } + } + ] + }, + { + "$ref": "#/definitions/Meta" + } ] } }, diff --git a/libraries/dictionary/src/types/dictionaryTypes.ts b/libraries/dictionary/src/types/dictionaryTypes.ts index 060edc6..5d81c00 100644 --- a/libraries/dictionary/src/types/dictionaryTypes.ts +++ b/libraries/dictionary/src/types/dictionaryTypes.ts @@ -29,11 +29,20 @@ export type NameString = zod.infer; export const Integer = zod.number().int(); -// Unlike references, meta is not nested and accepts as values only primitives or arrays of them. -export const DictionaryMetaValue = zod.union([zod.string(), zod.number(), zod.boolean()]); +// Meta accepts as values only strings, numbers, booleans, arrays of numbers or arrays of strings +// Another Meta object can be nested inside a Meta property +export const DictionaryMetaValue = zod.union([ + zod.string(), + zod.number(), + zod.boolean(), + zod.array(zod.string()), + zod.array(zod.number()), +]); export type DictionaryMetaValue = zod.infer; -export const DictionaryMeta = zod.record(DictionaryMetaValue); -export type DictionaryMeta = zod.infer; +export type DictionaryMeta = { [key: string]: DictionaryMetaValue | DictionaryMeta }; +export const DictionaryMeta: zod.ZodType = zod.record( + zod.union([DictionaryMetaValue, zod.lazy(() => DictionaryMeta)]), +); export const SchemaFieldValueType = zod.enum(['string', 'integer', 'number', 'boolean']); export type SchemaFieldValueType = zod.infer; diff --git a/libraries/dictionary/src/types/referenceTypes.ts b/libraries/dictionary/src/types/referenceTypes.ts index dfe1a23..9a8daf7 100644 --- a/libraries/dictionary/src/types/referenceTypes.ts +++ b/libraries/dictionary/src/types/referenceTypes.ts @@ -37,7 +37,7 @@ export type ReferenceArray = zod.infer; // References are recursive, but Zod can't do TS type inference for recursive definitions. // So in this one case we define the type first with the recursive structure and use it as a type-hint // for our zod schema. Reference: https://zod.dev/?id=recursive-types -export type References = { [x: string]: ReferenceArray | ReferenceValue | References }; +export type References = { [key: string]: ReferenceArray | ReferenceValue | References }; export const References: zod.ZodType = zod.record( zod.union([ReferenceValue, ReferenceArray, zod.lazy(() => References)]), //TODO: test this please. ); diff --git a/libraries/dictionary/test/dictionaryTypes.spec.ts b/libraries/dictionary/test/dictionaryTypes.spec.ts index ec06427..571e803 100644 --- a/libraries/dictionary/test/dictionaryTypes.spec.ts +++ b/libraries/dictionary/test/dictionaryTypes.spec.ts @@ -21,6 +21,7 @@ import { expect } from 'chai'; import { BooleanFieldRestrictions, Dictionary, + DictionaryMeta, Integer, IntegerFieldRestrictions, NameString, @@ -471,4 +472,34 @@ describe('Dictionary Types', () => { }); }); }); + describe('Meta', () => { + it('Can accept non-nested values', () => { + const meta = { a: 'string', b: 123, c: true }; + expect(DictionaryMeta.safeParse(meta).success).true; + }); + it('Can accept nested values', () => { + const singleNested = { a: 'string', b: 123, c: true, nested: { d: 'asdf' } }; + const doubleNested = { a: 'string', b: 123, c: true, nested: { d: 'asdf', nested2: { e: 'asdf' } } }; + expect(DictionaryMeta.safeParse(singleNested).success).true; + expect(DictionaryMeta.safeParse(doubleNested).success).true; + }); + it('Can accept arrays of strings', () => { + const meta = { a: 'string', b: 123, c: true, array: ['asdf', 'qwerty'] }; + expect(DictionaryMeta.safeParse(meta).success).true; + }); + it('Can accept arrays of numbers', () => { + const meta = { a: 'string', b: 123, c: true, array: [123, 456, 789] }; + expect(DictionaryMeta.safeParse(meta).success).true; + }); + it('Cannot accept arrays of booleans', () => { + // This constraint feels a bit arbitrary but I can't imagine a clear use case for this. + // Could be changed with discussion + const meta = { a: 'string', b: 123, c: true, array: [true, false, true, false] }; + expect(DictionaryMeta.safeParse(meta).success).false; + }); + it('Cannot accept arrays of mixed types', () => { + const meta = { a: 'string', b: 123, c: true, array: ['asdf', 123] }; + expect(DictionaryMeta.safeParse(meta).success).false; + }); + }); }); From 96fd13ca4e756bb9a1d89d44f3bffac7d55c66e0 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Tue, 11 Jun 2024 09:55:02 -0400 Subject: [PATCH 05/27] Add Lectern Client to monorepo (#208) * Create client package and copy files from js-lectern-client * Use rimraf in build scripts for cross platform compatibility * Target ESNext for internal library builds * Update copyright year of copied client files to 2024 * Remove -rf options from rimraf * Move lectern-client into @overture-stack org * Add mocha tests config file * Ensure we are using `===` not `==` * Remove `baseUrl` from tsconfig --- README.md | 10 +- libraries/common/package.json | 7 +- libraries/common/tsconfig.json | 2 +- libraries/dictionary/package.json | 5 +- libraries/dictionary/tsconfig.json | 2 +- packages/client/.mocharc.json | 5 + packages/client/README.md | 12 + packages/client/package.json | 52 + packages/client/src/change-analyzer.ts | 254 + packages/client/src/index.ts | 26 + packages/client/src/logger.ts | 65 + packages/client/src/parallel.ts | 49 + packages/client/src/records-operations.ts | 103 + packages/client/src/schema-entities.ts | 221 + packages/client/src/schema-error-messages.ts | 109 + packages/client/src/schema-functions.ts | 855 ++ packages/client/src/schema-rest-client.ts | 154 + packages/client/src/schema-worker.js | 51 + packages/client/src/utils.ts | 149 + packages/client/test/change-analyzer.spec.ts | 134 + packages/client/test/schema-diff.json | 312 + packages/client/test/schema-functions.spec.ts | 9295 +++++++++++++++++ packages/client/test/schema.json | 522 + packages/client/tsconfig.json | 21 + packages/client/tslint.json | 43 + pnpm-lock.yaml | 571 +- 26 files changed, 13011 insertions(+), 18 deletions(-) create mode 100644 packages/client/.mocharc.json create mode 100644 packages/client/README.md create mode 100644 packages/client/package.json create mode 100644 packages/client/src/change-analyzer.ts create mode 100644 packages/client/src/index.ts create mode 100644 packages/client/src/logger.ts create mode 100644 packages/client/src/parallel.ts create mode 100644 packages/client/src/records-operations.ts create mode 100644 packages/client/src/schema-entities.ts create mode 100644 packages/client/src/schema-error-messages.ts create mode 100644 packages/client/src/schema-functions.ts create mode 100644 packages/client/src/schema-rest-client.ts create mode 100644 packages/client/src/schema-worker.js create mode 100644 packages/client/src/utils.ts create mode 100644 packages/client/test/change-analyzer.spec.ts create mode 100644 packages/client/test/schema-diff.json create mode 100644 packages/client/test/schema-functions.spec.ts create mode 100644 packages/client/test/schema.json create mode 100644 packages/client/tsconfig.json create mode 100644 packages/client/tslint.json diff --git a/README.md b/README.md index c1cc124..f03434b 100644 --- a/README.md +++ b/README.md @@ -36,11 +36,11 @@ The modules in the monorepo are organized into three categories: * __libraries/__ - Interal modules shared between other apps, libraries, and packages. * __packages/__ - Packages published to [NPM](https://npmjs.com) meant to be imported into other TypeScript applications. -| Component | Type | Package Name | Path | Published Location | Description | -| ----------------------------------- | ----------- | ------------------------------ | --------------------- | ------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------- | -| [Lectern Server](apps/server/README.md) | Application | server | apps/server/ | [GHCR](https://github.com/overture-stack/lectern/pkgs/container/lectern) | Lectern Server web application. | -| [Lectern Client](https://github.com/overture-stack/js-lectern-client) | Package | @overture-stack/js-lectern-client | | [NPM](https://www.npmjs.com/package/@overturebio-stack/lectern-client) | TypeScript Client to interact with Lectern Server and perform data validation. | -| [common](libraries/common/README.md) | Library | common | libraries/common/ | N/A | Non-specific but commonly reusable utilities. Includes shared Error classes. | +| Component | Type | Package Name | Path | Published Location | Description | +| -------------------------------------------- | ----------- | ------------------------------ | --------------------- | ------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [Lectern Server](apps/server/README.md) | Application | server | apps/server/ | [GHCR](https://github.com/overture-stack/lectern/pkgs/container/lectern) | Lectern Server web application. | +| [Lectern Client](packages/client/README.md) | Package | @overture-stack/lectern-client | packages/client | [NPM](https://www.npmjs.com/package/@overturebio-stack/lectern-client) | TypeScript Client to interact with Lectern Server and perform data validation. | +| [common](libraries/common/README.md) | Library | common | libraries/common/ | N/A | Non-specific but commonly reusable utilities. Includes shared Error classes. | | [dictionary](libraries/dictionary/README.md) | Library | dictionary | libraries/dictionary/ | N/A | Dictionary meta-schema definition, includes TS types, and Zod schemas. This also exports all utilities for getting the diff of two dictionaries, and for validating data records with a Dictionary. | ## Developer Instructions diff --git a/libraries/common/package.json b/libraries/common/package.json index dca1016..322db78 100644 --- a/libraries/common/package.json +++ b/libraries/common/package.json @@ -5,9 +5,12 @@ "main": "dist/index.js", "scripts": { "build": "pnpm build:clean && tsc", - "build:clean": "rm -rf dist/ && mkdir dist" + "build:clean": "rimraf dist/ && mkdir dist" }, "keywords": [], "author": "Ontario Institute for Cancer Research", - "license": "AGPL-3.0" + "license": "AGPL-3.0", + "devDependencies": { + "rimraf": "^3.0.2" + } } diff --git a/libraries/common/tsconfig.json b/libraries/common/tsconfig.json index d14a93f..90dcbbd 100644 --- a/libraries/common/tsconfig.json +++ b/libraries/common/tsconfig.json @@ -1,6 +1,6 @@ { "compilerOptions": { - "target": "ES5", + "target": "ESNext", "lib": ["ESNext"], "module": "CommonJS", "moduleResolution": "node", diff --git a/libraries/dictionary/package.json b/libraries/dictionary/package.json index 6223256..0b74639 100644 --- a/libraries/dictionary/package.json +++ b/libraries/dictionary/package.json @@ -5,7 +5,7 @@ "main": "dist/index.js", "scripts": { "build": "pnpm build:clean && tsc", - "build:clean": "rm -rf dist/ && mkdir dist", + "build:clean": "rimraf dist/ && mkdir dist", "test": "nyc mocha" }, "keywords": [], @@ -18,6 +18,7 @@ "zod": "^3.21.4" }, "devDependencies": { - "@types/lodash": "^4.14.195" + "@types/lodash": "^4.14.195", + "rimraf": "^3.0.2" } } diff --git a/libraries/dictionary/tsconfig.json b/libraries/dictionary/tsconfig.json index 38783ea..c480d7e 100644 --- a/libraries/dictionary/tsconfig.json +++ b/libraries/dictionary/tsconfig.json @@ -1,6 +1,6 @@ { "compilerOptions": { - "target": "ES5", + "target": "ESNext", "lib": ["ESNext"], "module": "CommonJS", "moduleResolution": "node", diff --git a/packages/client/.mocharc.json b/packages/client/.mocharc.json new file mode 100644 index 0000000..6d0022d --- /dev/null +++ b/packages/client/.mocharc.json @@ -0,0 +1,5 @@ +{ + "extension": ["ts"], + "require": "ts-node/register", + "spec": "test/**/*.spec.ts" +} diff --git a/packages/client/README.md b/packages/client/README.md new file mode 100644 index 0000000..e6ac7ed --- /dev/null +++ b/packages/client/README.md @@ -0,0 +1,12 @@ +# Lectern TypeScript Client + +## Features: +- Runs different restrictions validations: regex, range, scripts, required fields, type checks, etc. +- Transforms the data from string to their proper type. +- Report validation errors. +- Fetch dictionaries from the configured lectern service. +- Provide typed definitions for the dictionary object. +- Analyze dictionary versions diff. + +## Usage examples: +- icgc-argo/argo-clinical [https://github.com/icgc-argo/argo-clinical] \ No newline at end of file diff --git a/packages/client/package.json b/packages/client/package.json new file mode 100644 index 0000000..dbdbef7 --- /dev/null +++ b/packages/client/package.json @@ -0,0 +1,52 @@ +{ + "name": "@overture-stack/lectern-client", + "version": "1.5.0", + "files": [ + "dist/" + ], + "main": "./dist/index.js", + "typings": "./dist/index.d.ts", + "description": "TypeScript client to interact with Lectern servers and perform data validation versus Lectern dictionaries.", + "scripts": { + "build": "rimraf dist && tsc", + "test": "nyc mocha --exit --timeout 5000 -r ts-node/register test/**.spec.ts", + "lint": "tslint -c tslint.json -e node_modules -p tsconfig.json" + }, + "repository": { + "type": "git", + "url": "https://github.com/overture-stack/lectern.git" + }, + "publishConfig": { + "access": "public" + }, + "license": "AGPL-3.0", + "devDependencies": { + "@types/chai": "^4.2.16", + "@types/deep-freeze": "^0.1.2", + "@types/lodash": "^4.14.195", + "@types/mocha": "^8.2.2", + "@types/node": "^12.0.10", + "@types/node-fetch": "^2.5.10", + "chai": "^4.3.4", + "husky": "^6.0.0", + "mocha": "^8.3.2", + "prettier": "^2.2.1", + "pretty-quick": "^3.1.0", + "rimraf": "^3.0.2", + "ts-node": "^9.1.1", + "tslint": "^6.1.3", + "typedoc": "^0.17.7", + "typescript": "^5.1.6" + }, + "dependencies": { + "cd": "^0.3.3", + "common": "workspace:^", + "deep-freeze": "^0.0.1", + "lodash": "^4.17.21", + "node-fetch": "^2.6.1", + "node-worker-threads-pool": "^1.4.3", + "promise-tools": "^2.1.0", + "winston": "^3.3.3" + }, + "author": "Ontario Institute for Cancer Research" +} diff --git a/packages/client/src/change-analyzer.ts b/packages/client/src/change-analyzer.ts new file mode 100644 index 0000000..02e2c08 --- /dev/null +++ b/packages/client/src/change-analyzer.ts @@ -0,0 +1,254 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { restClient } from './schema-rest-client'; +import { + SchemasDictionaryDiffs, + FieldChanges, + FieldDiff, + Change, + ChangeAnalysis, + ChangeTypeName, + RestrictionChanges, + FieldDefinition, +} from './schema-entities'; + +const isFieldChange = (obj: any): obj is Change => { + return obj.type !== undefined; +}; + +const isNestedChange = (obj: any): obj is { [field: string]: FieldChanges } => { + return obj.type === undefined; +}; + +const isRestrictionChange = (obj: any): obj is { [field: string]: FieldChanges } => { + return obj.type === undefined; +}; + +export const fetchDiffAndAnalyze = async (serviceUrl: string, name: string, fromVersion: string, toVersion: string) => { + const changes = await restClient.fetchDiff(serviceUrl, name, fromVersion, toVersion); + return analyzeChanges(changes); +}; + +export const analyzeChanges = (schemasDiff: SchemasDictionaryDiffs): ChangeAnalysis => { + const analysis: ChangeAnalysis = { + fields: { + addedFields: [], + renamedFields: [], + deletedFields: [], + }, + isArrayDesignationChanges: [], + restrictionsChanges: { + codeList: { + created: [], + deleted: [], + updated: [], + }, + regex: { + updated: [], + created: [], + deleted: [], + }, + required: { + updated: [], + created: [], + deleted: [], + }, + script: { + updated: [], + created: [], + deleted: [], + }, + range: { + updated: [], + created: [], + deleted: [], + }, + }, + metaChanges: { + core: { + changedToCore: [], + changedFromCore: [], + }, + }, + valueTypeChanges: [], + }; + + for (const field of Object.keys(schemasDiff)) { + const fieldChange: FieldDiff = schemasDiff[field]; + if (fieldChange) { + const fieldDiff = fieldChange.diff; + // if we have type at first level then it's a field add/delete + if (isFieldChange(fieldDiff)) { + categorizeFieldChanges(analysis, field, fieldDiff); + } + + if (isNestedChange(fieldDiff)) { + if (fieldDiff.meta) { + categorizeMetaChagnes(analysis, field, fieldDiff.meta); + } + + if (fieldDiff.restrictions) { + categorizeRestrictionChanges(analysis, field, fieldDiff.restrictions, fieldChange.after); + } + + if (fieldDiff.isArray) { + categorizeFieldArrayDesignationChange(analysis, field, fieldDiff.isArray); + } + + if (fieldDiff.valueType) { + categorizerValueTypeChange(analysis, field, fieldDiff.valueType); + } + } + } + } + + return analysis; +}; + +const categorizeFieldArrayDesignationChange = ( + analysis: ChangeAnalysis, + field: string, + changes: { [field: string]: FieldChanges } | Change, +) => { + // changing isArray designation is a relevant change for all cases except if it is created and set to false + if (!(changes.type === 'created' && changes.data === false)) { + analysis.isArrayDesignationChanges.push(field); + } +}; + +const categorizerValueTypeChange = ( + analysis: ChangeAnalysis, + field: string, + changes: { [field: string]: FieldChanges } | Change, +) => { + analysis.valueTypeChanges.push(field); +}; + +const categorizeRestrictionChanges = ( + analysis: ChangeAnalysis, + field: string, + restrictionsChange: { [field: string]: FieldChanges } | Change, + fieldDefinitionAfter?: FieldDefinition, +) => { + const restrictionsToCheck = ['regex', 'script', 'required', 'codeList', 'range']; + + // additions or deletions of a restriction object as whole (i.e. contains 1 or many restrictions within the 'data') + if (restrictionsChange.type) { + const createOrAddChange = restrictionsChange as Change; + const restrictionsData = createOrAddChange.data as any; + + for (const k of restrictionsToCheck) { + if (restrictionsData[k]) { + analysis.restrictionsChanges[k as keyof RestrictionChanges][restrictionsChange.type as ChangeTypeName].push({ + field: field, + definition: restrictionsData[k], + } as any); + } + } + return; + } + + // in case 'restrictions' key was already there but we modified its contents + const restrictionUpdate = restrictionsChange as { [field: string]: FieldChanges }; + for (const k of restrictionsToCheck) { + if (restrictionUpdate[k]) { + const change = restrictionUpdate[k] as Change; + // we need the '|| change' in case of nested attributes like ranges + /* + "diff": { + "restrictions": { + "range": { + "exclusiveMin": { + "type": "deleted", + "data": 0 + }, + "max": { + "type": "updated", + "data": 200000 + }, + "min": { + "type": "created", + "data": 0 + } + } + } + } + */ + if (k == 'range' && !change.type) { + // if the change is nested (type is at min max level) then the boundries were updated only : ex: + /* + change = { + "max" : { + type: "updated" + data: "..." + }, + "exclusiveMin": { + type: "deleted" + data .. + } + } + */ + const def: any = {}; + if (Object.keys(change).some((k) => k == 'max' || k == 'min' || k == 'exclusiveMin' || k == 'exclusiveMax')) { + analysis.restrictionsChanges[k]['updated'].push({ + field: field, + // we push the whole range definition since it doesnt make sense to just + // push one boundary. + definition: fieldDefinitionAfter?.restrictions?.range, + }); + } + return; + } + const definition = change.data || change; + analysis.restrictionsChanges[k as keyof RestrictionChanges][change.type as ChangeTypeName].push({ + field: field, + definition, + } as any); + } + } +}; + +const categorizeFieldChanges = (analysis: ChangeAnalysis, field: string, changes: Change) => { + const changeType = changes.type; + if (changeType == 'created') { + analysis.fields.addedFields.push({ + name: field, + definition: changes.data, + }); + } else if (changeType == 'deleted') { + analysis.fields.deletedFields.push(field); + } +}; + +const categorizeMetaChagnes = ( + analysis: ChangeAnalysis, + field: string, + metaChanges: { [field: string]: FieldChanges } | Change, +) => { + // **** meta changes - core *** + if (metaChanges?.data?.core === true) { + const changeType = metaChanges.type; + if (changeType === 'created' || changeType === 'updated') { + analysis.metaChanges?.core.changedToCore.push(field); + } else if (changeType === 'deleted') { + analysis.metaChanges?.core.changedFromCore.push(field); + } + } +}; diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts new file mode 100644 index 0000000..e7cf786 --- /dev/null +++ b/packages/client/src/index.ts @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import * as entities from './schema-entities'; +import * as analyzer from './change-analyzer'; +import * as functions from './schema-functions'; +import * as parallel from './parallel'; + +import { restClient } from './schema-rest-client'; +export { entities, analyzer, functions, parallel, restClient }; diff --git a/packages/client/src/logger.ts b/packages/client/src/logger.ts new file mode 100644 index 0000000..750389d --- /dev/null +++ b/packages/client/src/logger.ts @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import winston from 'winston'; +const { createLogger, format, transports } = winston; +const { combine, timestamp, label, prettyPrint, json, align, simple } = format; + +// read the log level from the env directly since this is a very high priority value. +const LOG_LEVEL = process.env.LOG_LEVEL || 'info'; +console.log('log level configured: ', LOG_LEVEL); + +// Logger configuration +const logConfiguration = { + level: LOG_LEVEL, + format: combine(json(), simple(), timestamp()), + transports: [new winston.transports.Console()], +}; + +export interface Logger { + error(msg: string, err?: Error): void; + info(msg: string): void; + debug(msg: string): void; + profile(s: string): void; +} + +const winstonLogger = winston.createLogger(logConfiguration); +if (process.env.LOG_LEVEL === 'debug') { + console.log('logger configured: ', winstonLogger); +} +export const loggerFor = (fileName: string): Logger => { + if (process.env.LOG_LEVEL === 'debug') { + console.debug('creating logger for', fileName); + } + const source = fileName.substring(fileName.indexOf('argo-clinical')); + return { + error: (msg: string, err: Error): void => { + winstonLogger.error(msg, err, { source }); + }, + debug: (msg: string): void => { + winstonLogger.debug(msg, { source }); + }, + info: (msg: string): void => { + winstonLogger.info(msg, { source }); + }, + profile: (id: string): void => { + winstonLogger.profile(id); + }, + }; +}; diff --git a/packages/client/src/parallel.ts b/packages/client/src/parallel.ts new file mode 100644 index 0000000..603d57f --- /dev/null +++ b/packages/client/src/parallel.ts @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { DataRecord, SchemasDictionary, SchemaProcessingResult } from './schema-entities'; +import { StaticPool } from 'node-worker-threads-pool'; +import * as os from 'os'; +import { loggerFor } from './logger'; +const L = loggerFor(__filename); + +// check allowed cpus or use available +const cpuCount = os.cpus().length; +L.info(`available cpus: ${cpuCount}`); +const availableCpus = Number(process.env.ALLOWED_CPUS) || cpuCount; +L.info(`using ${availableCpus} cpus`); + +const pool = new StaticPool({ + size: availableCpus, + task: __dirname + '/schema-worker.js', +}); + +export const processRecord = async ( + dictionary: SchemasDictionary, + schemaName: string, + record: Readonly, + index: number, +): Promise => { + return (await pool.exec({ + dictionary, + schemaName, + record, + index, + })) as Promise; +}; diff --git a/packages/client/src/records-operations.ts b/packages/client/src/records-operations.ts new file mode 100644 index 0000000..7b964f3 --- /dev/null +++ b/packages/client/src/records-operations.ts @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2023 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { differenceWith, isEqual } from 'lodash'; + +/** + * Renames properties in a record using a mapping between current and new names. + * @param record The record whose properties should be renamed. + * @param fieldsMapping A mapping of current property names to new property names. + * @returns A new record with the properties' names changed according to the mapping. + */ +const renameProperties = ( + record: Record, + fieldsMapping: Map, +): Record => { + const renamed: Record = {}; + Object.entries(record).forEach(([propertyName, propertyValue]) => { + const newName = fieldsMapping.get(propertyName) ?? propertyName; + renamed[newName] = propertyValue; + }); + return renamed; +}; + +/** + * Returns a string representation of a record. The record is sorted by its properties so + * 2 records which have the same properties and values (even if in different order) will produce the + * same string with this function. + * @param record Record to be processed. + * @returns String representation of the record sorted by its properties. + */ +const getSortedRecordKey = (record: Record): string => { + const sortedKeys = Object.keys(record).sort(); + const sortedRecord: Record = {}; + for (const key of sortedKeys) { + sortedRecord[key] = record[key]; + } + return JSON.stringify(sortedRecord); +}; + +/** + * Find missing foreign keys by calculating the difference between 2 dataset keys (similar to a set difference). + * Returns rows in `dataKeysA` which are not present in `dataKeysB`. + * @param datasetKeysA Keys of the dataset A. The returned value of this function is a subset of this array. + * @param datasetKeysB Keys of the dataset B. Elements to be substracted from `datasetKeysA`. + * @param fieldsMapping Mapping of the field names so the keys can be compared correctly. + */ +export const findMissingForeignKeys = ( + datasetKeysA: [number, Record][], + datasetKeysB: [number, Record][], + fieldsMapping: Map, +): [number, Record][] => { + const diff = differenceWith(datasetKeysA, datasetKeysB, (a, b) => + isEqual(a[1], renameProperties(b[1], fieldsMapping)), + ); + return diff; +}; + +/** + * Find duplicate keys in a dataset. + * @param datasetKeys Array with the keys to evaluate. + * @returns An Array with all the values that appear more than once in the dataset. + */ +export const findDuplicateKeys = ( + datasetKeys: [number, Record][], +): [number, Record][] => { + const duplicateKeys: [number, Record][] = []; + const recordKeysMap: Map<[number, Record], string> = new Map(); + const keyCount: Map = new Map(); + + // Calculate a key per record, which is a string representation that allows to compare records even if their properties + // are in different order + datasetKeys.forEach((row) => { + const recordKey = getSortedRecordKey(row[1]); + const count = keyCount.get(recordKey) || 0; + keyCount.set(recordKey, count + 1); + recordKeysMap.set(row, recordKey); + }); + + // Find duplicates by checking the count of they key on each record + recordKeysMap.forEach((value, key) => { + const count = keyCount.get(value) ?? 0; + if (count > 1) { + duplicateKeys.push(key); + } + }); + return duplicateKeys; +}; diff --git a/packages/client/src/schema-entities.ts b/packages/client/src/schema-entities.ts new file mode 100644 index 0000000..810c9c8 --- /dev/null +++ b/packages/client/src/schema-entities.ts @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { loggerFor } from './logger'; +import { DeepReadonly } from 'deep-freeze'; +const L = loggerFor(__filename); + +export class DataRecord { + readonly [k: string]: string | string[]; +} + +export class TypedDataRecord { + readonly [k: string]: SchemaTypes; +} + +export type SchemaTypes = string | string[] | boolean | boolean[] | number | number[] | undefined; + +export interface SchemasDictionary { + version: string; + name: string; + schemas: Array; +} + +export interface SchemaDefinition { + readonly name: string; + readonly description: string; + readonly restrictions: SchemaRestriction; + readonly fields: ReadonlyArray; +} + +export interface SchemasDictionaryDiffs { + [fieldName: string]: FieldDiff; +} + +export interface FieldDiff { + before?: FieldDefinition; + after?: FieldDefinition; + diff: FieldChanges; +} + +export type SchemaData = ReadonlyArray; + +// changes can be nested +// in case of created/delete field we get Change +// in case of simple field change we get {"fieldName": {"data":.., "type": ..}} +// in case of nested fields: {"fieldName1": {"fieldName2": {"data":.., "type": ..}}} +export type FieldChanges = { [field: string]: FieldChanges } | Change; + +export enum ChangeTypeName { + CREATED = 'created', + DELETED = 'deleted', + UPDATED = 'updated', +} + +export interface Change { + type: ChangeTypeName; + data: any; +} + +export interface SchemaRestriction { + foreignKey?: { + schema: string; + mappings: { + local: string; + foreign: string; + }[]; + }[]; + uniqueKey?: string[]; +} + +export interface FieldDefinition { + name: string; + valueType: ValueType; + description: string; + meta?: { key?: boolean; default?: SchemaTypes; core?: boolean; examples?: string }; + restrictions?: { + codeList?: CodeListRestriction; + regex?: string; + script?: Array | string; + required?: boolean; + unique?: boolean; + range?: RangeRestriction; + }; + isArray?: boolean; +} + +export type CodeListRestriction = Array; + +export type RangeRestriction = { + min?: number; + max?: number; + exclusiveMin?: number; + exclusiveMax?: number; +}; + +export enum ValueType { + STRING = 'string', + INTEGER = 'integer', + NUMBER = 'number', + BOOLEAN = 'boolean', +} + +export type SchemaProcessingResult = DeepReadonly<{ + validationErrors: SchemaValidationError[]; + processedRecord: TypedDataRecord; +}>; + +export type BatchProcessingResult = DeepReadonly<{ + validationErrors: SchemaValidationError[]; + processedRecords: TypedDataRecord[]; +}>; + +export enum SchemaValidationErrorTypes { + MISSING_REQUIRED_FIELD = 'MISSING_REQUIRED_FIELD', + INVALID_FIELD_VALUE_TYPE = 'INVALID_FIELD_VALUE_TYPE', + INVALID_BY_REGEX = 'INVALID_BY_REGEX', + INVALID_BY_RANGE = 'INVALID_BY_RANGE', + INVALID_BY_SCRIPT = 'INVALID_BY_SCRIPT', + INVALID_ENUM_VALUE = 'INVALID_ENUM_VALUE', + UNRECOGNIZED_FIELD = 'UNRECOGNIZED_FIELD', + INVALID_BY_UNIQUE = 'INVALID_BY_UNIQUE', + INVALID_BY_FOREIGN_KEY = 'INVALID_BY_FOREIGN_KEY', + INVALID_BY_UNIQUE_KEY = 'INVALID_BY_UNIQUE_KEY', +} + +export interface SchemaValidationError { + readonly errorType: SchemaValidationErrorTypes; + readonly index: number; + readonly fieldName: string; + readonly info: Record; + readonly message: string; +} + +export interface FieldNamesByPriorityMap { + required: string[]; + optional: string[]; +} + +export interface ChangeAnalysis { + fields: { + addedFields: AddedFieldChange[]; + renamedFields: string[]; + deletedFields: string[]; + }; + isArrayDesignationChanges: string[]; + restrictionsChanges: RestrictionChanges; + metaChanges?: MetaChanges; + valueTypeChanges: string[]; +} + +export type RestrictionChanges = { + range: { + [key in ChangeTypeName]: ObjectChange[]; + }; + codeList: { + [key in ChangeTypeName]: ObjectChange[]; + }; + regex: RegexChanges; + required: RequiredChanges; + script: ScriptChanges; +}; + +export type MetaChanges = { + core: { + changedToCore: string[]; // fields that are core now + changedFromCore: string[]; // fields that are not core now + }; +}; + +export type RegexChanges = { + [key in ChangeTypeName]: StringAttributeChange[]; +}; + +export type RequiredChanges = { + [key in ChangeTypeName]: BooleanAttributeChange[]; +}; + +export type ScriptChanges = { + [key in ChangeTypeName]: StringAttributeChange[]; +}; + +export interface AddedFieldChange { + name: string; + definition: FieldDefinition; +} + +export interface ObjectChange { + field: string; + definition: any; +} + +export interface CodeListChange { + field: string; + definition: any; +} + +export interface StringAttributeChange { + field: string; + definition: string; +} + +export interface BooleanAttributeChange { + field: string; + definition: boolean; +} diff --git a/packages/client/src/schema-error-messages.ts b/packages/client/src/schema-error-messages.ts new file mode 100644 index 0000000..3ed08ab --- /dev/null +++ b/packages/client/src/schema-error-messages.ts @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { isArray } from 'lodash'; +import { RangeRestriction } from './schema-entities'; + +function getForeignKeyErrorMsg(errorData: any) { + const valueEntries = Object.entries(errorData.info.value); + const formattedKeyValues: string[] = valueEntries.map(([key, value]) => { + if (isArray(value)) { + return `${key}: [${value.join(', ')}]`; + } else { + return `${key}: ${value}`; + } + }); + const valuesAsString = formattedKeyValues.join(', '); + const detail = `Key ${valuesAsString} is not present in schema ${errorData.info.foreignSchema}`; + const msg = `Record violates foreign key restriction defined for field(s) ${errorData.fieldName}. ${detail}.`; + return msg; +} + +function getUniqueKeyErrorMsg(errorData: any) { + const uniqueKeyFields: string[] = errorData.info.uniqueKeyFields; + const formattedKeyValues: string[] = uniqueKeyFields.map((fieldName) => { + const value = errorData.info.value[fieldName]; + if (isArray(value)) { + return `${fieldName}: [${value.join(', ')}]`; + } else { + return `${fieldName}: ${value === '' ? 'null' : value}`; + } + }); + const valuesAsString = formattedKeyValues.join(', '); + const msg = `Key ${valuesAsString} must be unique.`; + return msg; +} + +const INVALID_VALUE_ERROR_MESSAGE = 'The value is not permissible for this field.'; +const ERROR_MESSAGES: { [key: string]: (errorData: any) => string } = { + INVALID_FIELD_VALUE_TYPE: () => INVALID_VALUE_ERROR_MESSAGE, + INVALID_BY_REGEX: (errData) => getRegexErrorMsg(errData.info), + INVALID_BY_RANGE: (errorData) => `Value is out of permissible range, value must be ${rangeToSymbol(errorData.info)}.`, + INVALID_BY_SCRIPT: (error) => error.info.message, + INVALID_ENUM_VALUE: () => INVALID_VALUE_ERROR_MESSAGE, + MISSING_REQUIRED_FIELD: (errorData) => `${errorData.fieldName} is a required field.`, + INVALID_BY_UNIQUE: (errorData) => `Value for ${errorData.fieldName} must be unique.`, + INVALID_BY_FOREIGN_KEY: (errorData) => getForeignKeyErrorMsg(errorData), + INVALID_BY_UNIQUE_KEY: (errorData) => getUniqueKeyErrorMsg(errorData), +}; + +// Returns the formatted message for the given error key, taking any required properties from the info object +// Default value is the errorType itself (so we can identify errorTypes that we are missing messages for and the user could look up the error meaning in our docs) +const schemaErrorMessage = (errorType: string, errorData: any = {}): string => { + return errorType && Object.keys(ERROR_MESSAGES).includes(errorType) + ? ERROR_MESSAGES[errorType](errorData) + : errorType; +}; + +const rangeToSymbol = (range: RangeRestriction): string => { + let minString = ''; + let maxString = ''; + + const hasBothRange = + (range.min !== undefined || range.exclusiveMin !== undefined) && + (range.max != undefined || range.exclusiveMax !== undefined); + + if (range.min !== undefined) { + minString = `>= ${range.min}`; + } + + if (range.exclusiveMin !== undefined) { + minString = `> ${range.exclusiveMin}`; + } + + if (range.max !== undefined) { + maxString = `<= ${range.max}`; + } + + if (range.exclusiveMax !== undefined) { + maxString = `< ${range.exclusiveMax}`; + } + + return hasBothRange ? `${minString} and ${maxString}` : `${minString}${maxString}`; +}; + +function getRegexErrorMsg(info: any) { + let msg = `The value is not a permissible for this field, it must meet the regular expression: "${info.regex}".`; + if (info.examples) { + msg = msg + ` Examples: ${info.examples}`; + } + return msg; +} + +export default schemaErrorMessage; diff --git a/packages/client/src/schema-functions.ts b/packages/client/src/schema-functions.ts new file mode 100644 index 0000000..52c0d7f --- /dev/null +++ b/packages/client/src/schema-functions.ts @@ -0,0 +1,855 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { + SchemaValidationError, + TypedDataRecord, + SchemaTypes, + SchemaProcessingResult, + FieldNamesByPriorityMap, + BatchProcessingResult, + CodeListRestriction, + RangeRestriction, + SchemaData, +} from './schema-entities'; +import vm from 'vm'; +import { + SchemasDictionary, + SchemaDefinition, + FieldDefinition, + ValueType, + DataRecord, + SchemaValidationErrorTypes, +} from './schema-entities'; + +import { + Checks, + notEmpty, + isEmptyString, + isAbsent, + F, + isNotAbsent, + isStringArray, + isString, + isEmpty, + convertToArray, + isNumberArray, +} from './utils'; +import schemaErrorMessage from './schema-error-messages'; +import { loggerFor } from './logger'; +import { DeepReadonly } from 'deep-freeze'; +import _, { isArray } from 'lodash'; +import { findDuplicateKeys, findMissingForeignKeys } from './records-operations'; +const L = loggerFor(__filename); + +export const getSchemaFieldNamesWithPriority = ( + schema: SchemasDictionary, + definition: string, +): FieldNamesByPriorityMap => { + const schemaDef: SchemaDefinition | undefined = schema.schemas.find((schema) => schema.name === definition); + if (!schemaDef) { + throw new Error(`no schema found for : ${definition}`); + } + const fieldNamesMapped: FieldNamesByPriorityMap = { required: [], optional: [] }; + schemaDef.fields.forEach((field) => { + if (field.restrictions && field.restrictions.required) { + fieldNamesMapped.required.push(field.name); + } else { + fieldNamesMapped.optional.push(field.name); + } + }); + return fieldNamesMapped; +}; + +const getNotNullSchemaDefinitionFromDictionary = ( + dictionary: SchemasDictionary, + schemaName: string, +): SchemaDefinition => { + const schemaDef: SchemaDefinition | undefined = dictionary.schemas.find((e) => e.name === schemaName); + if (!schemaDef) { + throw new Error(`no schema found for : ${schemaName}`); + } + return schemaDef; +}; + +export const processSchemas = ( + dictionary: SchemasDictionary, + schemasData: Record, +): Record => { + Checks.checkNotNull('dictionary', dictionary); + Checks.checkNotNull('schemasData', schemasData); + + const results: Record = {}; + + Object.keys(schemasData).forEach((schemaName) => { + // Run validations at the record level + const recordLevelValidationResults = processRecords(dictionary, schemaName, schemasData[schemaName]); + + // Run cross-schema validations + const schemaDef: SchemaDefinition = getNotNullSchemaDefinitionFromDictionary(dictionary, schemaName); + const crossSchemaLevelValidationResults = validation + .runCrossSchemaValidationPipeline(schemaDef, schemasData, [validation.validateForeignKey]) + .filter(notEmpty); + + const recordLevelErrors = recordLevelValidationResults.validationErrors.map((x) => { + return { + errorType: x.errorType, + index: x.index, + fieldName: x.fieldName, + info: x.info, + message: x.message, + }; + }); + + const crossSchemaLevelErrors = crossSchemaLevelValidationResults.map((x) => { + return { + errorType: x.errorType, + index: x.index, + fieldName: x.fieldName, + info: x.info, + message: x.message, + }; + }); + + const allErrorsBySchema = [...recordLevelErrors, ...crossSchemaLevelErrors]; + + results[schemaName] = F({ + validationErrors: allErrorsBySchema, + processedRecords: recordLevelValidationResults.processedRecords, + }); + }); + + return results; +}; + +export const processRecords = ( + dataSchema: SchemasDictionary, + definition: string, + records: ReadonlyArray, +): BatchProcessingResult => { + Checks.checkNotNull('records', records); + Checks.checkNotNull('dataSchema', dataSchema); + Checks.checkNotNull('definition', definition); + + const schemaDef: SchemaDefinition = getNotNullSchemaDefinitionFromDictionary(dataSchema, definition); + + let validationErrors: SchemaValidationError[] = []; + const processedRecords: TypedDataRecord[] = []; + + records.forEach((r, i) => { + const result = process(dataSchema, definition, r, i); + validationErrors = validationErrors.concat(result.validationErrors); + processedRecords.push(_.cloneDeep(result.processedRecord) as TypedDataRecord); + }); + // Record set level validations + const newErrors = validateRecordsSet(schemaDef, processedRecords); + validationErrors.push(...newErrors); + L.debug( + `done processing all rows, validationErrors: ${validationErrors.length}, validRecords: ${processedRecords.length}`, + ); + + return F({ + validationErrors, + processedRecords, + }); +}; + +export const process = ( + dataSchema: SchemasDictionary, + definition: string, + rec: Readonly, + index: number, +): SchemaProcessingResult => { + Checks.checkNotNull('records', rec); + Checks.checkNotNull('dataSchema', dataSchema); + Checks.checkNotNull('definition', definition); + + const schemaDef: SchemaDefinition | undefined = dataSchema.schemas.find((e) => e.name === definition); + + if (!schemaDef) { + throw new Error(`no schema found for : ${definition}`); + } + + let validationErrors: SchemaValidationError[] = []; + + const defaultedRecord: DataRecord = populateDefaults(schemaDef, F(rec), index); + L.debug(`done populating defaults for record #${index}`); + const result = validate(schemaDef, defaultedRecord, index); + L.debug(`done validation for record #${index}`); + if (result && result.length > 0) { + L.debug(`${result.length} validation errors for record #${index}`); + validationErrors = validationErrors.concat(result); + } + const convertedRecord = convertFromRawStrings(schemaDef, defaultedRecord, index, result); + L.debug(`converted row #${index} from raw strings`); + const postTypeConversionValidationResult = validateAfterTypeConversion( + schemaDef, + _.cloneDeep(convertedRecord) as DataRecord, + index, + ); + + if (postTypeConversionValidationResult && postTypeConversionValidationResult.length > 0) { + validationErrors = validationErrors.concat(postTypeConversionValidationResult); + } + + L.debug(`done processing all rows, validationErrors: ${validationErrors.length}, validRecords: ${convertedRecord}`); + + return F({ + validationErrors, + processedRecord: convertedRecord, + }); +}; + +/** + * Populate the passed records with the default value based on the field name if the field is + * missing from the records it will NOT be added. + * @param definition the name of the schema definition to use for these records + * @param records the list of records to populate with the default values. + */ +const populateDefaults = ( + schemaDef: Readonly, + record: DeepReadonly, + index: number, +): DataRecord => { + Checks.checkNotNull('records', record); + L.debug(`in populateDefaults ${schemaDef.name}, ${record}`); + const mutableRecord: RawMutableRecord = _.cloneDeep(record) as RawMutableRecord; + const x: SchemaDefinition = schemaDef; + schemaDef.fields.forEach((field) => { + const defaultValue = field.meta && field.meta.default; + if (isEmpty(defaultValue)) return undefined; + + const value = record[field.name]; + + // data record value is (or is expected to be) just one string + if (isString(value) && !field.isArray) { + if (isNotAbsent(value) && value.trim() === '') { + L.debug(`populating Default: ${defaultValue} for ${field.name} in record : ${record}`); + mutableRecord[field.name] = `${defaultValue}`; + } + return undefined; + } + + // data record value is (or is expected to be) array of string + if (isStringArray(value) && field.isArray) { + if (notEmpty(value) && value.every((v) => v.trim() === '')) { + L.debug(`populating Default: ${defaultValue} for ${field.name} in record : ${record}`); + const arrayDefaultValue = convertToArray(defaultValue); + mutableRecord[field.name] = arrayDefaultValue.map((v) => `${v}`); + } + return undefined; + } + }); + + return _.cloneDeep(mutableRecord); +}; + +const convertFromRawStrings = ( + schemaDef: SchemaDefinition, + record: DataRecord, + index: number, + recordErrors: ReadonlyArray, +): DeepReadonly => { + const mutableRecord: MutableRecord = { ...record }; + schemaDef.fields.forEach((field) => { + // if there was an error for this field don't convert it. this means a string was passed instead of number or boolean + // this allows us to continue other validations without hiding possible errors down. + if ( + recordErrors.find( + (er) => er.errorType == SchemaValidationErrorTypes.INVALID_FIELD_VALUE_TYPE && er.fieldName == field.name, + ) + ) { + return undefined; + } + + /* + * if the field is missing from the records don't set it to undefined + */ + if (!_.has(record, field.name)) { + return; + } + + // need to check how it behaves for record[field.name] == "" + if (isEmpty(record[field.name])) { + mutableRecord[field.name] = undefined; + return; + } + + const valueType = field.valueType; + const rawValue = record[field.name]; + + if (field.isArray) { + const rawValues = convertToArray(rawValue); + mutableRecord[field.name] = rawValues.map( + (rv) => getTypedValue(field, valueType, rv) as any, // fix type here + ); + } else { + mutableRecord[field.name] = getTypedValue(field, valueType, rawValue as string); + } + }); + return F(mutableRecord); +}; + +const getTypedValue = (field: FieldDefinition, valueType: ValueType, rawValue: string) => { + let formattedFieldValue = rawValue; + // convert field to match corresponding enum from codelist, if possible + if (field.restrictions && field.restrictions.codeList && valueType === ValueType.STRING) { + const formattedField = field.restrictions.codeList.find( + (e) => e.toString().toLowerCase() === rawValue.toString().toLowerCase(), + ); + if (formattedField) { + formattedFieldValue = formattedField as string; + } + } + + let typedValue: SchemaTypes = rawValue; + switch (valueType) { + case ValueType.STRING: + typedValue = formattedFieldValue; + break; + case ValueType.INTEGER: + typedValue = Number(rawValue); + break; + case ValueType.NUMBER: + typedValue = Number(rawValue); + break; + case ValueType.BOOLEAN: + // we have to lower case in case of inconsistent letters (boolean requires all small letters). + typedValue = Boolean(rawValue.toLowerCase()); + break; + } + + return typedValue; +}; + +/** + * A "select" function that retrieves specific fields from the dataset as a record, as well as the numeric position of each row in the dataset. + * @param dataset Dataset to select fields from. + * @param fields Array with names of the fields to select. + * @returns A tuple array. In each tuple, the first element is the index of the row in the dataset, and the second value is the record with the + * selected values. + */ +const selectFieldsFromDataset = ( + dataset: SchemaData, + fields: string[], +): [number, Record][] => { + const records: [number, Record][] = []; + dataset.forEach((row, index) => { + const values: Record = {}; + fields.forEach((field) => { + values[field] = row[field] || ''; + }); + records.push([index, values]); + }); + return records; +}; + +/** + * Run schema validation pipeline for a schema defintion on the list of records provided. + * @param definition the schema definition name. + * @param record the records to validate. + */ +const validate = ( + schemaDef: SchemaDefinition, + record: DataRecord, + index: number, +): ReadonlyArray => { + const majorErrors = validation + .runValidationPipeline(record, index, schemaDef.fields, [ + validation.validateFieldNames, + validation.validateNonArrayFields, + validation.validateRequiredFields, + validation.validateValueTypes, + ]) + .filter(notEmpty); + return [...majorErrors]; +}; + +const validateAfterTypeConversion = ( + schemaDef: SchemaDefinition, + record: TypedDataRecord, + index: number, +): ReadonlyArray => { + const validationErrors = validation + .runValidationPipeline(record, index, schemaDef.fields, [ + validation.validateRegex, + validation.validateRange, + validation.validateEnum, + validation.validateScript, + ]) + .filter(notEmpty); + + return [...validationErrors]; +}; +export type ProcessingFunction = (schema: SchemaDefinition, rec: Readonly, index: number) => any; + +type MutableRecord = { [key: string]: SchemaTypes }; +type RawMutableRecord = { [key: string]: string | string[] }; + +namespace validation { + // these validation functions run AFTER the record has been converted to the correct types from raw strings + export type TypedValidationFunction = ( + rec: TypedDataRecord, + index: number, + fields: Array, + ) => Array; + + // these validation functions run BEFORE the record has been converted to the correct types from raw strings + export type ValidationFunction = ( + rec: DataRecord, + index: number, + fields: Array, + ) => Array; + + // these validation functions run AFTER the records has been converted to the correct types from raw strings, and apply to a dataset instead of + // individual records + export type TypedDatasetValidationFunction = ( + dataset: Array, + schemaDef: SchemaDefinition, + ) => Array; + + export type CrossSchemaValidationFunction = ( + schemaDef: SchemaDefinition, + schemasData: Record, + ) => Array; + + export const runValidationPipeline = ( + rec: DataRecord | TypedDataRecord, + index: number, + fields: ReadonlyArray, + funs: Array, + ) => { + let result: Array = []; + for (const fun of funs) { + if (rec instanceof DataRecord) { + const typedFunc = fun as ValidationFunction; + result = result.concat(typedFunc(rec as DataRecord, index, getValidFields(result, fields))); + } else { + const typedFunc = fun as TypedValidationFunction; + result = result.concat(typedFunc(rec as TypedDataRecord, index, getValidFields(result, fields))); + } + } + return result; + }; + + export const runDatasetValidationPipeline = ( + dataset: Array, + schemaDef: SchemaDefinition, + funs: Array, + ) => { + let result: Array = []; + for (const fun of funs) { + const typedFunc = fun as TypedDatasetValidationFunction; + result = result.concat(typedFunc(dataset, schemaDef)); + } + return result; + }; + + export const runCrossSchemaValidationPipeline = ( + schemaDef: SchemaDefinition, + schemasData: Record, + funs: Array, + ) => { + let result: Array = []; + for (const fun of funs) { + const typedFunc = fun as CrossSchemaValidationFunction; + result = result.concat(typedFunc(schemaDef, schemasData)); + } + return result; + }; + + export const validateRegex: TypedValidationFunction = ( + rec: TypedDataRecord, + index: number, + fields: ReadonlyArray, + ) => { + return fields + .map((field) => { + const recordFieldValues = convertToArray(rec[field.name]); + if (!isStringArray(recordFieldValues)) return undefined; + + const regex = field.restrictions?.regex; + if (isEmpty(regex)) return undefined; + + const invalidValues = recordFieldValues.filter((v) => isInvalidRegexValue(regex, v)); + if (invalidValues.length !== 0) { + const examples = field.meta?.examples; + const info = { value: invalidValues, regex, examples }; + return buildError(SchemaValidationErrorTypes.INVALID_BY_REGEX, field.name, index, info); + } + return undefined; + }) + .filter(notEmpty); + }; + + export const validateRange: TypedValidationFunction = ( + rec: TypedDataRecord, + index: number, + fields: ReadonlyArray, + ) => { + return fields + .map((field) => { + const recordFieldValues = convertToArray(rec[field.name]); + if (!isNumberArray(recordFieldValues)) return undefined; + + const range = field.restrictions?.range; + if (isEmpty(range)) return undefined; + + const invalidValues = recordFieldValues.filter((v) => isOutOfRange(range, v)); + if (invalidValues.length !== 0) { + const info = { value: invalidValues, ...range }; + return buildError(SchemaValidationErrorTypes.INVALID_BY_RANGE, field.name, index, info); + } + return undefined; + }) + .filter(notEmpty); + }; + + export const validateScript: TypedValidationFunction = ( + rec: TypedDataRecord, + index: number, + fields: Array, + ) => { + return fields + .map((field) => { + if (field.restrictions && field.restrictions.script) { + const scriptResult = validateWithScript(field, rec); + if (!scriptResult.valid) { + return buildError(SchemaValidationErrorTypes.INVALID_BY_SCRIPT, field.name, index, { + message: scriptResult.message, + value: rec[field.name], + }); + } + } + return undefined; + }) + .filter(notEmpty); + }; + + export const validateEnum: TypedValidationFunction = ( + rec: TypedDataRecord, + index: number, + fields: Array, + ) => { + return fields + .map((field) => { + const codeList = field.restrictions?.codeList || undefined; + if (isEmpty(codeList)) return undefined; + + const recordFieldValues = convertToArray(rec[field.name]); // put all values into array for easier validation + const invalidValues = recordFieldValues.filter((val) => isInvalidEnumValue(codeList, val)); + + if (invalidValues.length !== 0) { + const info = { value: invalidValues }; + return buildError(SchemaValidationErrorTypes.INVALID_ENUM_VALUE, field.name, index, info); + } + return undefined; + }) + .filter(notEmpty); + }; + + export const validateUnique: TypedDatasetValidationFunction = ( + dataset: Array, + schemaDef: SchemaDefinition, + ) => { + const errors: Array = []; + schemaDef.fields.forEach((field) => { + const unique = field.restrictions?.unique || undefined; + if (!unique) return undefined; + const keysToValidate = selectFieldsFromDataset(dataset as DataRecord[], [field.name]); + const duplicateKeys = findDuplicateKeys(keysToValidate); + + duplicateKeys.forEach(([index, record]) => { + const info = { value: record[field.name] }; + errors.push(buildError(SchemaValidationErrorTypes.INVALID_BY_UNIQUE, field.name, index, info)); + }); + }); + return errors; + }; + + export const validateUniqueKey: TypedDatasetValidationFunction = ( + dataset: Array, + schemaDef: SchemaDefinition, + ) => { + const errors: Array = []; + const uniqueKeyRestriction = schemaDef?.restrictions?.uniqueKey; + if (uniqueKeyRestriction) { + const uniqueKeyFields: string[] = uniqueKeyRestriction; + const keysToValidate = selectFieldsFromDataset(dataset as SchemaData, uniqueKeyFields); + const duplicateKeys = findDuplicateKeys(keysToValidate); + + duplicateKeys.forEach(([index, record]) => { + const info = { value: record, uniqueKeyFields: uniqueKeyFields }; + errors.push( + buildError(SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, uniqueKeyFields.join(', '), index, info), + ); + }); + } + return errors; + }; + + export const validateValueTypes: ValidationFunction = ( + rec: DataRecord, + index: number, + fields: Array, + ) => { + return fields + .map((field) => { + if (isEmpty(rec[field.name])) return undefined; + + const recordFieldValues = convertToArray(rec[field.name]); // put all values into array + const invalidValues = recordFieldValues.filter((v) => isInvalidFieldType(field.valueType, v)); + const info = { value: invalidValues }; + + if (invalidValues.length !== 0) { + return buildError(SchemaValidationErrorTypes.INVALID_FIELD_VALUE_TYPE, field.name, index, info); + } + return undefined; + }) + .filter(notEmpty); + }; + + export const validateRequiredFields = (rec: DataRecord, index: number, fields: Array) => { + return fields + .map((field) => { + if (isRequiredMissing(field, rec)) { + return buildError(SchemaValidationErrorTypes.MISSING_REQUIRED_FIELD, field.name, index); + } + return undefined; + }) + .filter(notEmpty); + }; + + export const validateFieldNames: ValidationFunction = ( + record: Readonly, + index: number, + fields: Array, + ) => { + const expectedFields = new Set(fields.map((field) => field.name)); + return Object.keys(record) + .map((recFieldName) => { + if (!expectedFields.has(recFieldName)) { + return buildError(SchemaValidationErrorTypes.UNRECOGNIZED_FIELD, recFieldName, index); + } + return undefined; + }) + .filter(notEmpty); + }; + + export const validateNonArrayFields: ValidationFunction = ( + record: Readonly, + index: number, + fields: Array, + ) => { + return fields + .map((field) => { + if (!field.isArray && isStringArray(record[field.name])) { + return buildError(SchemaValidationErrorTypes.INVALID_FIELD_VALUE_TYPE, field.name, index); + } + return undefined; + }) + .filter(notEmpty); + }; + + export const validateForeignKey: CrossSchemaValidationFunction = ( + schemaDef: SchemaDefinition, + schemasData: Record, + ) => { + const errors: Array = []; + const foreignKeyDefinitions = schemaDef?.restrictions?.foreignKey; + if (foreignKeyDefinitions) { + foreignKeyDefinitions.forEach((foreignKeyDefinition) => { + const localSchemaData = schemasData[schemaDef.name] || []; + const foreignSchemaData = schemasData[foreignKeyDefinition.schema] || []; + + // A foreign key can have more than one field, in which case is a composite foreign key. + const localFields = foreignKeyDefinition.mappings.map((x) => x.local); + const foreignFields = foreignKeyDefinition.mappings.map((x) => x.foreign); + + const fieldsMappings = new Map(foreignKeyDefinition.mappings.map((x) => [x.foreign, x.local])); + + // Select the keys of the datasets to compare. The keys are records to support the scenario where the fk is composite. + const localValues: [number, Record][] = selectFieldsFromDataset( + localSchemaData, + localFields, + ); + const foreignValues: [number, Record][] = selectFieldsFromDataset( + foreignSchemaData, + foreignFields, + ); + + // This artificial record in foreignValues allows null references in localValues to be valid. + const emptyRow: Record = {}; + foreignFields.forEach((field) => (emptyRow[field] = '')); + foreignValues.push([-1, emptyRow]); + + const missingForeignKeys = findMissingForeignKeys(localValues, foreignValues, fieldsMappings); + + missingForeignKeys.forEach((record) => { + const index = record[0]; + const info = { + value: record[1], + foreignSchema: foreignKeyDefinition.schema, + }; + + errors.push( + buildError(SchemaValidationErrorTypes.INVALID_BY_FOREIGN_KEY, localFields.join(', '), index, info), + ); + }); + }); + } + return errors; + }; + + export const getValidFields = ( + errs: ReadonlyArray, + fields: ReadonlyArray, + ) => { + return fields.filter((field) => { + return !errs.find((e) => e.fieldName == field.name); + }); + }; + + // return false if the record value is a valid type + export const isInvalidFieldType = (valueType: ValueType, value: string) => { + // optional field if the value is absent at this point + if (isAbsent(value) || isEmptyString(value)) return false; + switch (valueType) { + case ValueType.STRING: + return false; + case ValueType.INTEGER: + return isNaN(Number(value)) || !Number.isInteger(Number(value)); + case ValueType.NUMBER: + return isNaN(Number(value)); + case ValueType.BOOLEAN: + return !(value.toLowerCase() === 'true' || value.toLowerCase() === 'false'); + } + }; + + export const isRequiredMissing = (field: FieldDefinition, record: DataRecord) => { + const isRequired = field.restrictions && field.restrictions.required; + if (!isRequired) return false; + + const recordFieldValues = convertToArray(record[field.name]); + return recordFieldValues.every(isEmptyString); + }; + + const isOutOfRange = (range: RangeRestriction, value: number | undefined) => { + if (value == undefined) return false; + const invalidRange = + // less than the min if defined ? + (range.min !== undefined && value < range.min) || + (range.exclusiveMin !== undefined && value <= range.exclusiveMin) || + // bigger than max if defined ? + (range.max !== undefined && value > range.max) || + (range.exclusiveMax !== undefined && value >= range.exclusiveMax); + return invalidRange; + }; + + const isInvalidEnumValue = (codeList: CodeListRestriction, value: string | boolean | number | undefined) => { + // optional field if the value is absent at this point + if (isAbsent(value) || isEmptyString(value as string)) return false; + return !codeList.find((e) => e === value); + }; + + const isInvalidRegexValue = (regex: string, value: string) => { + // optional field if the value is absent at this point + if (isAbsent(value) || isEmptyString(value)) return false; + const regexPattern = new RegExp(regex); + return !regexPattern.test(value); + }; + + const ctx = vm.createContext(); + + const validateWithScript = ( + field: FieldDefinition, + record: TypedDataRecord, + ): { + valid: boolean; + message: string; + } => { + try { + const args = { + $row: record, + $field: record[field.name], + $name: field.name, + }; + + if (!field.restrictions || !field.restrictions.script) { + throw new Error('called validation by script without script provided'); + } + + // scripts should already be strings inside arrays, but ensure that they are to help transition between lectern versions + // checking for this can be removed in future versions of lectern (feb 2020) + const scripts = + typeof field.restrictions.script === 'string' ? [field.restrictions.script] : field.restrictions.script; + + let result: { + valid: boolean; + message: string; + } = { + valid: false, + message: '', + }; + + for (const scriptString of scripts) { + const script = getScript(scriptString); + const valFunc = script.runInContext(ctx); + if (!valFunc) throw new Error('Invalid script'); + result = valFunc(args); + /* Return the first script that's invalid. Otherwise result will be valid with message: 'ok'*/ + if (!result.valid) break; + } + + return result; + } catch (err) { + console.error( + `failed running validation script ${field.name} for record: ${JSON.stringify(record)}. Error message: ${err}`, + ); + return { + valid: false, + message: 'failed to run script validation, check script and the input', + }; + } + }; + + const getScript = (scriptString: string) => { + const script = new vm.Script(scriptString); + return script; + }; + + const buildError = ( + errorType: SchemaValidationErrorTypes, + fieldName: string, + index: number, + info: object = {}, + ): SchemaValidationError => { + const errorData = { errorType, fieldName, index, info }; + return { ...errorData, message: schemaErrorMessage(errorType, errorData) }; + }; +} +function validateRecordsSet(schemaDef: SchemaDefinition, processedRecords: TypedDataRecord[]) { + const validationErrors = validation + .runDatasetValidationPipeline(processedRecords, schemaDef, [ + validation.validateUnique, + validation.validateUniqueKey, + ]) + .filter(notEmpty); + return validationErrors; +} diff --git a/packages/client/src/schema-rest-client.ts b/packages/client/src/schema-rest-client.ts new file mode 100644 index 0000000..a9a944c --- /dev/null +++ b/packages/client/src/schema-rest-client.ts @@ -0,0 +1,154 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { loggerFor } from './logger'; +import fetch from 'node-fetch'; +import { SchemasDictionary, SchemasDictionaryDiffs, FieldChanges, FieldDiff } from './schema-entities'; +import promiseTools from 'promise-tools'; +import { unknownToString } from 'common'; +const L = loggerFor(__filename); + +export interface SchemaServiceRestClient { + fetchSchema(schemaSvcUrl: string, name: string, version: string): Promise; + fetchDiff( + schemaSvcUrl: string, + name: string, + fromVersion: string, + toVersion: string, + ): Promise; +} + +export const restClient: SchemaServiceRestClient = { + fetchSchema: async (schemaSvcUrl: string, name: string, version: string): Promise => { + // for testing where we need to work against stub schema + if (schemaSvcUrl.startsWith('file://')) { + return await loadSchemaFromFile(version, schemaSvcUrl, name); + } + + if (!schemaSvcUrl) { + throw new Error('please configure a valid url to get schema from'); + } + const url = `${schemaSvcUrl}/dictionaries?name=${name}&version=${version}`; + try { + L.debug(`in fetch live schema ${version}`); + const schemaDictionary = await doRequest(url); + // todo validate response and map it to a schema + return schemaDictionary[0] as SchemasDictionary; + } catch (error: unknown) { + L.error(`failed to fetch schema at url: ${url} - ${unknownToString(error)}`); + throw error; + } + }, + fetchDiff: async ( + schemaSvcBaseUrl: string, + name: string, + fromVersion: string, + toVersion: string, + ): Promise => { + // for testing where we need to work against stub schema + let diffResponse: any; + if (schemaSvcBaseUrl.startsWith('file://')) { + diffResponse = await loadDiffFromFile(schemaSvcBaseUrl, name, fromVersion, toVersion); + } else { + const url = `${schemaSvcBaseUrl}/diff?name=${name}&left=${fromVersion}&right=${toVersion}`; + diffResponse = (await doRequest(url)) as any[]; + } + const result: SchemasDictionaryDiffs = {}; + for (const entry of diffResponse) { + const fieldName = entry[0] as string; + if (entry[1]) { + const fieldDiff: FieldDiff = { + before: entry[1].left, + after: entry[1].right, + diff: entry[1].diff, + }; + result[fieldName] = fieldDiff; + } + } + return result; + }, +}; + +const doRequest = async (url: string) => { + let response: any; + try { + const retryAttempt = 1; + response = await promiseTools.retry({ times: 5, interval: 1000 }, async () => { + L.debug(`fetching schema attempt #${retryAttempt}`); + return promiseTools.timeout(fetch(url), 5000); + }); + return await response.json(); + } catch (error: unknown) { + L.error(`failed to fetch schema at url: ${url} - ${unknownToString(error)}`); + throw response.status == 404 ? new Error('Not Found') : new Error('Request Failed'); + } +}; + +async function loadSchemaFromFile(version: string, schemaSvcUrl: string, name: string) { + L.debug(`in fetch stub schema ${version}`); + const result = delay(1000); + const dictionary = await result(() => { + const dictionaries: SchemasDictionary[] = require(schemaSvcUrl.substring(7, schemaSvcUrl.length)) + .dictionaries as SchemasDictionary[]; + if (!dictionaries) { + throw new Error('your mock json is not structured correctly, see sampleFiles/sample-schema.json'); + } + const dic = dictionaries.find((d: any) => d.version == version && d.name == name); + if (!dic) { + return undefined; + } + return dic; + }); + if (dictionary == undefined) { + throw new Error("couldn't load stub dictionary with the criteria specified"); + } + L.debug(`schema found ${dictionary.version}`); + return dictionary; +} + +async function loadDiffFromFile(schemaSvcBaseUrl: string, name: string, fromVersion: string, toVersion: string) { + L.debug(`in fetch stub diffs ${name} ${fromVersion} ${toVersion}`); + const result = delay(1000); + const diff = await result(() => { + const diffResponse = require(schemaSvcBaseUrl.substring(7, schemaSvcBaseUrl.length)).diffs as any[]; + if (!diffResponse) { + throw new Error('your mock json is not structured correctly, see sampleFiles/sample-schema.json'); + } + + const diff = diffResponse.find( + (d: any) => d.fromVersion == fromVersion && d.toVersion == toVersion && d.name == name, + ); + if (!diff) { + return undefined; + } + return diff; + }); + if (diff == undefined) { + throw new Error("couldn't load stub diff with the criteria specified, check your stub file"); + } + return diff.data; +} + +function delay(milliseconds: number) { + return async (result: () => T | undefined) => { + return new Promise((resolve, reject) => { + setTimeout(() => resolve(result()), milliseconds); + }); + }; +} diff --git a/packages/client/src/schema-worker.js b/packages/client/src/schema-worker.js new file mode 100644 index 0000000..cdb44c0 --- /dev/null +++ b/packages/client/src/schema-worker.js @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2020 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +const workerThreads = require('worker_threads'); +const parentPort = workerThreads.parentPort; +// ts node is required to import ts modules like "schema-functions.ts" in this case since +// worker threads run in their own V8 instance +const tsNode = require('ts-node'); + +/** + * when we run the app with node directly like this: + * node -r ts-node/register server.ts + * we will have a registered ts node instance, registering another one will result in wierd behaviour + * however when we run with mocha: + * mocha -r ts-node/register .ts + * (same applies if we run with ts node directly: ts-node server.ts) + * the worker thread won't have an instance of ts node transpiler + * unlike node for some reason which seem to attach the isntance to the worker thread process. + * + * so we had to add this work around to avoid double registry in different run modes. + * root cause of why mocha acts different than node is not found yet. + */ +if (!process[tsNode.REGISTER_INSTANCE]) { + tsNode.register(); +} +const service = require('./schema-functions'); + +function processProxy(args) { + return service.process(args.dictionary, args.schemaName, args.record, args.index); +} + +parentPort.on('message', (args) => { + const result = processProxy(args); + parentPort.postMessage(result); +}); diff --git a/packages/client/src/utils.ts b/packages/client/src/utils.ts new file mode 100644 index 0000000..391b266 --- /dev/null +++ b/packages/client/src/utils.ts @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import fs from 'fs'; +import deepFreeze from 'deep-freeze'; +import _ from 'lodash'; +import { isArray } from 'util'; + +const fsPromises = fs.promises; + +export namespace Checks { + export const checkNotNull = (argName: string, arg: any) => { + if (!arg) { + throw new Errors.InvalidArgument(argName); + } + }; +} + +export namespace Errors { + export class InvalidArgument extends Error { + constructor(argumentName: string) { + super(`Invalid argument : ${argumentName}`); + } + } + + export class NotFound extends Error { + constructor(msg: string) { + super(msg); + } + } + + export class StateConflict extends Error { + constructor(msg: string) { + super(msg); + } + } +} + +// type gaurd to filter out undefined and null +// https://stackoverflow.com/questions/43118692/typescript-filter-out-nulls-from-an-array +export function notEmpty(value: TValue | null | undefined): value is TValue { + // lodash 4.14 behavior note, these are all evaluated to true: + // _.isEmpty(null) _.isEmpty(undefined) _.isEmpty([]) + // _.isEmpty({}) _.isEmpty('') _.isEmpty(12) & _.isEmpty(NaN) + + // so check number seperately since it will evaluate to isEmpty=true + return (isNumber(value) && !isNaN(value)) || !_.isEmpty(value); +} + +export function isEmpty(value: TValue | null | undefined): value is undefined { + return !notEmpty(value); +} + +export const convertToArray = (val: T | T[]): T[] => { + if (Array.isArray(val)) { + return val; + } else { + return [val]; + } +}; + +export function isString(value: any): value is string { + return typeof value === 'string' || value instanceof String; +} + +export function isStringArray(value: any | undefined | null): value is string[] { + return Array.isArray(value) && value.every(isString); +} + +export function isNumber(value: any): value is number { + return typeof value === 'number'; +} + +export function isNumberArray(values: any): values is number[] { + return Array.isArray(values) && values.every(isNumber); +} + +// returns true if value matches at least one of the expressions +export const isStringMatchRegex = (expressions: RegExp[], value: string) => { + return expressions.filter((exp) => RegExp(exp).test(value)).length >= 1; +}; + +export const isNotEmptyString = (value: string | undefined): value is string => { + return isNotAbsent(value) && value.trim() !== ''; +}; + +export const isEmptyString = (value: string) => { + return !isNotEmptyString(value); +}; + +export const isAbsent = (value: string | number | boolean | undefined): value is undefined => { + return !isNotAbsent(value); +}; + +export const isNotAbsent = (value: string | number | boolean | undefined): value is string | number | boolean => { + return value !== null && value !== undefined; +}; + +export const sleep = async (milliSeconds: number = 2000) => { + return new Promise((resolve) => setTimeout(resolve, milliSeconds)); +}; + +export function toString(obj: any) { + if (!obj) { + return undefined; + } + Object.keys(obj).forEach((k) => { + if (typeof obj[k] === 'object') { + return toString(obj[k]); + } + obj[k] = `${obj[k]}`; + }); + + return obj; +} + +export function isValueEqual(value: any, other: any) { + if (isArray(value) && isArray(other)) { + return _.difference(value, other).length === 0; // check equal, ignore order + } + + return _.isEqual(value, other); +} + +export function isValueNotEqual(value: any, other: any) { + return !isValueEqual(value, other); +} + +export function convertToTrimmedString(val: unknown | undefined | string | number | boolean | null) { + return val === undefined || val === null ? '' : String(val).trim(); +} + +export const F = deepFreeze; diff --git a/packages/client/test/change-analyzer.spec.ts b/packages/client/test/change-analyzer.spec.ts new file mode 100644 index 0000000..b0fb319 --- /dev/null +++ b/packages/client/test/change-analyzer.spec.ts @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import chai from 'chai'; +import * as analyzer from '../src/change-analyzer'; +import { SchemasDictionaryDiffs, FieldDiff, ChangeAnalysis } from '../src/schema-entities'; +import _ from 'lodash'; +chai.should(); +const diffResponse: any = require('./schema-diff.json'); +const schemaDiff: SchemasDictionaryDiffs = {}; +for (const entry of diffResponse) { + const fieldName = entry[0] as string; + if (entry[1]) { + const fieldDiff: FieldDiff = { + before: entry[1].left, + after: entry[1].right, + diff: entry[1].diff, + }; + schemaDiff[fieldName] = fieldDiff; + } +} + +const expectedResult: ChangeAnalysis = { + fields: { + addedFields: [], + renamedFields: [], + deletedFields: ['primary_diagnosis.menopause_status'], + }, + metaChanges: { + core: { + changedToCore: [], + changedFromCore: [], + }, + }, + restrictionsChanges: { + codeList: { + created: [], + deleted: [ + { + field: 'donor.vital_status', + definition: ['Alive', 'Deceased', 'Not reported', 'Unknown'], + }, + ], + updated: [ + { + field: 'donor.cause_of_death', + definition: { + added: ['N/A'], + deleted: ['Died of cancer', 'Unknown'], + }, + }, + ], + }, + regex: { + updated: [ + { + field: 'donor.submitter_donor_id', + definition: '[A-Za-z0-9\\-\\._]{3,64}', + }, + { + field: 'primary_diagnosis.cancer_type_code', + definition: '[A-Z]{1}[0-9]{2}.[0-9]{0,3}[A-Z]{2,3}$', + }, + ], + created: [ + { + field: 'donor.vital_status', + definition: '[A-Z]{3,100}', + }, + ], + deleted: [], + }, + required: { + updated: [], + created: [], + deleted: [], + }, + script: { + updated: [], + created: [ + { + field: 'donor.survival_time', + definition: ' $field / 2 == 0 ', + }, + ], + deleted: [], + }, + range: { + updated: [ + { + definition: { + max: 1, + }, + field: 'specimen.percent_stromal_cells', + }, + ], + created: [ + { + field: 'donor.survival_time', + definition: { + min: 0, + max: 200000, + }, + }, + ], + deleted: [], + }, + }, + isArrayDesignationChanges: ['primary_diagnosis.presenting_symptoms'], + valueTypeChanges: ['sample_registration.program_id'], +}; + +describe('change-analyzer', () => { + it('categorize changes correctly', () => { + const result = analyzer.analyzeChanges(schemaDiff); + result.should.deep.eq(expectedResult); + }); +}); diff --git a/packages/client/test/schema-diff.json b/packages/client/test/schema-diff.json new file mode 100644 index 0000000..e4f7a4d --- /dev/null +++ b/packages/client/test/schema-diff.json @@ -0,0 +1,312 @@ +[ + [ + "donor.submitter_donor_id", + { + "left": { + "description": "Unique identifier of the donor, assigned by the data provider.", + "name": "submitter_donor_id", + "restrictions": { + "required": true, + "regex": "[A-Za-z0-9\\-\\._]{1,64}" + }, + "valueType": "string" + }, + "right": { + "description": "Unique identifier of the donor, assigned by the data provider.", + "name": "submitter_donor_id", + "restrictions": { + "required": true, + "regex": "[A-Za-z0-9\\-\\._]{3,64}" + }, + "valueType": "string" + }, + "diff": { + "restrictions": { + "regex": { + "type": "updated", + "data": "[A-Za-z0-9\\-\\._]{3,64}" + } + } + } + } + ], + [ + "donor.vital_status", + { + "left": { + "description": "Donors last known state of living or deceased.", + "name": "vital_status", + "restrictions": { + "codeList": ["Alive", "Deceased", "Not reported", "Unknown"], + "required": true + }, + "valueType": "string" + }, + "right": { + "description": "Donors last known state of living or deceased.", + "name": "vital_status", + "restrictions": { + "regex": "[A-Z]{3,100}", + "required": true + }, + "valueType": "string" + }, + "diff": { + "restrictions": { + "codeList": { + "type": "deleted", + "data": ["Alive", "Deceased", "Not reported", "Unknown"] + }, + "regex": { + "type": "created", + "data": "[A-Z]{3,100}" + } + } + } + } + ], + [ + "donor.cause_of_death", + { + "left": { + "description": "Description of the cause of a donor's death.", + "name": "cause_of_death", + "restrictions": { + "codeList": ["Died of cancer", "Died of other reasons", "Not reported", "Unknown"] + }, + "valueType": "string" + }, + "right": { + "description": "Description of the cause of a donor's death.", + "name": "cause_of_death", + "restrictions": { + "codeList": ["Died of other reasons", "Not reported", "N/A"] + }, + "valueType": "string" + }, + "diff": { + "restrictions": { + "codeList": { + "type": "updated", + "data": { + "added": ["N/A"], + "deleted": ["Died of cancer", "Unknown"] + } + } + } + } + } + ], + [ + "donor.survival_time", + { + "left": { + "description": "Interval of how long the donor has survived since primary diagnosis, in days.", + "meta": { + "units": "days" + }, + "name": "survival_time", + "valueType": "integer" + }, + "right": { + "description": "Interval of how long the donor has survived since primary diagnosis, in days.", + "meta": { + "units": "days" + }, + "name": "survival_time", + "valueType": "integer", + "restrictions": { + "script": " $field / 2 == 0 ", + "range": { + "min": 0, + "max": 200000 + } + } + }, + "diff": { + "restrictions": { + "type": "created", + "data": { + "range": { + "min": 0, + "max": 200000 + }, + "script": " $field / 2 == 0 " + } + } + } + } + ], + [ + "primary_diagnosis.cancer_type_code", + { + "left": { + "name": "cancer_type_code", + "valueType": "string", + "description": "The code to represent the cancer type using the WHO ICD-10 code (https://icd.who.int/browse10/2016/en#/) classification.", + "restrictions": { + "required": true, + "regex": "[A-Z]{1}[0-9]{2}.[0-9]{0,3}[A-Z]{0,1}$" + } + }, + "right": { + "name": "cancer_type_code", + "valueType": "string", + "description": "The code to represent the cancer type using the WHO ICD-10 code (https://icd.who.int/browse10/2016/en#/) classification.", + "restrictions": { + "required": true, + "regex": "[A-Z]{1}[0-9]{2}.[0-9]{0,3}[A-Z]{2,3}$" + } + }, + "diff": { + "restrictions": { + "regex": { + "type": "updated", + "data": "[A-Z]{1}[0-9]{2}.[0-9]{0,3}[A-Z]{2,3}$" + } + } + } + } + ], + [ + "primary_diagnosis.menopause_status", + { + "left": { + "name": "menopause_status", + "description": "Indicate the menopause status of the patient at the time of primary diagnosis.", + "valueType": "string", + "restrictions": { + "codeList": ["Perimenopausal", "Postmenopausal", "Premenopausal", "Unknown"] + } + }, + "diff": { + "type": "deleted", + "data": { + "name": "menopause_status", + "description": "Indicate the menopause status of the patient at the time of primary diagnosis.", + "valueType": "string", + "restrictions": { + "codeList": ["Perimenopausal", "Postmenopausal", "Premenopausal", "Unknown"] + } + } + } + } + ], + [ + "primary_diagnosis.presenting_symptoms", + { + "left": { + "name": "presenting_symptoms", + "description": "Indicate presenting symptoms at time of primary diagnosis.", + "valueType": "string", + "restrictions": { + "codeList": ["Abdominal Pain", "Anemia", "Diabetes", "Diarrhea", "Nausea", "None"] + } + }, + "right": { + "name": "presenting_symptoms", + "description": "Indicate presenting symptoms at time of primary diagnosis.", + "valueType": "string", + "isArray": true, + "restrictions": { + "codeList": ["Abdominal Pain", "Anemia", "Diabetes", "Diarrhea", "Nausea", "None"] + } + }, + "diff": { + "isArray": { + "type": "created", + "data": true + } + } + } + ], + [ + "sample_registration.program_id", + { + "left": { + "name": "program_id", + "valueType": "string", + "description": "Unique identifier of the ARGO program.", + "meta": { + "validationDependency": true, + "primaryId": true, + "examples": "PACA-AU,BR-CA", + "displayName": "Program ID" + }, + "restrictions": { + "required": true + } + }, + "right": { + "name": "program_id", + "valueType": "integer", + "description": "Unique identifier of the ARGO program.", + "meta": { + "validationDependency": true, + "primaryId": true, + "examples": "PACA-AU,BR-CA", + "displayName": "Program ID" + }, + "restrictions": { + "required": true + } + }, + "diff": { + "valueType": { + "type": "updated", + "data": "integer" + } + } + } + ], + [ + "specimen.percent_stromal_cells", + { + "left": { + "name": "percent_stromal_cells", + "description": "Indicate a value, in decimals, that represents the percentage of reactive cells that are present in a malignant tumour specimen but are not malignant such as fibroblasts, vascular structures, etc.", + "valueType": "number", + "meta": { + "dependsOn": "sample_registration.tumour_normal_designation", + "notes": "", + "displayName": "Percent Stromal Cells" + }, + "restrictions": { + "range": { + "min": 0, + "max": 0.1 + } + } + }, + "right": { + "name": "percent_stromal_cells", + "description": "Indicate a value, in decimals, that represents the percentage of reactive cells that are present in a malignant tumour specimen but are not malignant such as fibroblasts, vascular structures, etc.", + "valueType": "number", + "meta": { + "dependsOn": "sample_registration.tumour_normal_designation", + "notes": "", + "displayName": "Percent Stromal Cells" + }, + "restrictions": { + "range": { + "max": 1 + } + } + }, + "diff": { + "restrictions": { + "range": { + "min": { + "type": "deleted", + "data": 0 + }, + "max": { + "type": "updated", + "data": 1 + } + } + } + } + } + ] +] diff --git a/packages/client/test/schema-functions.spec.ts b/packages/client/test/schema-functions.spec.ts new file mode 100644 index 0000000..315fa81 --- /dev/null +++ b/packages/client/test/schema-functions.spec.ts @@ -0,0 +1,9295 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import chai from 'chai'; +import * as schemaService from '../src/schema-functions'; +import { SchemasDictionary, SchemaValidationErrorTypes } from '../src/schema-entities'; +import schemaErrorMessage from '../src/schema-error-messages'; +import { loggerFor } from '../src/logger'; +const L = loggerFor(__filename); + +chai.should(); +const schema: SchemasDictionary = require('./schema.json')[0]; + +const VALUE_NOT_ALLOWED = 'The value is not permissible for this field.'; +const PROGRAM_ID_REQ = 'program_id is a required field.'; + +describe('schema-functions', () => { + it('should populate records based on default value ', () => { + const result = schemaService.processRecords(schema, 'registration', [ + { + program_id: 'PEME-CA', + submitter_donor_id: 'OD1234', + gender: '', + submitter_specimen_id: '87813', + specimen_type: 'Skin', + tumour_normal_designation: 'Normal', + submitter_sample_id: 'MAS123', + sample_type: 'ctDNA', + }, + { + program_id: 'PEME-CA', + submitter_donor_id: 'OD1234', + gender: '', + submitter_specimen_id: '87812', + specimen_type: 'Skin', + tumour_normal_designation: 'Normal', + submitter_sample_id: 'MAS1234', + sample_type: 'ctDNA', + }, + ]); + chai.expect(result.processedRecords[0].gender).to.eq('Other'); + chai.expect(result.processedRecords[1].gender).to.eq('Other'); + }); + + it('should NOT populate missing columns based on default value ', () => { + const result = schemaService.processRecords(schema, 'registration', [ + { + program_id: 'PEME-CA', + submitter_donor_id: 'OD1234', + gendr: '', + submitter_specimen_id: '87813', + specimen_type: 'Skin', + tumour_normal_designation: 'Normal', + submitter_sample_id: 'MAS123', + sample_type: 'ctDNA', + }, + { + program_id: 'PEME-CA', + submitter_donor_id: 'OD1234', + gender: '', + submitter_specimen_id: '87812', + specimen_type: 'Skin', + tumour_normal_designation: 'Normal', + submitter_sample_id: 'MAS1234', + sample_type: 'ctDNA', + }, + ]); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.MISSING_REQUIRED_FIELD, + fieldName: 'gender', + index: 0, + info: {}, + message: 'gender is a required field.', + }); + }); + + it('should validate required', () => { + const result = schemaService.processRecords(schema, 'registration', [ + { + submitter_donor_id: 'OD1234', + gender: 'Female', + submitter_specimen_id: '87813', + specimen_type: 'Skin', + tumour_normal_designation: 'Normal', + submitter_sample_id: 'MAS123', + sample_type: 'ctDNA', + }, + ]); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.MISSING_REQUIRED_FIELD, + fieldName: 'program_id', + index: 0, + info: {}, + message: PROGRAM_ID_REQ, + }); + }); + + it('should validate value types', () => { + const result = schemaService.processRecords(schema, 'address', [ + { + country: 'US', + unit_number: 'abc', + postal_code: '12345', + }, + ]); + + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_FIELD_VALUE_TYPE, + fieldName: 'unit_number', + index: 0, + info: { value: ['abc'] }, + message: VALUE_NOT_ALLOWED, + }); + }); + + it('should convert string to integer after processing', () => { + const result = schemaService.processRecords(schema, 'address', [ + { + country: 'US', + unit_number: '123', + postal_code: '12345', + }, + ]); + chai.expect(result.processedRecords).to.deep.include({ + country: 'US', + unit_number: 123, + postal_code: '12345', + }); + }); + + it('should validate regex', () => { + const result = schemaService.processRecords(schema, 'registration', [ + { + program_id: 'PEME-CAA', + submitter_donor_id: 'OD1234', + gender: 'Female', + submitter_specimen_id: '87813', + specimen_type: 'Skin', + tumour_normal_designation: 'Normal', + submitter_sample_id: 'MAS123', + sample_type: 'ctDNA', + }, + ]); + chai.expect(result.validationErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_REGEX, + fieldName: 'program_id', + index: 0, + info: { + examples: 'PACA-CA, BASHAR-LA', + regex: '^[A-Z1-9][-_A-Z1-9]{2,7}(-[A-Z][A-Z])$', + value: ['PEME-CAA'], + }, + message: + 'The value is not a permissible for this field, it must meet the regular expression: "^[A-Z1-9][-_A-Z1-9]{2,7}(-[A-Z][A-Z])$". Examples: PACA-CA, BASHAR-LA', + }); + }); + + it('should validate range', () => { + const result = schemaService.processRecords(schema, 'address', [ + { + country: 'US', + postal_code: '12345', + unit_number: '-1', + }, + { + country: 'US', + postal_code: '12345', + unit_number: '223', + }, + { + country: 'US', + postal_code: '12345', + unit_number: '500000', + }, + ]); + + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_BY_RANGE, + fieldName: 'unit_number', + index: 0, + info: { + exclusiveMax: 999, + min: 0, + value: [-1], + }, + message: schemaErrorMessage(SchemaValidationErrorTypes.INVALID_BY_RANGE, { + info: { + exclusiveMax: 999, + min: 0, + }, + }), + }); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_BY_RANGE, + fieldName: 'unit_number', + index: 2, + info: { + exclusiveMax: 999, + min: 0, + value: [500000], + }, + message: schemaErrorMessage(SchemaValidationErrorTypes.INVALID_BY_RANGE, { + info: { + exclusiveMax: 999, + min: 0, + }, + }), + }); + }); + + it('should validate script', () => { + const result = schemaService.processRecords(schema, 'address', [ + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + ]); + + chai.expect(result.validationErrors.length).to.eq(2); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_BY_SCRIPT, + fieldName: 'postal_code', + index: 0, + info: { message: 'invalid postal code for US', value: '12' }, + message: 'invalid postal code for US', + }); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_BY_SCRIPT, + fieldName: 'postal_code', + index: 1, + info: { message: 'invalid postal code for CANADA', value: 'ABC' }, + message: 'invalid postal code for CANADA', + }); + }); + + it('should validate if non-required feilds are not provided', () => { + const result = schemaService.processRecords(schema, 'donor', [ + // optional enum field not provided + { + program_id: 'PACA-AU', + submitter_donor_id: 'ICGC_0004', + gender: 'Female', + ethnicity: 'black or african american', + vital_status: 'alive', + }, + // optional enum field provided with proper value + { + program_id: 'PACA-AU', + submitter_donor_id: 'ICGC_0002', + gender: 'Male', + ethnicity: 'asian', + vital_status: 'deceased', + cause_of_death: 'died of cancer', + survival_time: '124', + }, + // optional enum field provided with no value + { + program_id: 'PACA-AU', + submitter_donor_id: 'ICGC_0002', + gender: 'Male', + ethnicity: 'asian', + vital_status: 'deceased', + cause_of_death: '', + survival_time: '124', + }, + ]); + chai.expect(result.validationErrors.length).to.eq(0); + }); + + it('should error if integer fields are not valid', () => { + const result = schemaService.processRecords(schema, 'donor', [ + { + program_id: 'PACA-AU', + submitter_donor_id: 'ICGC_0002', + gender: 'Other', + ethnicity: 'asian', + vital_status: 'deceased', + cause_of_death: 'died of cancer', + survival_time: '0.5', + }, + ]); + + chai.expect(result.validationErrors.length).to.eq(1); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_FIELD_VALUE_TYPE, + fieldName: 'survival_time', + index: 0, + info: { value: ['0.5'] }, + message: VALUE_NOT_ALLOWED, + }); + }); + + it('should validate case insensitive enums, return proper format', () => { + const result = schemaService.processRecords(schema, 'registration', [ + { + program_id: 'PACA-AU', + submitter_donor_id: 'OD1234', + gender: 'feMale', + submitter_specimen_id: '87813', + specimen_type: 'sKiN', + tumour_normal_designation: 'Normal', + submitter_sample_id: 'MAS123', + sample_type: 'CTdna', + }, + ]); + chai.expect(result.validationErrors.length).to.eq(0); + chai.expect(result.processedRecords[0]).to.deep.eq({ + program_id: 'PACA-AU', + submitter_donor_id: 'OD1234', + gender: 'Female', + submitter_specimen_id: '87813', + specimen_type: 'Skin', + tumour_normal_designation: 'Normal', + submitter_sample_id: 'MAS123', + sample_type: 'ctDNA', + }); + }); + + it('should not validate if unrecognized fields are provided', () => { + const result = schemaService.processRecords(schema, 'donor', [ + { + program_id: 'PACA-AU', + submitter_donor_id: 'ICGC_0002', + gender: 'Other', + ethnicity: 'asian', + vital_status: 'deceased', + cause_of_death: 'died of cancer', + survival_time: '5', + hackField: 'muchHack', + }, + ]); + chai.expect(result.validationErrors.length).to.eq(1); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.UNRECOGNIZED_FIELD, + message: SchemaValidationErrorTypes.UNRECOGNIZED_FIELD, + fieldName: 'hackField', + index: 0, + info: {}, + }); + }); + + it('should validate number/integer array with field defined ranges', () => { + const result = schemaService.processRecords(schema, 'favorite_things', [ + { + id: 'TH-ING', + fraction: ['0.2', '2', '3'], + integers: ['-100', '-2'], + }, + ]); + + chai.expect(result.validationErrors.length).to.eq(2); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_BY_RANGE, + message: 'Value is out of permissible range, value must be > 0 and <= 1.', + index: 0, + fieldName: 'fraction', + info: { value: [2, 3], max: 1, exclusiveMin: 0 }, + }); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_BY_RANGE, + message: 'Value is out of permissible range, value must be >= -10 and <= 10.', + index: 0, + fieldName: 'integers', + info: { value: [-100], max: 10, min: -10 }, + }); + }); + + it('should validate string array with field defined codelist', () => { + const result = schemaService.processRecords(schema, 'favorite_things', [ + { + id: 'TH-ING', + fruit: ['Mango', '2'], + }, + ]); + chai.expect(result.validationErrors.length).to.eq(1); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_ENUM_VALUE, + message: 'The value is not permissible for this field.', + fieldName: 'fruit', + index: 0, + info: { value: ['2'] }, + }); + }); + + it('should validate string with field defined codelist', () => { + const result = schemaService.processRecords(schema, 'favorite_things', [ + { + id: 'TH-ING', + fruit_single_value: 'Banana', + }, + ]); + chai.expect(result.validationErrors.length).to.eq(1); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_ENUM_VALUE, + message: 'The value is not permissible for this field.', + fieldName: 'fruit_single_value', + index: 0, + info: { value: ['Banana'] }, + }); + }); + + it('should validate string array with field defined regex', () => { + const result = schemaService.processRecords(schema, 'favorite_things', [ + { + id: 'TH-ING', + qWords: ['que', 'not_q'], + }, + ]); + chai.expect(result.validationErrors.length).to.eq(1); + chai.expect(result.validationErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_REGEX, + message: 'The value is not a permissible for this field, it must meet the regular expression: "^q.*$".', + fieldName: 'qWords', + index: 0, + info: { value: ['not_q'], regex: '^q.*$', examples: undefined }, + }); + }); + + it('should validate string with field defined regex', () => { + const result = schemaService.processRecords(schema, 'favorite_things', [ + { + id: 'TH-ING', + qWord: 'not_q', + }, + ]); + chai.expect(result.validationErrors.length).to.eq(1); + chai.expect(result.validationErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_REGEX, + message: 'The value is not a permissible for this field, it must meet the regular expression: "^q.*$".', + fieldName: 'qWord', + index: 0, + info: { value: ['not_q'], regex: '^q.*$', examples: undefined }, + }); + }); + + it('should pass unique restriction validation when only null values exists', () => { + const result = schemaService.processRecords(schema, 'favorite_things', [ + { + id: 'TH-ING', + unique_value: '', + }, + ]); + chai.expect(result.validationErrors.length).to.eq(0); + }); + + it('should pass unique restriction validation when only one record exists', () => { + const result = schemaService.processRecords(schema, 'favorite_things', [ + { + id: 'TH-ING', + unique_value: 'unique_value_1', + }, + ]); + chai.expect(result.validationErrors.length).to.eq(0); + }); + + it('should fail unique restriction validation when duplicate values exist (scalar)', () => { + const result = schemaService.processRecords(schema, 'favorite_things', [ + { + id: 'ID-1', + unique_value: 'unique_value_1', + }, + { + id: 'ID-2', + unique_value: 'unique_value_1', + }, + ]); + + chai.expect(result.validationErrors.length).to.eq(2); + chai.expect(result.validationErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE, + message: 'Value for unique_value must be unique.', + fieldName: 'unique_value', + index: 0, + info: { value: ['unique_value_1'] }, + }); + chai.expect(result.validationErrors[1]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE, + message: 'Value for unique_value must be unique.', + fieldName: 'unique_value', + index: 1, + info: { value: ['unique_value_1'] }, + }); + }); + it('should fail unique restriction validation when duplicate values exist (array)', () => { + const result = schemaService.processRecords(schema, 'favorite_things', [ + { + id: 'ID-1', + unique_value: ['unique_value_1', 'unique_value_2'], + }, + { + id: 'ID-2', + unique_value: ['unique_value_1', 'unique_value_2'], + }, + ]); + + chai.expect(result.validationErrors.length).to.eq(2); + chai.expect(result.validationErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE, + message: 'Value for unique_value must be unique.', + fieldName: 'unique_value', + index: 0, + info: { value: ['unique_value_1', 'unique_value_2'] }, + }); + chai.expect(result.validationErrors[1]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE, + message: 'Value for unique_value must be unique.', + fieldName: 'unique_value', + index: 1, + info: { value: ['unique_value_1', 'unique_value_2'] }, + }); + }); + + it('should pass foreignKey restriction validation when values exist in foreign schema', () => { + const parent_schema_1_data = [ + { + id: 'parent_schema_1_id_1', + name: 'parent_schema_1_name_1', + }, + { + id: 'parent_schema_1_id_2', + name: 'parent_schema_1_name_2', + }, + ]; + + const child_schema_simple_fk_data = [ + { + id: '1', + parent_schema_1_id: 'parent_schema_1_id_1', + }, + { + id: '2', + parent_schema_1_id: 'parent_schema_1_id_2', + }, + ]; + const schemaData = { + parent_schema_1: parent_schema_1_data, + child_schema_simple_fk: child_schema_simple_fk_data, + }; + + const result = schemaService.processSchemas(schema, schemaData); + + chai.expect(result['parent_schema_1'].validationErrors.length).to.eq(0); + chai.expect(result['child_schema_simple_fk'].validationErrors.length).to.eq(0); + }); + + it('should pass foreignKey restriction validation when local schema has null values', () => { + const parent_schema_1_data = [ + { + id: 'parent_schema_1_id_1', + name: 'parent_schema_1_name_1', + }, + { + id: 'parent_schema_1_id_2', + name: 'parent_schema_1_name_2', + }, + ]; + + const child_schema_simple_fk_data = [ + { + id: '1', + parent_schema_1_id: 'parent_schema_1_id_1', + }, + { + id: '2', + parent_schema_1_id: '', + }, + ]; + const schemaData = { + parent_schema_1: parent_schema_1_data, + child_schema_simple_fk: child_schema_simple_fk_data, + }; + + const result = schemaService.processSchemas(schema, schemaData); + + chai.expect(result['parent_schema_1'].validationErrors.length).to.eq(0); + chai.expect(result['child_schema_simple_fk'].validationErrors.length).to.eq(0); + }); + + it('should pass foreignKey restriction validation when values exist in foreign schema (composite fk)', () => { + const parent_schema_1_data = [ + { + id: 'parent_schema_1_id_1', + external_id: 'parent_schema_1_external_id_1', + name: 'parent_schema_1_name_1', + }, + { + id: 'parent_schema_1_id_2', + external_id: 'parent_schema_1_external_id_2', + name: 'parent_schema_1_name_2', + }, + ]; + + const child_schema_composite_fk_data = [ + { + id: '1', + parent_schema_1_id: 'parent_schema_1_id_1', + parent_schema_1_external_id: 'parent_schema_1_external_id_1', + }, + { + id: '2', + parent_schema_1_id: 'parent_schema_1_id_2', + parent_schema_1_external_id: 'parent_schema_1_external_id_2', + }, + ]; + const schemaData = { + parent_schema_1: parent_schema_1_data, + child_schema_composite_fk: child_schema_composite_fk_data, + }; + + const result = schemaService.processSchemas(schema, schemaData); + + chai.expect(result['parent_schema_1'].validationErrors.length).to.eq(0); + chai.expect(result['child_schema_composite_fk'].validationErrors.length).to.eq(0); + }); + + it('should fail foreignKey restriction validation when value does not exist in foreign schema', () => { + const parent_schema_1_data = [ + { + id: 'parent_schema_1_id_1', + name: 'parent_schema_1_name_1', + }, + { + id: 'parent_schema_1_id_2', + name: 'parent_schema_1_name_2', + }, + ]; + + const child_schema_simple_fk_data = [ + { + id: '1', + parent_schema_1_id: 'parent_schema_1_id_1', + }, + { + id: '2', + parent_schema_1_id: 'non_existing_value_in_foreign_schema', + }, + ]; + const schemaData = { + parent_schema_1: parent_schema_1_data, + child_schema_simple_fk: child_schema_simple_fk_data, + }; + + const result = schemaService.processSchemas(schema, schemaData); + const childSchemaErrors = result['child_schema_simple_fk'].validationErrors; + + chai.expect(childSchemaErrors.length).to.eq(1); + chai.expect(childSchemaErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_FOREIGN_KEY, + message: + 'Record violates foreign key restriction defined for field(s) parent_schema_1_id. Key parent_schema_1_id: non_existing_value_in_foreign_schema is not present in schema parent_schema_1.', + fieldName: 'parent_schema_1_id', + index: 1, + info: { foreignSchema: 'parent_schema_1', value: { parent_schema_1_id: 'non_existing_value_in_foreign_schema' } }, + }); + }); + + it('should fail foreignKey restriction validation when values do not exist in foreign schema (composite fk)', () => { + const parent_schema_1_data = [ + { + id: 'parent_schema_1_id_1', + external_id: 'parent_schema_1_external_id_1', + name: 'parent_schema_1_name_1', + }, + { + id: 'parent_schema_1_id_2', + external_id: 'parent_schema_1_external_id_2', + name: 'parent_schema_1_name_2', + }, + ]; + + const child_schema_composite_fk_data = [ + { + id: '1', + parent_schema_1_id: 'parent_schema_1_id_1', + parent_schema_1_external_id: 'parent_schema_1_external_id_1', + }, + { + id: '2', + parent_schema_1_id: 'parent_schema_1_id_2', + parent_schema_1_external_id: 'non_existing_value_in_foreign_schema', + }, + ]; + const schemaData = { + parent_schema_1: parent_schema_1_data, + child_schema_composite_fk: child_schema_composite_fk_data, + }; + + const result = schemaService.processSchemas(schema, schemaData); + const childSchemaErrors = result['child_schema_composite_fk'].validationErrors; + + chai.expect(childSchemaErrors.length).to.eq(1); + chai.expect(childSchemaErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_FOREIGN_KEY, + message: + 'Record violates foreign key restriction defined for field(s) parent_schema_1_id, parent_schema_1_external_id. Key parent_schema_1_id: parent_schema_1_id_2, parent_schema_1_external_id: non_existing_value_in_foreign_schema is not present in schema parent_schema_1.', + fieldName: 'parent_schema_1_id, parent_schema_1_external_id', + index: 1, + info: { + foreignSchema: 'parent_schema_1', + value: { + parent_schema_1_external_id: 'non_existing_value_in_foreign_schema', + parent_schema_1_id: 'parent_schema_1_id_2', + }, + }, + }); + }); + + it('should fail foreignKey restriction validation when values (array) do not match in foreign schema (composite fk)', () => { + const parent_schema_2_data = [ + { + id1: ['id1_1', 'id1_2'], + id2: ['id2_1'], + }, + ]; + + const child_schema_composite_array_values_fk_data = [ + { + id: '1', + parent_schema_2_id1: ['id1_1'], + parent_schema_2_id12: ['id1_2', 'id2_1'], + }, + ]; + const schemaData = { + parent_schema_2: parent_schema_2_data, + child_schema_composite_array_values_fk: child_schema_composite_array_values_fk_data, + }; + + const result = schemaService.processSchemas(schema, schemaData); + const childSchemaErrors = result['child_schema_composite_array_values_fk'].validationErrors; + + chai.expect(childSchemaErrors.length).to.eq(1); + chai.expect(childSchemaErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_FOREIGN_KEY, + message: + 'Record violates foreign key restriction defined for field(s) parent_schema_2_id1, parent_schema_2_id12. Key parent_schema_2_id1: [id1_1], parent_schema_2_id12: [id1_2, id2_1] is not present in schema parent_schema_2.', + fieldName: 'parent_schema_2_id1, parent_schema_2_id12', + index: 0, + info: { + foreignSchema: 'parent_schema_2', + value: { + parent_schema_2_id1: ['id1_1'], + parent_schema_2_id12: ['id1_2', 'id2_1'], + }, + }, + }); + }); + + it('should pass uniqueKey restriction validation when only a record exists', () => { + const result = schemaService.processRecords(schema, 'unique_key_schema', [ + { + numeric_id_1: '1', + string_id_2: 'string_value', + array_string_id_3: ['array_element_1', 'array_element_2'], + }, + ]); + + chai.expect(result.validationErrors.length).to.eq(0); + }); + + it('should pass uniqueKey restriction validation when values are unique', () => { + const result = schemaService.processRecords(schema, 'unique_key_schema', [ + { + numeric_id_1: '1', + string_id_2: 'string_value', + array_string_id_3: ['array_element_1', 'array_element_2'], + }, + { + numeric_id_1: '1', + string_id_2: 'string_value', + array_string_id_3: ['array_element_1', 'array_element_x'], + }, + ]); + + chai.expect(result.validationErrors.length).to.eq(0); + }); + + it('should fail uniqueKey restriction validation when missing values are part of the key and they are not unique', () => { + const result = schemaService.processRecords(schema, 'unique_key_schema', [ + { + numeric_id_1: '', + string_id_2: '', + array_string_id_3: [], + }, + { + numeric_id_1: '', + string_id_2: '', + array_string_id_3: [], + }, + ]); + + chai.expect(result.validationErrors.length).to.eq(2); + chai.expect(result.validationErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, + message: 'Key numeric_id_1: null, string_id_2: null, array_string_id_3: null must be unique.', + fieldName: 'numeric_id_1, string_id_2, array_string_id_3', + index: 0, + info: { + uniqueKeyFields: ['numeric_id_1', 'string_id_2', 'array_string_id_3'], + value: { + numeric_id_1: '', + string_id_2: '', + array_string_id_3: '', + }, + }, + }); + chai.expect(result.validationErrors[1]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, + message: 'Key numeric_id_1: null, string_id_2: null, array_string_id_3: null must be unique.', + fieldName: 'numeric_id_1, string_id_2, array_string_id_3', + index: 1, + info: { + uniqueKeyFields: ['numeric_id_1', 'string_id_2', 'array_string_id_3'], + value: { + numeric_id_1: '', + string_id_2: '', + array_string_id_3: '', + }, + }, + }); + }); + + it('should fail uniqueKey restriction validation when values are not unique', () => { + const result = schemaService.processRecords(schema, 'unique_key_schema', [ + { + numeric_id_1: '1', + string_id_2: 'string_value', + array_string_id_3: ['array_element_1', 'array_element_2'], + }, + { + numeric_id_1: '1', + string_id_2: 'string_value', + array_string_id_3: ['array_element_1', 'array_element_2'], + }, + ]); + + chai.expect(result.validationErrors.length).to.eq(2); + chai.expect(result.validationErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, + message: + 'Key numeric_id_1: 1, string_id_2: string_value, array_string_id_3: [array_element_1, array_element_2] must be unique.', + fieldName: 'numeric_id_1, string_id_2, array_string_id_3', + index: 0, + info: { + uniqueKeyFields: ['numeric_id_1', 'string_id_2', 'array_string_id_3'], + value: { + numeric_id_1: 1, + string_id_2: 'string_value', + array_string_id_3: ['array_element_1', 'array_element_2'], + }, + }, + }); + chai.expect(result.validationErrors[1]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, + message: + 'Key numeric_id_1: 1, string_id_2: string_value, array_string_id_3: [array_element_1, array_element_2] must be unique.', + fieldName: 'numeric_id_1, string_id_2, array_string_id_3', + index: 1, + info: { + uniqueKeyFields: ['numeric_id_1', 'string_id_2', 'array_string_id_3'], + value: { + numeric_id_1: 1, + string_id_2: 'string_value', + array_string_id_3: ['array_element_1', 'array_element_2'], + }, + }, + }); + }); +}); + +const records = [ + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, +]; diff --git a/packages/client/test/schema.json b/packages/client/test/schema.json new file mode 100644 index 0000000..19411a6 --- /dev/null +++ b/packages/client/test/schema.json @@ -0,0 +1,522 @@ +[ + { + "schemas": [ + { + "name": "registration", + "description": "TSV for Registration of Donor-Specimen-Sample", + "key": "submitter_donor_id", + "fields": [ + { + "name": "program_id", + "valueType": "string", + "description": "Unique identifier for program", + "meta": { + "key": true, + "examples": "PACA-CA, BASHAR-LA" + }, + "restrictions": { + "required": true, + "regex": "^[A-Z1-9][-_A-Z1-9]{2,7}(-[A-Z][A-Z])$" + } + }, + { + "name": "submitter_donor_id", + "valueType": "string", + "description": "Unique identifier for donor, assigned by the data provider.", + "meta": { + "key": true + }, + "restrictions": { + "required": true, + "regex": "^(?!(DO|do)).+" + } + }, + { + "name": "gender", + "valueType": "string", + "description": "The gender of the patient", + "meta": { + "default": "Other" + }, + "restrictions": { + "required": true, + "codeList": ["Male", "Female", "Other"] + } + }, + { + "name": "submitter_specimen_id", + "valueType": "string", + "description": "Submitter assigned specimen id", + "meta": { + "key": true + }, + "restrictions": { + "required": true, + "regex": "^(?!(SP|sp)).+" + } + }, + { + "name": "specimen_type", + "valueType": "string", + "description": "Indicate the tissue source of the biospecimen", + "meta": { + "default": "Other" + }, + "restrictions": { + "required": true, + "codeList": [ + "Blood derived", + "Blood derived - bone marrow", + "Blood derived - peripheral blood", + "Bone marrow", + "Buccal cell", + "Lymph node", + "Solid tissue", + "Plasma", + "Serum", + "Urine", + "Cerebrospinal fluid", + "Sputum", + "NOS (Not otherwise specified)", + "Other", + "FFPE", + "Pleural effusion", + "Mononuclear cells from bone marrow", + "Saliva", + "Skin" + ] + } + }, + { + "name": "tumour_normal_designation", + "valueType": "string", + "description": "Indicate whether specimen is tumour or normal type", + "restrictions": { + "required": true, + "codeList": [ + "Normal", + "Normal - tissue adjacent to primary tumour", + "Primary tumour", + "Primary tumour - adjacent to normal", + "Primary tumour - additional new primary", + "Recurrent tumour", + "Metastatic tumour", + "Metastatic tumour - metastasis local to lymph node", + "Metastatic tumour - metastasis to distant location", + "Metastatic tumour - additional metastatic", + "Xenograft - derived from primary tumour", + "Xenograft - derived from tumour cell line", + "Cell line - derived from xenograft tissue", + "Cell line - derived from tumour", + "Cell line - derived from normal" + ] + } + }, + { + "name": "submitter_sample_id", + "valueType": "string", + "description": "Submitter assigned sample id", + "restrictions": { + "required": true, + "regex": "^(?!(SA|sa)).+" + } + }, + { + "name": "sample_type", + "valueType": "string", + "description": "Specimen Type", + "restrictions": { + "required": true, + "codeList": [ + "Total DNA", + "Amplified DNA", + "ctDNA", + "other DNA enrichments", + "Total RNA", + "Ribo-Zero RNA", + "polyA+ RNA", + "other RNA fractions" + ] + } + } + ] + }, + { + "name": "address", + "description": "adderss schema", + "fields": [ + { + "name": "postal_code", + "valueType": "string", + "description": "postal code", + "restrictions": { + "required": true, + "script": "/** important to return the result object here here */\r\n(function validate(inputs) {\r\n const {$row, $field, $name} = inputs; var person = $row;\r\nvar postalCode = $field; var result = { valid: true, message: \"ok\"};\r\n\r\n /* custom logic start */\r\n if (person.country === \"US\") {\r\n var valid = /^[0-9]{5}(?:-[0-9]{4})?$/.test(postalCode);\r\n if (!valid) {\r\n result.valid = false;\r\n result.message = \"invalid postal code for US\";\r\n }\r\n } else if (person.country === \"CANADA\") {\r\n var valid = /^[A-Za-z]\\d[A-Za-z][ -]?\\d[A-Za-z]\\d$/.test(postalCode);\r\n if (!valid) {\r\n result.valid = false;\r\n result.message = \"invalid postal code for CANADA\";\r\n }\r\n }\r\n /* custom logic end */\r\n\r\n return result;\r\n})\r\n\r\n" + } + }, + { + "name": "unit_number", + "valueType": "integer", + "description": "unit number", + "restrictions": { + "range": { + "min": 0, + "exclusiveMax": 999 + } + } + }, + { + "name": "country", + "valueType": "string", + "description": "Country", + "restrictions": { + "required": true, + "codeList": ["US", "CANADA"] + } + } + ] + }, + { + "name": "donor", + "description": "TSV for donor", + "key": "submitter_donor_id", + "fields": [ + { + "name": "program_id", + "valueType": "string", + "description": "Unique identifier for program", + "meta": { + "key": true + }, + "restrictions": { + "required": true, + "regex": "^[A-Z1-9][-_A-Z1-9]{2,7}(-[A-Z][A-Z])$" + } + }, + { + "name": "submitter_donor_id", + "valueType": "string", + "description": "Unique identifier for donor, assigned by the data provider.", + "meta": { + "key": true + }, + "restrictions": { + "required": true, + "regex": "^(?!(DO|do)).+" + } + }, + { + "name": "gender", + "valueType": "string", + "description": "The gender of the patient", + "meta": { + "default": "Other" + }, + "restrictions": { + "required": true, + "codeList": ["Male", "Female", "Other"] + } + }, + { + "name": "ethnicity", + "valueType": "string", + "description": "The ethnicity of the patient", + "restrictions": { + "required": true, + "codeList": ["asian", "black or african american", "caucasian", "not reported"] + } + }, + { + "name": "vital_status", + "valueType": "string", + "description": "Indicate the vital status of the patient", + "restrictions": { + "required": true, + "codeList": ["alive", "deceased"] + } + }, + { + "name": "cause_of_death", + "valueType": "string", + "description": "Indicate the cause of death of patient", + "restrictions": { + "required": false, + "codeList": ["died of cancer", "died of other reasons", "N/A"] + } + }, + { + "name": "survival_time", + "valueType": "integer", + "description": "Survival time", + "restrictions": { + "required": false + } + } + ] + }, + { + "name": "favorite_things", + "description": "favorite things listed", + "fields": [ + { + "name": "id", + "valueType": "string", + "description": "Favourite id values", + "restrictions": { + "required": true, + "regex": "^[A-Z1-9][-_A-Z1-9]{2,7}$" + } + }, + { + "name": "qWords", + "valueType": "string", + "description": "Words starting with q", + "restrictions": { + "required": false, + "regex": "^q.*$" + }, + "isArray": true + }, + { + "name": "qWord", + "valueType": "string", + "description": "Word starting with q", + "restrictions": { + "required": false, + "regex": "^q.*$" + }, + "isArray": false + }, + { + "name": "fruit", + "valueType": "string", + "description": "fruit", + "restrictions": { + "required": false, + "codeList": ["Mango", "Orange", "None"] + }, + "isArray": true + }, + { + "name": "fruit_single_value", + "valueType": "string", + "description": "fruit", + "restrictions": { + "required": false, + "codeList": ["Mango", "Orange", "None"] + }, + "isArray": false + }, + { + "name": "animal", + "valueType": "string", + "description": "animal", + "restrictions": { + "required": false, + "codeList": ["Dog", "Cat", "None"] + }, + "isArray": true + }, + { + "name": "fraction", + "valueType": "number", + "description": "numbers between 0 and 1 exclusive", + "restrictions": { "required": false, "range": { "max": 1, "exclusiveMin": 0 } }, + "isArray": true + }, + { + "name": "integers", + "valueType": "integer", + "description": "integers between -10 and 10", + "restrictions": { "required": false, "range": { "max": 10, "min": -10 } }, + "isArray": true + }, + { + "name": "unique_value", + "valueType": "string", + "description": "unique value", + "restrictions": { "required": false, "unique": true }, + "isArray": true + } + ] + }, + { + "name": "parent_schema_1", + "description": "Parent schema 1. Used to test relational validations", + "fields": [ + { + "name": "id", + "valueType": "string", + "description": "Id" + }, + { + "name": "external_id", + "valueType": "string", + "description": "External Id" + }, + { + "name": "name", + "valueType": "string", + "description": "Name" + } + ] + }, + { + "name": "parent_schema_2", + "description": "Parent schema 1. Used to test relational validations", + "fields": [ + { + "name": "id1", + "valueType": "string", + "description": "Id 1", + "isArray": true + }, + { + "name": "id2", + "valueType": "string", + "description": "Id 2", + "isArray": true + } + ] + }, + { + "name": "child_schema_simple_fk", + "description": "Child schema referencing a field in a foreign schema", + "restrictions": { + "foreignKey": [ + { + "schema": "parent_schema_1", + "mappings": [ + { + "local": "parent_schema_1_id", + "foreign": "id" + } + ] + } + ] + }, + "fields": [ + { + "name": "id", + "valueType": "number", + "description": "Id" + }, + { + "name": "parent_schema_1_id", + "valueType": "string", + "description": "Reference to id in schema parent_schema_1" + } + ] + }, + { + "name": "child_schema_composite_fk", + "description": "Child schema referencing several fields in a foreign schema", + "restrictions": { + "foreignKey": [ + { + "schema": "parent_schema_1", + "mappings": [ + { + "local": "parent_schema_1_id", + "foreign": "id" + }, + { + "local": "parent_schema_1_external_id", + "foreign": "external_id" + } + ] + } + ] + }, + "fields": [ + { + "name": "id", + "valueType": "number", + "description": "Id" + }, + { + "name": "parent_schema_1_id", + "valueType": "string", + "description": "Reference to id in schema parent_schema_1" + }, + { + "name": "parent_schema_1_external_id", + "valueType": "string", + "description": "Reference to external id in schema parent_schema_1" + } + ] + }, + { + "name": "child_schema_composite_array_values_fk", + "description": "Child schema referencing several fields in a foreign schema", + "restrictions": { + "foreignKey": [ + { + "schema": "parent_schema_2", + "mappings": [ + { + "local": "parent_schema_2_id1", + "foreign": "id1" + }, + { + "local": "parent_schema_2_id12", + "foreign": "id2" + } + ] + } + ] + }, + "fields": [ + { + "name": "id", + "valueType": "number", + "description": "Id" + }, + { + "name": "parent_schema_2_id1", + "valueType": "string", + "description": "Reference to id1 in schema parent_schema_2", + "isArray": true + }, + { + "name": "parent_schema_2_id12", + "valueType": "string", + "description": "Reference to external id2 in schema parent_schema_2", + "isArray": true + } + ] + }, + { + "name": "unique_key_schema", + "description": "Schema to test uniqueKey restriction", + "restrictions": { + "uniqueKey": ["numeric_id_1", "string_id_2", "array_string_id_3"] + }, + "fields": [ + { + "name": "numeric_id_1", + "valueType": "number", + "description": "Id 1. Numeric value as part of a composite unique key" + }, + { + "name": "string_id_2", + "valueType": "string", + "description": "Id 2. String value as part of a composite unique key" + }, + { + "name": "array_string_id_3", + "valueType": "string", + "description": "Id 3. String array as part of a composite unique key", + "isArray": true + } + ] + } + ], + "_id": "5d250369f38d1f0d9376fd38", + "name": "ARGO Clinical Submission", + "version": "1.0", + "createdAt": "2019-07-09T21:13:13.683Z", + "updatedAt": "2019-07-09T21:13:13.683Z", + "__v": 0 + } +] diff --git a/packages/client/tsconfig.json b/packages/client/tsconfig.json new file mode 100644 index 0000000..651ab3a --- /dev/null +++ b/packages/client/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "module": "CommonJS", + "target": "ESNext", + "lib": ["ESNext", "DOM"], + "strict": true, + "esModuleInterop": true, + "allowSyntheticDefaultImports": true, + "strictNullChecks": true, + "noImplicitAny": true, + "allowJs": true, + "moduleResolution": "node", + "experimentalDecorators": true, + "sourceMap": true, + "declaration": true, + "outDir": "dist", + "skipLibCheck": true + }, + "include": ["./src/**/*.ts"], + "exclude": ["node_modules/"] +} diff --git a/packages/client/tslint.json b/packages/client/tslint.json new file mode 100644 index 0000000..afe5fb9 --- /dev/null +++ b/packages/client/tslint.json @@ -0,0 +1,43 @@ +{ + "rules": { + "class-name": true, + "comment-format": [true, "check-space"], + "no-async-without-await": true, + "indent": [true, "tabs"], + "one-line": [true, "check-open-brace", "check-whitespace"], + "no-var-keyword": true, + "quotemark": [true, "single", "avoid-escape"], + "semicolon": [true, "always", "ignore-bound-class-methods"], + "whitespace": [ + true, + "check-branch", + "check-decl", + "check-operator", + "check-module", + "check-separator", + "check-type" + ], + "typedef-whitespace": [ + true, + { + "call-signature": "nospace", + "index-signature": "nospace", + "parameter": "nospace", + "property-declaration": "nospace", + "variable-declaration": "nospace" + }, + { + "call-signature": "onespace", + "index-signature": "onespace", + "parameter": "onespace", + "property-declaration": "onespace", + "variable-declaration": "onespace" + } + ], + "no-internal-module": true, + "no-trailing-whitespace": true, + "no-null-keyword": true, + "prefer-const": true, + "jsdoc-format": true + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d077607..48ca307 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -158,7 +158,11 @@ importers: specifier: ^3.21.3 version: 3.21.3(zod@3.21.4) - libraries/common: {} + libraries/common: + devDependencies: + rimraf: + specifier: ^3.0.2 + version: 3.0.2 libraries/dictionary: dependencies: @@ -178,6 +182,85 @@ importers: '@types/lodash': specifier: ^4.14.195 version: 4.14.195 + rimraf: + specifier: ^3.0.2 + version: 3.0.2 + + packages/client: + dependencies: + cd: + specifier: ^0.3.3 + version: 0.3.3 + common: + specifier: workspace:^ + version: link:../../libraries/common + deep-freeze: + specifier: ^0.0.1 + version: 0.0.1 + lodash: + specifier: ^4.17.21 + version: 4.17.21 + node-fetch: + specifier: ^2.6.1 + version: 2.7.0 + node-worker-threads-pool: + specifier: ^1.4.3 + version: 1.5.1 + promise-tools: + specifier: ^2.1.0 + version: 2.1.0 + winston: + specifier: ^3.3.3 + version: 3.9.0 + devDependencies: + '@types/chai': + specifier: ^4.2.16 + version: 4.3.5 + '@types/deep-freeze': + specifier: ^0.1.2 + version: 0.1.5 + '@types/lodash': + specifier: ^4.14.195 + version: 4.14.195 + '@types/mocha': + specifier: ^8.2.2 + version: 8.2.3 + '@types/node': + specifier: ^12.0.10 + version: 12.20.55 + '@types/node-fetch': + specifier: ^2.5.10 + version: 2.6.11 + chai: + specifier: ^4.3.4 + version: 4.3.7 + husky: + specifier: ^6.0.0 + version: 6.0.0 + mocha: + specifier: ^8.3.2 + version: 8.4.0 + prettier: + specifier: ^2.2.1 + version: 2.8.8 + pretty-quick: + specifier: ^3.1.0 + version: 3.3.1(prettier@2.8.8) + rimraf: + specifier: ^3.0.2 + version: 3.0.2 + ts-node: + specifier: ^9.1.1 + version: 9.1.1(typescript@5.1.6) + tslint: + specifier: ^6.1.3 + version: 6.1.3(typescript@5.1.6) + typedoc: + specifier: ^0.17.7 + version: 0.17.8(typescript@5.1.6) + typescript: + specifier: ^5.1.6 + version: 5.1.6 packages: @@ -186,7 +269,7 @@ packages: engines: {node: '>=6.0.0'} dependencies: '@jridgewell/gen-mapping': 0.3.3 - '@jridgewell/trace-mapping': 0.3.9 + '@jridgewell/trace-mapping': 0.3.18 dev: true /@babel/code-frame@7.22.5: @@ -444,6 +527,7 @@ packages: /@jridgewell/resolve-uri@3.1.1: resolution: {integrity: sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA==} engines: {node: '>=6.0.0'} + dev: false /@jridgewell/set-array@1.1.2: resolution: {integrity: sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==} @@ -469,6 +553,7 @@ packages: dependencies: '@jridgewell/resolve-uri': 3.1.1 '@jridgewell/sourcemap-codec': 1.4.15 + dev: false /@nicolo-ribaudo/semver-v6@6.3.3: resolution: {integrity: sha512-3Yc1fUTs69MG/uZbJlLSI3JISMn2UV2rg+1D/vROUqZyh3l6iYHCs7GMp+M40ZD7yOdDbYjJcU1oTJhrc+dGKg==} @@ -679,6 +764,10 @@ packages: resolution: {integrity: sha512-t73xJJrvdTjXrn4jLS9VSGRbz0nUY3cl2DMGDU48lKl+HR9dbbjW2A9r3g40VA++mQpy6uuHg33gy7du2BKpog==} dev: true + /@types/deep-freeze@0.1.5: + resolution: {integrity: sha512-KZtR+jtmgkCpgE0f+We/QEI2Fi0towBV/tTkvHVhMzx+qhUVGXMx7pWvAtDp6vEWIjdKLTKpqbI/sORRCo8TKg==} + dev: true + /@types/errorhandler@0.0.32: resolution: {integrity: sha512-wC9CfwPMIzklPd5lEYC8HnQdlMC1PswlohWmEDMWlw+E/rMYuz5eSqKBc72Earb29KptKJrRl77qVRJzrZndww==} dependencies: @@ -733,10 +822,25 @@ packages: resolution: {integrity: sha512-rADY+HtTOA52l9VZWtgQfn4p+UDVM2eDVkMZT1I6syp0YKxW2F9v+0pbRZLsvskhQv/vMb6ZfCay81GHbz5SHg==} dev: true + /@types/mocha@8.2.3: + resolution: {integrity: sha512-ekGvFhFgrc2zYQoX4JeZPmVzZxw6Dtllga7iGHzfbYIYkAMUx/sAFP2GdFpLff+vdHXu5fl7WX9AT+TtqYcsyw==} + dev: true + /@types/ms@0.7.31: resolution: {integrity: sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==} dev: true + /@types/node-fetch@2.6.11: + resolution: {integrity: sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g==} + dependencies: + '@types/node': 12.20.55 + form-data: 4.0.0 + dev: true + + /@types/node@12.20.55: + resolution: {integrity: sha512-J8xLz7q2OFulZ2cyGTLE1TbbZcjpno7FaN6zdJNrgAdrJ+DZzh/uFR6YrTb4C+nXakvud8Q4+rbhoIWlYQbUFQ==} + dev: true + /@types/node@20.4.1: resolution: {integrity: sha512-JIzsAvJeA/5iY6Y/OxZbv1lUcc8dNSE77lb2gnBH+/PJ3lFR1Ccvgwl5JWnHAkNHcRsT0TbpVOsiMKZ1F/yyJg==} @@ -903,6 +1007,11 @@ packages: engines: {node: '>=6'} dev: true + /ansi-regex@3.0.1: + resolution: {integrity: sha512-+O9Jct8wf++lXxxFc4hc8LsjaSq0HFzzL7cVsw8pRDIPdjKD2mT4ytDZlLuSBZ4cLKZFXIrMGO7DbQCtMJJMKw==} + engines: {node: '>=4'} + dev: true + /ansi-regex@4.1.1: resolution: {integrity: sha512-ILlv4k/3f6vfQ4OoP2AGvirOktlQ98ZEL1k9FaQjxa3L1abBgbuTDAdPOpvbGncC0BTVQrl+OM8xZGK6tWXt7g==} engines: {node: '>=6'} @@ -952,7 +1061,6 @@ packages: /arg@4.1.3: resolution: {integrity: sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==} - dev: false /argparse@1.0.10: resolution: {integrity: sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==} @@ -1155,6 +1263,11 @@ packages: ieee754: 1.2.1 dev: true + /builtin-modules@1.1.1: + resolution: {integrity: sha512-wxXCdllwGhI2kCC0MnvTGYTMvnVZTvqgypkiTI8Pa5tcz2i6VqsqwYGgqwXji+4RgCzms6EajE4IxiUH6HH8nQ==} + engines: {node: '>=0.10.0'} + dev: true + /byline@5.0.0: resolution: {integrity: sha512-s6webAy+R4SR8XVuJWt2V2rGvhnrhxN+9S15GNuTK3wKPOXFF6RNc+8ug2XhH+2s4f+uudG4kUVYmYOQWL2g0Q==} engines: {node: '>=0.10.0'} @@ -1218,6 +1331,10 @@ packages: resolution: {integrity: sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==} dev: false + /cd@0.3.3: + resolution: {integrity: sha512-X2y0Ssu48ucdkrNgCdg6k3EZWjWVy/dsEywUUTeZEIW31f3bQfq65Svm+TzU1Hz+qqhdmyCdjGhUvRsSKHl/mw==} + dev: false + /chai-as-promised@7.1.1(chai@4.3.7): resolution: {integrity: sha512-azL6xMoi+uxu6z4rhWQ1jbdUhOMhis2PvscD/xjLqNMkv3BPPp2JyyuTHOrf9BOosGpNQ11v6BKv/g57RXbiaA==} peerDependencies: @@ -1282,6 +1399,21 @@ packages: resolution: {integrity: sha512-BrgHpW9NURQgzoNyjfq0Wu6VFO6D7IZEmJNdtgNqpzGG8RuNFHt2jQxWlAs4HMe119chBnv+34syEZtc6IhLtA==} dev: true + /chokidar@3.5.1: + resolution: {integrity: sha512-9+s+Od+W0VJJzawDma/gvBNQqkTiqYTWLuZoyAsivsI4AaWTCzHG06/TMjsf1cYe9Cb97UCEhjz7HvnPk2p/tw==} + engines: {node: '>= 8.10.0'} + dependencies: + anymatch: 3.1.3 + braces: 3.0.2 + glob-parent: 5.1.2 + is-binary-path: 2.1.0 + is-glob: 4.0.3 + normalize-path: 3.0.0 + readdirp: 3.5.0 + optionalDependencies: + fsevents: 2.3.2 + dev: true + /chokidar@3.5.3: resolution: {integrity: sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==} engines: {node: '>= 8.10.0'} @@ -1400,6 +1532,10 @@ packages: dependencies: delayed-stream: 1.0.0 + /commander@2.20.3: + resolution: {integrity: sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==} + dev: true + /commondir@1.0.1: resolution: {integrity: sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==} dev: true @@ -1487,7 +1623,6 @@ packages: /create-require@1.1.1: resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==} - dev: false /cross-spawn@6.0.5: resolution: {integrity: sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==} @@ -1564,6 +1699,19 @@ packages: supports-color: 5.5.0 dev: true + /debug@4.3.1(supports-color@8.1.1): + resolution: {integrity: sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + dependencies: + ms: 2.1.2 + supports-color: 8.1.1 + dev: true + /debug@4.3.4(supports-color@8.1.1): resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==} engines: {node: '>=6.0'} @@ -1593,6 +1741,10 @@ packages: type-detect: 4.0.8 dev: true + /deep-freeze@0.0.1: + resolution: {integrity: sha512-Z+z8HiAvsGwmjqlphnHW5oz6yWlOwu6EQfFTjmeTWlDeda3FS2yv3jhq35TX/ewmsnqB+RX2IdsIOyjJCQN5tg==} + dev: false + /default-require-extensions@3.0.1: resolution: {integrity: sha512-eXTJmRbm2TIt9MgWTsOH1wEuhew6XGZcMeGKCtLedIg/NCsg1iBePXkceTdK4Fii7pzmN9tGsZhKzZ4h7O/fxw==} engines: {node: '>=8'} @@ -1629,7 +1781,6 @@ packages: /diff@4.0.2: resolution: {integrity: sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==} engines: {node: '>=0.3.1'} - dev: false /diff@5.0.0: resolution: {integrity: sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w==} @@ -1831,6 +1982,21 @@ packages: strip-eof: 1.0.0 dev: true + /execa@4.1.0: + resolution: {integrity: sha512-j5W0//W7f8UxAn8hXVnwG8tLwdiUy4FJLcSupCg6maBYZDpyBvTApK7KyuI4bKj8KOh1r2YH+6ucuYtJv1bTZA==} + engines: {node: '>=10'} + dependencies: + cross-spawn: 7.0.3 + get-stream: 5.2.0 + human-signals: 1.1.1 + is-stream: 2.0.1 + merge-stream: 2.0.0 + npm-run-path: 4.0.1 + onetime: 5.1.2 + signal-exit: 3.0.7 + strip-final-newline: 2.0.0 + dev: true + /express@4.18.2: resolution: {integrity: sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==} engines: {node: '>= 0.10.0'} @@ -2062,6 +2228,15 @@ packages: universalify: 2.0.0 dev: true + /fs-extra@8.1.0: + resolution: {integrity: sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==} + engines: {node: '>=6 <7 || >=8'} + dependencies: + graceful-fs: 4.2.11 + jsonfile: 4.0.0 + universalify: 0.1.2 + dev: true + /fs.realpath@1.0.0: resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==} dev: true @@ -2121,6 +2296,13 @@ packages: pump: 3.0.0 dev: true + /get-stream@5.2.0: + resolution: {integrity: sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==} + engines: {node: '>=8'} + dependencies: + pump: 3.0.0 + dev: true + /getpass@0.1.7: resolution: {integrity: sha512-0fzj9JxOLfJ+XGLhR8ze3unN0KZCgZwiSSDz168VERjK8Wl8kVSdcu2kspd4s4wtAa1y/qrVRiAA0WclVsu0ng==} dependencies: @@ -2145,8 +2327,20 @@ packages: path-is-absolute: 1.0.1 dev: true + /glob@7.1.6: + resolution: {integrity: sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==} + dependencies: + fs.realpath: 1.0.0 + inflight: 1.0.6 + inherits: 2.0.4 + minimatch: 3.1.2 + once: 1.4.0 + path-is-absolute: 1.0.1 + dev: true + /glob@7.2.0: resolution: {integrity: sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==} + deprecated: Glob versions prior to v9 are no longer supported dependencies: fs.realpath: 1.0.0 inflight: 1.0.6 @@ -2165,6 +2359,24 @@ packages: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} dev: true + /growl@1.10.5: + resolution: {integrity: sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==} + engines: {node: '>=4.x'} + dev: true + + /handlebars@4.7.8: + resolution: {integrity: sha512-vafaFqs8MZkRrSX7sFVUdo3ap/eNiLnb4IakshzvP56X5Nr1iGKAIqdX6tMlm6HcNRIkr6AxO5jFEoJzzpT8aQ==} + engines: {node: '>=0.4.7'} + hasBin: true + dependencies: + minimist: 1.2.8 + neo-async: 2.6.2 + source-map: 0.6.1 + wordwrap: 1.0.0 + optionalDependencies: + uglify-js: 3.17.4 + dev: true + /har-schema@2.0.0: resolution: {integrity: sha512-Oqluz6zhGX8cyRaTQlFMPw80bSJVG2x/cFb8ZPhUILGgHka9SsokCCOQgpveePerqidZOrT14ipqfJb7ILcW5Q==} engines: {node: '>=4'} @@ -2220,6 +2432,10 @@ packages: engines: {node: '>=8'} dev: true + /highlight.js@10.7.3: + resolution: {integrity: sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A==} + dev: true + /hosted-git-info@2.8.9: resolution: {integrity: sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==} dev: true @@ -2248,6 +2464,11 @@ packages: sshpk: 1.17.0 dev: false + /human-signals@1.1.1: + resolution: {integrity: sha512-SEQu7vl8KjNL2eoGBLF3+wAjpsNfA9XMlXAYj/3EdaNfAlxKthD1xjEQfGOUhllCGGJVNY34bRr6lPINhNjyZw==} + engines: {node: '>=8.12.0'} + dev: true + /husky@3.1.0: resolution: {integrity: sha512-FJkPoHHB+6s4a+jwPqBudBDvYZsoQW5/HBuMSehC8qDiCe50kpcxeqFoDSlow+9I6wg47YxBoT3WxaURlrDIIQ==} engines: {node: '>=8.6.0'} @@ -2267,6 +2488,11 @@ packages: slash: 3.0.0 dev: true + /husky@6.0.0: + resolution: {integrity: sha512-SQS2gDTB7tBN486QSoKPKQItZw97BMOd+Kdb6ghfpBc0yXyzrddI0oDV5MkDAbuB4X2mO3/nj60TRMcYxwzZeQ==} + hasBin: true + dev: true + /iconv-lite@0.4.24: resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==} engines: {node: '>=0.10.0'} @@ -2287,6 +2513,11 @@ packages: engines: {node: '>= 4'} dev: true + /ignore@5.3.1: + resolution: {integrity: sha512-5Fytz/IraMjqpwfd34ke28PTVMjZjJG2MPn5t7OE4eUCUNf8BAa7b5WUS9/Qvr6mwOQS7Mk6vdsMno5he+T8Xw==} + engines: {node: '>= 4'} + dev: true + /immer@10.0.2: resolution: {integrity: sha512-Rx3CqeqQ19sxUtYV9CU911Vhy8/721wRFnJv3REVGWUmoAcIwzifTsdmJte/MV+0/XpM35LZdQMBGkRIoLPwQA==} dev: false @@ -2311,6 +2542,7 @@ packages: /inflight@1.0.6: resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==} + deprecated: This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful. dependencies: once: 1.4.0 wrappy: 1.0.2 @@ -2319,6 +2551,11 @@ packages: /inherits@2.0.4: resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} + /interpret@1.4.0: + resolution: {integrity: sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==} + engines: {node: '>= 0.10'} + dev: true + /ip-regex@2.1.0: resolution: {integrity: sha512-58yWmlHpp7VYfcdTwMTvwMmqx/Elfxjd9RXTDyMsbL7lLWmhMylLEqiYVLKuLzOZqVgiWXD9MfR62Vv89VRxkw==} engines: {node: '>=4'} @@ -2529,6 +2766,13 @@ packages: esprima: 4.0.1 dev: true + /js-yaml@4.0.0: + resolution: {integrity: sha512-pqon0s+4ScYUvX30wxQi3PogGFAlUyH0awepWvwkj4jD4v+ova3RiYw8bmA6x2rDrEaj8i/oWKoRxpVNW+Re8Q==} + hasBin: true + dependencies: + argparse: 2.0.1 + dev: true + /js-yaml@4.1.0: resolution: {integrity: sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==} hasBin: true @@ -2580,6 +2824,12 @@ packages: resolution: {integrity: sha512-gfFQZrcTc8CnKXp6Y4/CBT3fTc0OVuDofpre4aEeEpSBPV5X5v4+Vmx+8snU7RLPrNHPKSgLxGo9YuQzz20o+w==} dev: true + /jsonfile@4.0.0: + resolution: {integrity: sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==} + optionalDependencies: + graceful-fs: 4.2.11 + dev: true + /jsonfile@6.1.0: resolution: {integrity: sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==} dependencies: @@ -2717,6 +2967,13 @@ packages: /lodash@4.17.21: resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==} + /log-symbols@4.0.0: + resolution: {integrity: sha512-FN8JBzLx6CzeMrB0tg6pqlGU1wCrXW+ZXGH481kfsBqer0hToTIiHdjH4Mq8xJUbvATujKCvaREGWpGUionraA==} + engines: {node: '>=10'} + dependencies: + chalk: 4.1.2 + dev: true + /log-symbols@4.1.0: resolution: {integrity: sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==} engines: {node: '>=10'} @@ -2761,6 +3018,10 @@ packages: es5-ext: 0.10.62 dev: false + /lunr@2.3.9: + resolution: {integrity: sha512-zTU3DaZaF3Rt9rhN3uBMGQD3dD2/vFQqnvZCDv4dl5iOzq2IZQqTxu90r4E5J+nP70J3ilqVCrbho2eWaeW8Ow==} + dev: true + /make-dir@3.1.0: resolution: {integrity: sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==} engines: {node: '>=8'} @@ -2770,7 +3031,12 @@ packages: /make-error@1.3.6: resolution: {integrity: sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==} - dev: false + + /marked@1.0.0: + resolution: {integrity: sha512-Wo+L1pWTVibfrSr+TTtMuiMfNzmZWiOPeO7rZsQUY5bgsxpHesBEcIWJloWVTFnrMXnf/TL30eTFSGJddmQAng==} + engines: {node: '>= 8.16.2'} + hasBin: true + dev: true /media-typer@0.3.0: resolution: {integrity: sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=} @@ -2792,6 +3058,7 @@ packages: /memory-pager@1.5.0: resolution: {integrity: sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==} + requiresBuild: true dev: false optional: true @@ -2799,6 +3066,10 @@ packages: resolution: {integrity: sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=} dev: false + /merge-stream@2.0.0: + resolution: {integrity: sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==} + dev: true + /merge2@1.4.1: resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==} engines: {node: '>= 8'} @@ -2843,6 +3114,12 @@ packages: engines: {node: '>=6'} dev: true + /minimatch@3.0.4: + resolution: {integrity: sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==} + dependencies: + brace-expansion: 1.1.11 + dev: true + /minimatch@3.0.5: resolution: {integrity: sha512-tUpxzX0VAzJHjLu0xUfFv1gwVp9ba3IOuRAVH2EGuRW8a5emA2FlACLqiT/lDVtS1W+TGNwqz3sWaNyLgDJWuw==} dependencies: @@ -2906,6 +3183,38 @@ packages: yargs-unparser: 2.0.0 dev: true + /mocha@8.4.0: + resolution: {integrity: sha512-hJaO0mwDXmZS4ghXsvPVriOhsxQ7ofcpQdm8dE+jISUOKopitvnXFQmpRR7jd2K6VBG6E26gU3IAbXXGIbu4sQ==} + engines: {node: '>= 10.12.0'} + hasBin: true + dependencies: + '@ungap/promise-all-settled': 1.1.2 + ansi-colors: 4.1.1 + browser-stdout: 1.3.1 + chokidar: 3.5.1 + debug: 4.3.1(supports-color@8.1.1) + diff: 5.0.0 + escape-string-regexp: 4.0.0 + find-up: 5.0.0 + glob: 7.1.6 + growl: 1.10.5 + he: 1.2.0 + js-yaml: 4.0.0 + log-symbols: 4.0.0 + minimatch: 3.0.4 + ms: 2.1.3 + nanoid: 3.1.20 + serialize-javascript: 5.0.1 + strip-json-comments: 3.1.1 + supports-color: 8.1.1 + which: 2.0.2 + wide-align: 1.1.3 + workerpool: 6.1.0 + yargs: 16.2.0 + yargs-parser: 20.2.4 + yargs-unparser: 2.0.0 + dev: true + /mongodb-connection-string-url@2.6.0: resolution: {integrity: sha512-WvTZlI9ab0QYtTYnuMLgobULWhokRjtC7db9LtcVfJ+Hsnyr5eo6ZtNAt3Ly24XZScGMelOcGtm7lSn0332tPQ==} dependencies: @@ -2967,6 +3276,11 @@ packages: - supports-color dev: false + /mri@1.2.0: + resolution: {integrity: sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==} + engines: {node: '>=4'} + dev: true + /ms@2.0.0: resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==} dev: false @@ -2983,6 +3297,12 @@ packages: hasBin: true dev: false + /nanoid@3.1.20: + resolution: {integrity: sha512-a1cQNyczgKbLX9jwbS/+d7W8fX/RfgYR7lVWwWOGIPNgK2m0MWvrGF6/m4kk6U3QcFMnZf3RIhL0v2Jgh/0Uxw==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + dev: true + /nanoid@3.3.3: resolution: {integrity: sha512-p1sjXuopFs0xg+fPASzQ28agW1oHD7xDsd9Xkf3T15H3c/cifrFHVwrh74PdoklAPi+i7MdRsE47vm2r6JoB+w==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} @@ -2994,6 +3314,10 @@ packages: engines: {node: '>= 0.6'} dev: false + /neo-async@2.6.2: + resolution: {integrity: sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==} + dev: true + /next-tick@1.1.0: resolution: {integrity: sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ==} dev: false @@ -3020,6 +3344,18 @@ packages: resolution: {integrity: sha512-eUXYNSY7DL53vqfTosggWkvyIW3bhAcqBDIlolgNYlZhianXTrCL50rlUJWD1eRqkIxMppXTfiFbp+9SjpPrgA==} dev: true + /node-fetch@2.7.0: + resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==} + engines: {node: 4.x || >=6.0.0} + peerDependencies: + encoding: ^0.1.0 + peerDependenciesMeta: + encoding: + optional: true + dependencies: + whatwg-url: 5.0.0 + dev: false + /node-gyp-build@4.6.0: resolution: {integrity: sha512-NTZVKn9IylLwUzaKjkas1e4u2DLNcV4rdYagA4PWdPwW87Bi7z+BznyKSRwS/761tV/lzCGXplWsiaMjLqP2zQ==} hasBin: true @@ -3048,6 +3384,10 @@ packages: - supports-color dev: false + /node-worker-threads-pool@1.5.1: + resolution: {integrity: sha512-7TXAhpMm+jO4MfESxYLtMGSnJWv+itdNHMdaFmeZuPXxwFGU90mtEB42BciUULXOUAxYBfXILAuvrSG3rQZ7mw==} + dev: false + /nodemon@2.0.22: resolution: {integrity: sha512-B8YqaKMmyuCO7BowF1Z1/mkPqLk6cs/l63Ojtd6otKjMx47Dq1utxfRxcavH1I7VSaL8n5BUaoutadnsX3AAVQ==} engines: {node: '>=8.10.0'} @@ -3386,6 +3726,11 @@ packages: engines: {node: '>=8.6'} dev: true + /picomatch@3.0.1: + resolution: {integrity: sha512-I3EurrIQMlRc9IaAZnqRR044Phh2DXY+55o7uJ0V+hYZAcQYSuFWsc9q5PvyDHUSCe1Qxn/iBz+78s86zWnGag==} + engines: {node: '>=10'} + dev: true + /pify@3.0.0: resolution: {integrity: sha512-C3FsVNH1udSEX48gGX1xfvwTWfsYWj5U+8/uK15BGzIGrKoUpghX8hWZwa/OFnakBiiVNmBvemTJR5mcy7iPcg==} engines: {node: '>=4'} @@ -3404,12 +3749,35 @@ packages: semver-compare: 1.0.0 dev: true + /prettier@2.8.8: + resolution: {integrity: sha512-tdN8qQGvNjw4CHbY+XXk0JgCXn9QiF21a55rBe5LJAU+kDyC4WQn4+awm2Xfk2lQMk5fKup9XgzTZtGkjBdP9Q==} + engines: {node: '>=10.13.0'} + hasBin: true + dev: true + /prettier@3.0.0: resolution: {integrity: sha512-zBf5eHpwHOGPC47h0zrPyNn+eAEIdEzfywMoYn2XPi0P44Zp0tSq64rq0xAREh4auw2cJZHo9QUob+NqCQky4g==} engines: {node: '>=14'} hasBin: true dev: true + /pretty-quick@3.3.1(prettier@2.8.8): + resolution: {integrity: sha512-3b36UXfYQ+IXXqex6mCca89jC8u0mYLqFAN5eTQKoXO6oCQYcIVYZEB/5AlBHI7JPYygReM2Vv6Vom/Gln7fBg==} + engines: {node: '>=10.13'} + hasBin: true + peerDependencies: + prettier: ^2.0.0 + dependencies: + execa: 4.1.0 + find-up: 4.1.0 + ignore: 5.3.1 + mri: 1.2.0 + picocolors: 1.0.0 + picomatch: 3.0.1 + prettier: 2.8.8 + tslib: 2.6.2 + dev: true + /process-nextick-args@2.0.1: resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==} dev: true @@ -3421,6 +3789,15 @@ packages: fromentries: 1.3.2 dev: true + /progress@2.0.3: + resolution: {integrity: sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==} + engines: {node: '>=0.4.0'} + dev: true + + /promise-tools@2.1.0: + resolution: {integrity: sha512-/k0nnTyAJjuMfIgAbIsoqkrfQf1U+dZapfsLkFKafc0WvmFfHWjnYdUBlWxmipqRetiUHCor+KPoAChHdXpjow==} + dev: false + /proxy-addr@2.0.7: resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} engines: {node: '>= 0.10'} @@ -3565,6 +3942,13 @@ packages: string_decoder: 1.3.0 util-deprecate: 1.0.2 + /readdirp@3.5.0: + resolution: {integrity: sha512-cMhu7c/8rdhkHXWsY+osBhfSy0JikwpHK/5+imo+LpeasTF8ouErHrlYkwT0++njiyuDvc7OFY5T3ukvZ8qmFQ==} + engines: {node: '>=8.10.0'} + dependencies: + picomatch: 2.3.1 + dev: true + /readdirp@3.6.0: resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==} engines: {node: '>=8.10.0'} @@ -3572,6 +3956,13 @@ packages: picomatch: 2.3.1 dev: true + /rechoir@0.6.2: + resolution: {integrity: sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==} + engines: {node: '>= 0.10'} + dependencies: + resolve: 1.22.2 + dev: true + /regenerator-runtime@0.13.11: resolution: {integrity: sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==} dev: true @@ -3681,6 +4072,7 @@ packages: /rimraf@3.0.2: resolution: {integrity: sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==} + deprecated: Rimraf versions prior to v4 are no longer supported hasBin: true dependencies: glob: 7.2.0 @@ -3785,6 +4177,12 @@ packages: - supports-color dev: false + /serialize-javascript@5.0.1: + resolution: {integrity: sha512-SaaNal9imEO737H2c05Og0/8LUXG7EnsZyMa8MzkmuHoELfT6txuj0cMqRj6zfPKnmQ1yasR4PCJc8x+M4JSPA==} + dependencies: + randombytes: 2.1.0 + dev: true + /serialize-javascript@6.0.0: resolution: {integrity: sha512-Qr3TosvguFt8ePWqsvRfrKyQXIiW+nGbYpy8XK24NQHE83caxWt+mIymTT19DGFbNWNLfEwsrkSmN64lVWB9ag==} dependencies: @@ -3835,6 +4233,16 @@ packages: engines: {node: '>=8'} dev: true + /shelljs@0.8.5: + resolution: {integrity: sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==} + engines: {node: '>=4'} + hasBin: true + dependencies: + glob: 7.2.0 + interpret: 1.4.0 + rechoir: 0.6.2 + dev: true + /side-channel@1.0.4: resolution: {integrity: sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==} dependencies: @@ -3892,6 +4300,13 @@ packages: smart-buffer: 4.2.0 dev: false + /source-map-support@0.5.21: + resolution: {integrity: sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==} + dependencies: + buffer-from: 1.1.2 + source-map: 0.6.1 + dev: true + /source-map@0.6.1: resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==} engines: {node: '>=0.10.0'} @@ -3899,6 +4314,7 @@ packages: /sparse-bitfield@3.0.3: resolution: {integrity: sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==} + requiresBuild: true dependencies: memory-pager: 1.5.0 dev: false @@ -3986,6 +4402,14 @@ packages: any-promise: 1.3.0 dev: true + /string-width@2.1.1: + resolution: {integrity: sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==} + engines: {node: '>=4'} + dependencies: + is-fullwidth-code-point: 2.0.0 + strip-ansi: 4.0.0 + dev: true + /string-width@3.1.0: resolution: {integrity: sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==} engines: {node: '>=6'} @@ -4019,6 +4443,13 @@ packages: dependencies: safe-buffer: 5.2.1 + /strip-ansi@4.0.0: + resolution: {integrity: sha512-4XaJ2zQdCzROZDivEVIDPkcQn8LMFSa8kj8Gxb/Lnwzv9A8VctNZ+lfivC/sV3ivW8ElJTERXZoPBRrZKkNKow==} + engines: {node: '>=4'} + dependencies: + ansi-regex: 3.0.1 + dev: true + /strip-ansi@5.2.0: resolution: {integrity: sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==} engines: {node: '>=6'} @@ -4048,6 +4479,11 @@ packages: engines: {node: '>=0.10.0'} dev: true + /strip-final-newline@2.0.0: + resolution: {integrity: sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==} + engines: {node: '>=6'} + dev: true + /strip-json-comments@3.1.1: resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} engines: {node: '>=8'} @@ -4258,6 +4694,10 @@ packages: punycode: 2.3.0 dev: false + /tr46@0.0.3: + resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} + dev: false + /tr46@3.0.0: resolution: {integrity: sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==} engines: {node: '>=12'} @@ -4305,6 +4745,22 @@ packages: yn: 3.1.1 dev: false + /ts-node@9.1.1(typescript@5.1.6): + resolution: {integrity: sha512-hPlt7ZACERQGf03M253ytLY3dHbGNGrAq9qIHWUY9XHYl1z7wYngSr3OQ5xmui8o2AaxsONxIzjafLUiWBo1Fg==} + engines: {node: '>=10.0.0'} + hasBin: true + peerDependencies: + typescript: '>=2.7' + dependencies: + arg: 4.1.3 + create-require: 1.1.1 + diff: 4.0.2 + make-error: 1.3.6 + source-map-support: 0.5.21 + typescript: 5.1.6 + yn: 3.1.1 + dev: true + /tsconfig-paths@4.2.0: resolution: {integrity: sha512-NoZ4roiN7LnbKn9QqE1amc9DJfzvZXxF4xDavcOWt1BPkdx+m+0gJuPM+S0vCe7zTJMYUP0R8pO2XMr+Y8oLIg==} engines: {node: '>=6'} @@ -4322,6 +4778,43 @@ packages: resolution: {integrity: sha512-7At1WUettjcSRHXCyYtTselblcHl9PJFFVKiCAy/bY97+BPZXSQ2wbq0P9s8tK2G7dFQfNnlJnPAiArVBVBsfA==} dev: true + /tslib@2.6.2: + resolution: {integrity: sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==} + dev: true + + /tslint@6.1.3(typescript@5.1.6): + resolution: {integrity: sha512-IbR4nkT96EQOvKE2PW/djGz8iGNeJ4rF2mBfiYaR/nvUWYKJhLwimoJKgjIFEIDibBtOevj7BqCRL4oHeWWUCg==} + engines: {node: '>=4.8.0'} + deprecated: TSLint has been deprecated in favor of ESLint. Please see https://github.com/palantir/tslint/issues/4534 for more information. + hasBin: true + peerDependencies: + typescript: '>=2.3.0-dev || >=2.4.0-dev || >=2.5.0-dev || >=2.6.0-dev || >=2.7.0-dev || >=2.8.0-dev || >=2.9.0-dev || >=3.0.0-dev || >= 3.1.0-dev || >= 3.2.0-dev || >= 4.0.0-dev' + dependencies: + '@babel/code-frame': 7.22.5 + builtin-modules: 1.1.1 + chalk: 2.4.2 + commander: 2.20.3 + diff: 4.0.2 + glob: 7.2.0 + js-yaml: 3.14.1 + minimatch: 3.1.2 + mkdirp: 0.5.6 + resolve: 1.22.2 + semver: 5.7.1 + tslib: 1.14.1 + tsutils: 2.29.0(typescript@5.1.6) + typescript: 5.1.6 + dev: true + + /tsutils@2.29.0(typescript@5.1.6): + resolution: {integrity: sha512-g5JVHCIJwzfISaXpXE1qvNalca5Jwob6FjI4AoPlqMusJ6ftFE7IkkFoMhVLRgK+4Kx3gkzb8UZK5t5yTTvEmA==} + peerDependencies: + typescript: '>=2.1.0 || >=2.1.0-dev || >=2.2.0-dev || >=2.3.0-dev || >=2.4.0-dev || >=2.5.0-dev || >=2.6.0-dev || >=2.7.0-dev || >=2.8.0-dev || >=2.9.0-dev || >= 3.0.0-dev || >= 3.1.0-dev' + dependencies: + tslib: 1.14.1 + typescript: 5.1.6 + dev: true + /tunnel-agent@0.6.0: resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} dependencies: @@ -4378,15 +4871,55 @@ packages: resolution: {integrity: sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==} dev: true + /typedoc-default-themes@0.10.2: + resolution: {integrity: sha512-zo09yRj+xwLFE3hyhJeVHWRSPuKEIAsFK5r2u47KL/HBKqpwdUSanoaz5L34IKiSATFrjG5ywmIu98hPVMfxZg==} + engines: {node: '>= 8'} + dependencies: + lunr: 2.3.9 + dev: true + + /typedoc@0.17.8(typescript@5.1.6): + resolution: {integrity: sha512-/OyrHCJ8jtzu+QZ+771YaxQ9s4g5Z3XsQE3Ma7q+BL392xxBn4UMvvCdVnqKC2T/dz03/VXSLVKOP3lHmDdc/w==} + engines: {node: '>= 8.0.0'} + hasBin: true + peerDependencies: + typescript: '>=3.8.3' + dependencies: + fs-extra: 8.1.0 + handlebars: 4.7.8 + highlight.js: 10.7.3 + lodash: 4.17.21 + lunr: 2.3.9 + marked: 1.0.0 + minimatch: 3.1.2 + progress: 2.0.3 + shelljs: 0.8.5 + typedoc-default-themes: 0.10.2 + typescript: 5.1.6 + dev: true + /typescript@5.1.6: resolution: {integrity: sha512-zaWCozRZ6DLEWAWFrVDz1H6FVXzUSfTy5FUMWsQlU8Ym5JP9eO4xkTIROFCQvhQf61z6O/G6ugw3SgAnvvm+HA==} engines: {node: '>=14.17'} hasBin: true + /uglify-js@3.17.4: + resolution: {integrity: sha512-T9q82TJI9e/C1TAxYvfb16xO120tMVFZrGA3f9/P4424DNu6ypK103y0GPFVa17yotwSyZW5iYXgjYHkGrJW/g==} + engines: {node: '>=0.8.0'} + hasBin: true + requiresBuild: true + dev: true + optional: true + /undefsafe@2.0.5: resolution: {integrity: sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==} dev: true + /universalify@0.1.2: + resolution: {integrity: sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==} + engines: {node: '>= 4.0.0'} + dev: true + /universalify@2.0.0: resolution: {integrity: sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==} engines: {node: '>= 10.0.0'} @@ -4462,6 +4995,10 @@ packages: extsprintf: 1.3.0 dev: false + /webidl-conversions@3.0.1: + resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} + dev: false + /webidl-conversions@7.0.0: resolution: {integrity: sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==} engines: {node: '>=12'} @@ -4475,6 +5012,13 @@ packages: webidl-conversions: 7.0.0 dev: false + /whatwg-url@5.0.0: + resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} + dependencies: + tr46: 0.0.3 + webidl-conversions: 3.0.1 + dev: false + /which-module@2.0.1: resolution: {integrity: sha512-iBdZ57RDvnOR9AGBhML2vFZf7h8vmBjhoaZqODJBFWHVtKkDmKuHai3cx5PgVMrX5YDNp27AofYbAwctSS+vhQ==} dev: true @@ -4494,6 +5038,12 @@ packages: isexe: 2.0.0 dev: true + /wide-align@1.1.3: + resolution: {integrity: sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==} + dependencies: + string-width: 2.1.1 + dev: true + /winston-transport@4.5.0: resolution: {integrity: sha512-YpZzcUzBedhlTAfJg6vJDlyEai/IFMIVcaEZZyl3UXIl4gmqRpU7AE89AHLkbzLUsv0NVmw7ts+iztqKxxPW1Q==} engines: {node: '>= 6.4.0'} @@ -4520,6 +5070,14 @@ packages: winston-transport: 4.5.0 dev: false + /wordwrap@1.0.0: + resolution: {integrity: sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==} + dev: true + + /workerpool@6.1.0: + resolution: {integrity: sha512-toV7q9rWNYha963Pl/qyeZ6wG+3nnsyvolaNUS8+R5Wtw6qJPTxIlOP1ZSvcGhEJw+l3HMMmtiNo9Gl61G4GVg==} + dev: true + /workerpool@6.2.1: resolution: {integrity: sha512-ILEIE97kDZvF9Wb9f6h5aXK4swSlKGUcOEGiIYb2OOu/IrDU9iwj0fD//SsA6E5ibwJxpEvhullJY4Sl4GcpAw==} dev: true @@ -4682,7 +5240,6 @@ packages: /yn@3.1.1: resolution: {integrity: sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==} engines: {node: '>=6'} - dev: false /yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} From 1ff401762616b119136413697825ea1fba48a091 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Fri, 14 Jun 2024 19:20:11 -0400 Subject: [PATCH 06/27] Refactor Client package to use shared Lectern types (#211) * Collect client errors into common, order alphabetically * Create Singular utility type to extract element types from array - helps simply type notation when processing DataRecords in client * Make a default RangeRestriction schema validator available * Update diff type FieldChange to potentially be undefined * Remove namespaces, error types moved to common * Refactor changeAnalysis with lectern shared types * Remove worker-threads code from client The client provides the logic for processing and validating data, any parallelization that is desired should be added to a server implementation and not in the reusable client. * Remove types that are duplicate of Lectern dictionary library * Move data processing functions out of schema-functions into their own directory * Reorganize validation and error message code * Move rest client code and update to Lectern shared types * Add Lectern dictionary library, remove worker-threads * Update client entry file for new code directories * Organize tests and fixtures with clean names Also updated messages in tests that have changed during code cleanup * updated lockfile for client package.json changes * Add standard copyright notice * Remove uneccessary comments left from refactor --- libraries/common/src/errors.ts | 34 +- libraries/common/src/types/index.ts | 1 + libraries/common/src/types/singular.ts | 16 + .../dictionary/src/types/dictionaryTypes.ts | 3 + libraries/dictionary/src/types/diffTypes.ts | 27 +- packages/client/package.json | 2 +- packages/client/src/change-analyzer.ts | 254 - .../src/changeAnalysis/changeAnalysisTypes.ts | 84 + .../src/changeAnalysis/changeAnalyzer.ts | 274 + packages/client/src/changeAnalysis/index.ts | 21 + packages/client/src/index.ts | 11 +- .../src/processing/convertDataValueTypes.ts | 116 + packages/client/src/processing/index.ts | 241 + .../processingResultTypes.ts} | 42 +- .../{schema-rest-client.ts => rest/index.ts} | 65 +- packages/client/src/schema-entities.ts | 221 - packages/client/src/schema-error-messages.ts | 109 - packages/client/src/schema-functions.ts | 855 -- packages/client/src/schema-worker.js | 51 - packages/client/src/types/dataRecords.ts | 39 + packages/client/src/types/index.ts | 20 + packages/client/src/utils.ts | 38 +- .../fieldNamesValidation.ts | 48 + .../valueTypeValidation.ts | 118 + .../fieldRestrictions/codeListValidation.ts | 81 + .../fieldRestrictions/rangeValidation.ts | 84 + .../fieldRestrictions/regexValidation.ts | 84 + .../fieldRestrictions/requiredValidation.ts | 72 + .../fieldRestrictions/scriptValidation.ts | 134 + packages/client/src/validation/index.ts | 31 + .../foreignKeysValidation.ts | 127 + .../schemaRestrictions/uniqueKeyValidation.ts | 69 + .../schemaRestrictions/uniqueValidation.ts | 63 + packages/client/src/validation/types/index.ts | 21 + .../validation/types/validationErrorTypes.ts | 110 + .../types/validationFunctionTypes.ts | 45 + .../utils/datasetUtils.ts} | 68 +- .../src/validation/utils/fieldTypeUtils.ts | 43 + .../src/validation/utils/rangeToSymbol.ts | 47 + .../src/validation/validationPipelines.ts | 79 + ...nalyzer.spec.ts => changeAnalyzer.spec.ts} | 47 +- packages/client/test/fixtures/diffResponse.ts | 317 + .../test/fixtures/registrationSchema.ts | 519 + packages/client/test/processing.spec.ts | 91 + packages/client/test/schema-diff.json | 312 - packages/client/test/schema-functions.spec.ts | 9295 ----------------- packages/client/test/schema.json | 522 - packages/client/test/validation.spec.ts | 828 ++ pnpm-lock.yaml | 10 +- 49 files changed, 3963 insertions(+), 11826 deletions(-) create mode 100644 libraries/common/src/types/singular.ts delete mode 100644 packages/client/src/change-analyzer.ts create mode 100644 packages/client/src/changeAnalysis/changeAnalysisTypes.ts create mode 100644 packages/client/src/changeAnalysis/changeAnalyzer.ts create mode 100644 packages/client/src/changeAnalysis/index.ts create mode 100644 packages/client/src/processing/convertDataValueTypes.ts create mode 100644 packages/client/src/processing/index.ts rename packages/client/src/{parallel.ts => processing/processingResultTypes.ts} (57%) rename packages/client/src/{schema-rest-client.ts => rest/index.ts} (72%) delete mode 100644 packages/client/src/schema-entities.ts delete mode 100644 packages/client/src/schema-error-messages.ts delete mode 100644 packages/client/src/schema-functions.ts delete mode 100644 packages/client/src/schema-worker.js create mode 100644 packages/client/src/types/dataRecords.ts create mode 100644 packages/client/src/types/index.ts create mode 100644 packages/client/src/validation/dataRecordValidation/fieldNamesValidation.ts create mode 100644 packages/client/src/validation/dataRecordValidation/valueTypeValidation.ts create mode 100644 packages/client/src/validation/fieldRestrictions/codeListValidation.ts create mode 100644 packages/client/src/validation/fieldRestrictions/rangeValidation.ts create mode 100644 packages/client/src/validation/fieldRestrictions/regexValidation.ts create mode 100644 packages/client/src/validation/fieldRestrictions/requiredValidation.ts create mode 100644 packages/client/src/validation/fieldRestrictions/scriptValidation.ts create mode 100644 packages/client/src/validation/index.ts create mode 100644 packages/client/src/validation/schemaRestrictions/foreignKeysValidation.ts create mode 100644 packages/client/src/validation/schemaRestrictions/uniqueKeyValidation.ts create mode 100644 packages/client/src/validation/schemaRestrictions/uniqueValidation.ts create mode 100644 packages/client/src/validation/types/index.ts create mode 100644 packages/client/src/validation/types/validationErrorTypes.ts create mode 100644 packages/client/src/validation/types/validationFunctionTypes.ts rename packages/client/src/{records-operations.ts => validation/utils/datasetUtils.ts} (53%) create mode 100644 packages/client/src/validation/utils/fieldTypeUtils.ts create mode 100644 packages/client/src/validation/utils/rangeToSymbol.ts create mode 100644 packages/client/src/validation/validationPipelines.ts rename packages/client/test/{change-analyzer.spec.ts => changeAnalyzer.spec.ts} (77%) create mode 100644 packages/client/test/fixtures/diffResponse.ts create mode 100644 packages/client/test/fixtures/registrationSchema.ts create mode 100644 packages/client/test/processing.spec.ts delete mode 100644 packages/client/test/schema-diff.json delete mode 100644 packages/client/test/schema-functions.spec.ts delete mode 100644 packages/client/test/schema.json create mode 100644 packages/client/test/validation.spec.ts diff --git a/libraries/common/src/errors.ts b/libraries/common/src/errors.ts index b191dad..4c4c8ad 100644 --- a/libraries/common/src/errors.ts +++ b/libraries/common/src/errors.ts @@ -17,24 +17,24 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -export class ConflictError extends Error { +export class BadRequestError extends Error { constructor(message: string) { super(message); - this.name = 'Conflict'; + this.name = 'BadRequest'; } } -export class BadRequestError extends Error { +export class ConflictError extends Error { constructor(message: string) { super(message); - this.name = 'BadRequest'; + this.name = 'Conflict'; } } -export class MalformedVersionError extends Error { +export class ForbiddenError extends Error { constructor(message: string) { super(message); - this.name = 'MalformedVersion'; + this.name = 'Forbidden'; } } @@ -45,30 +45,36 @@ export class InternalServerError extends Error { } } -export class NotFoundError extends Error { +export class InvalidArgument extends Error { + constructor(argumentName: string) { + super(`Invalid argument : ${argumentName}`); + } +} + +export class InvalidReferenceError extends Error { constructor(message: string) { super(message); - this.name = 'NotFound'; + this.name = 'InvalidReference'; } } -export class UnauthorizedError extends Error { +export class MalformedVersionError extends Error { constructor(message: string) { super(message); - this.name = 'Unauthorized'; + this.name = 'MalformedVersion'; } } -export class ForbiddenError extends Error { +export class NotFoundError extends Error { constructor(message: string) { super(message); - this.name = 'Forbidden'; + this.name = 'NotFound'; } } -export class InvalidReferenceError extends Error { +export class UnauthorizedError extends Error { constructor(message: string) { super(message); - this.name = 'InvalidReference'; + this.name = 'Unauthorized'; } } diff --git a/libraries/common/src/types/index.ts b/libraries/common/src/types/index.ts index 9fdaabd..68cee3d 100644 --- a/libraries/common/src/types/index.ts +++ b/libraries/common/src/types/index.ts @@ -1,2 +1,3 @@ export * from './generics'; export * from './result'; +export * from './singular'; diff --git a/libraries/common/src/types/singular.ts b/libraries/common/src/types/singular.ts new file mode 100644 index 0000000..06c3f1c --- /dev/null +++ b/libraries/common/src/types/singular.ts @@ -0,0 +1,16 @@ +/** + * Utility type that finds the type or types from inside an Array or nested Arrays. + * + * @example + * type String = Singular; // `String` is `string` + * type JustAString = Singular; // `JustAString` is `string` + * type StillJustAString = Singular<(string[][][][][]>; // `StillJustAString` is `string` + * type StringOrNumberFromUnion = Singular<(string[] | number)[] | string>; // `StringOrNumber` is `string | number` + * type TupleContents = Singular<[number, string, boolean]> // `TupleContents` is `number | string | boolean` + * type ArrayOfTuplesContents = Singular<[number, string, boolean]> // `ArrayOfTuplesContents` is `number | string | boolean` + */ +export type Singular = T extends Array + ? Singular + : T extends ReadonlyArray + ? Singular + : T; diff --git a/libraries/dictionary/src/types/dictionaryTypes.ts b/libraries/dictionary/src/types/dictionaryTypes.ts index 5d81c00..e7bcd5e 100644 --- a/libraries/dictionary/src/types/dictionaryTypes.ts +++ b/libraries/dictionary/src/types/dictionaryTypes.ts @@ -52,6 +52,7 @@ export type SchemaFieldValueType = zod.infer; * ************ */ export const RestrictionScript = zod.array(zod.string().or(ReferenceTag)).min(1); //TODO: script formatting validation export type RestrictionScript = zod.infer; + export const RestrictionNumberRange = zod .object({ exclusiveMax: zod.number().optional(), @@ -99,6 +100,8 @@ export const RestrictionIntegerRange = zod (data) => !(data.exclusiveMax !== undefined && data.max !== undefined), 'Range restriction cannot have both `exclusiveMax` and `max`.', ); + +export const RestrictionRange = RestrictionNumberRange; export type RestrictionRange = zod.infer; export const RestrictionRegex = zod.string().superRefine((data, context) => { diff --git a/libraries/dictionary/src/types/diffTypes.ts b/libraries/dictionary/src/types/diffTypes.ts index 44ed828..976e3a4 100644 --- a/libraries/dictionary/src/types/diffTypes.ts +++ b/libraries/dictionary/src/types/diffTypes.ts @@ -1,3 +1,22 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + import { z as zod } from 'zod'; import { SchemaField } from './dictionaryTypes'; @@ -24,8 +43,12 @@ export type ValueChange = zod.infer; // in case of created/delete field we get Change // in case of simple field change we get {"fieldName": {"data":.., "type": ..}} // in case of nested fields: {"fieldName1": {"fieldName2": {"data":.., "type": ..}}} -export type FieldChanges = { [field: string]: FieldChanges } | ValueChange; -export const FieldChanges: zod.ZodType = zod.union([zod.lazy(() => FieldChanges), ValueChange]); +export type FieldChanges = { [field: string]: FieldChanges } | ValueChange | undefined; +export const FieldChanges: zod.ZodType = zod.union([ + zod.lazy(() => FieldChanges), + ValueChange, + zod.undefined(), +]); export const FieldDiff = zod.object({ left: SchemaField.optional(), diff --git a/packages/client/package.json b/packages/client/package.json index dbdbef7..f1306e6 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -42,9 +42,9 @@ "cd": "^0.3.3", "common": "workspace:^", "deep-freeze": "^0.0.1", + "dictionary": "workspace:^", "lodash": "^4.17.21", "node-fetch": "^2.6.1", - "node-worker-threads-pool": "^1.4.3", "promise-tools": "^2.1.0", "winston": "^3.3.3" }, diff --git a/packages/client/src/change-analyzer.ts b/packages/client/src/change-analyzer.ts deleted file mode 100644 index 02e2c08..0000000 --- a/packages/client/src/change-analyzer.ts +++ /dev/null @@ -1,254 +0,0 @@ -/* - * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved - * - * This program and the accompanying materials are made available under the terms of - * the GNU Affero General Public License v3.0. You should have received a copy of the - * GNU Affero General Public License along with this program. - * If not, see . - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT - * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER - * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -import { restClient } from './schema-rest-client'; -import { - SchemasDictionaryDiffs, - FieldChanges, - FieldDiff, - Change, - ChangeAnalysis, - ChangeTypeName, - RestrictionChanges, - FieldDefinition, -} from './schema-entities'; - -const isFieldChange = (obj: any): obj is Change => { - return obj.type !== undefined; -}; - -const isNestedChange = (obj: any): obj is { [field: string]: FieldChanges } => { - return obj.type === undefined; -}; - -const isRestrictionChange = (obj: any): obj is { [field: string]: FieldChanges } => { - return obj.type === undefined; -}; - -export const fetchDiffAndAnalyze = async (serviceUrl: string, name: string, fromVersion: string, toVersion: string) => { - const changes = await restClient.fetchDiff(serviceUrl, name, fromVersion, toVersion); - return analyzeChanges(changes); -}; - -export const analyzeChanges = (schemasDiff: SchemasDictionaryDiffs): ChangeAnalysis => { - const analysis: ChangeAnalysis = { - fields: { - addedFields: [], - renamedFields: [], - deletedFields: [], - }, - isArrayDesignationChanges: [], - restrictionsChanges: { - codeList: { - created: [], - deleted: [], - updated: [], - }, - regex: { - updated: [], - created: [], - deleted: [], - }, - required: { - updated: [], - created: [], - deleted: [], - }, - script: { - updated: [], - created: [], - deleted: [], - }, - range: { - updated: [], - created: [], - deleted: [], - }, - }, - metaChanges: { - core: { - changedToCore: [], - changedFromCore: [], - }, - }, - valueTypeChanges: [], - }; - - for (const field of Object.keys(schemasDiff)) { - const fieldChange: FieldDiff = schemasDiff[field]; - if (fieldChange) { - const fieldDiff = fieldChange.diff; - // if we have type at first level then it's a field add/delete - if (isFieldChange(fieldDiff)) { - categorizeFieldChanges(analysis, field, fieldDiff); - } - - if (isNestedChange(fieldDiff)) { - if (fieldDiff.meta) { - categorizeMetaChagnes(analysis, field, fieldDiff.meta); - } - - if (fieldDiff.restrictions) { - categorizeRestrictionChanges(analysis, field, fieldDiff.restrictions, fieldChange.after); - } - - if (fieldDiff.isArray) { - categorizeFieldArrayDesignationChange(analysis, field, fieldDiff.isArray); - } - - if (fieldDiff.valueType) { - categorizerValueTypeChange(analysis, field, fieldDiff.valueType); - } - } - } - } - - return analysis; -}; - -const categorizeFieldArrayDesignationChange = ( - analysis: ChangeAnalysis, - field: string, - changes: { [field: string]: FieldChanges } | Change, -) => { - // changing isArray designation is a relevant change for all cases except if it is created and set to false - if (!(changes.type === 'created' && changes.data === false)) { - analysis.isArrayDesignationChanges.push(field); - } -}; - -const categorizerValueTypeChange = ( - analysis: ChangeAnalysis, - field: string, - changes: { [field: string]: FieldChanges } | Change, -) => { - analysis.valueTypeChanges.push(field); -}; - -const categorizeRestrictionChanges = ( - analysis: ChangeAnalysis, - field: string, - restrictionsChange: { [field: string]: FieldChanges } | Change, - fieldDefinitionAfter?: FieldDefinition, -) => { - const restrictionsToCheck = ['regex', 'script', 'required', 'codeList', 'range']; - - // additions or deletions of a restriction object as whole (i.e. contains 1 or many restrictions within the 'data') - if (restrictionsChange.type) { - const createOrAddChange = restrictionsChange as Change; - const restrictionsData = createOrAddChange.data as any; - - for (const k of restrictionsToCheck) { - if (restrictionsData[k]) { - analysis.restrictionsChanges[k as keyof RestrictionChanges][restrictionsChange.type as ChangeTypeName].push({ - field: field, - definition: restrictionsData[k], - } as any); - } - } - return; - } - - // in case 'restrictions' key was already there but we modified its contents - const restrictionUpdate = restrictionsChange as { [field: string]: FieldChanges }; - for (const k of restrictionsToCheck) { - if (restrictionUpdate[k]) { - const change = restrictionUpdate[k] as Change; - // we need the '|| change' in case of nested attributes like ranges - /* - "diff": { - "restrictions": { - "range": { - "exclusiveMin": { - "type": "deleted", - "data": 0 - }, - "max": { - "type": "updated", - "data": 200000 - }, - "min": { - "type": "created", - "data": 0 - } - } - } - } - */ - if (k == 'range' && !change.type) { - // if the change is nested (type is at min max level) then the boundries were updated only : ex: - /* - change = { - "max" : { - type: "updated" - data: "..." - }, - "exclusiveMin": { - type: "deleted" - data .. - } - } - */ - const def: any = {}; - if (Object.keys(change).some((k) => k == 'max' || k == 'min' || k == 'exclusiveMin' || k == 'exclusiveMax')) { - analysis.restrictionsChanges[k]['updated'].push({ - field: field, - // we push the whole range definition since it doesnt make sense to just - // push one boundary. - definition: fieldDefinitionAfter?.restrictions?.range, - }); - } - return; - } - const definition = change.data || change; - analysis.restrictionsChanges[k as keyof RestrictionChanges][change.type as ChangeTypeName].push({ - field: field, - definition, - } as any); - } - } -}; - -const categorizeFieldChanges = (analysis: ChangeAnalysis, field: string, changes: Change) => { - const changeType = changes.type; - if (changeType == 'created') { - analysis.fields.addedFields.push({ - name: field, - definition: changes.data, - }); - } else if (changeType == 'deleted') { - analysis.fields.deletedFields.push(field); - } -}; - -const categorizeMetaChagnes = ( - analysis: ChangeAnalysis, - field: string, - metaChanges: { [field: string]: FieldChanges } | Change, -) => { - // **** meta changes - core *** - if (metaChanges?.data?.core === true) { - const changeType = metaChanges.type; - if (changeType === 'created' || changeType === 'updated') { - analysis.metaChanges?.core.changedToCore.push(field); - } else if (changeType === 'deleted') { - analysis.metaChanges?.core.changedFromCore.push(field); - } - } -}; diff --git a/packages/client/src/changeAnalysis/changeAnalysisTypes.ts b/packages/client/src/changeAnalysis/changeAnalysisTypes.ts new file mode 100644 index 0000000..853a918 --- /dev/null +++ b/packages/client/src/changeAnalysis/changeAnalysisTypes.ts @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { SchemaField, ValueChangeTypeName } from 'dictionary'; + +type ChangeOnlyTypeNames = Exclude; + +export interface ChangeAnalysis { + fields: { + addedFields: AddedFieldChange[]; + renamedFields: string[]; + deletedFields: string[]; + }; + isArrayDesignationChanges: string[]; + restrictionsChanges: RestrictionChanges; + metaChanges?: MetaChanges; + valueTypeChanges: string[]; +} + +export type RestrictionChanges = { + range: { + [key in ChangeOnlyTypeNames]: ObjectChange[]; + }; + codeList: { + [key in ChangeOnlyTypeNames]: ObjectChange[]; + }; + regex: { + [key in ChangeOnlyTypeNames]: StringAttributeChange[]; + }; + required: { + [key in ChangeOnlyTypeNames]: BooleanAttributeChange[]; + }; + script: { + [key in ChangeOnlyTypeNames]: StringAttributeChange[]; + }; +}; +export interface AddedFieldChange { + name: string; + definition: SchemaField; +} + +export interface ObjectChange { + field: string; + definition: any; +} + +export interface CodeListChange { + field: string; + definition: any; +} + +export interface StringAttributeChange { + field: string; + definition: string; +} + +export interface BooleanAttributeChange { + field: string; + definition: boolean; +} + +// TODO: This references a specific project's meta properties that should be removed from the client +export type MetaChanges = { + core: { + changedToCore: string[]; // fields that are core now + changedFromCore: string[]; // fields that are not core now + }; +}; diff --git a/packages/client/src/changeAnalysis/changeAnalyzer.ts b/packages/client/src/changeAnalysis/changeAnalyzer.ts new file mode 100644 index 0000000..be430a3 --- /dev/null +++ b/packages/client/src/changeAnalysis/changeAnalyzer.ts @@ -0,0 +1,274 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { DictionaryDiff, FieldChanges, RestrictionRange, SchemaField, ValueChange } from 'dictionary'; +import { restClient } from '../rest'; +import { ChangeAnalysis, RestrictionChanges } from './changeAnalysisTypes'; + +const isValueChange = (input: FieldChanges): input is ValueChange => ValueChange.safeParse(input).success; + +type NestedChanges = { [field: string]: FieldChanges }; +const isNestedChange = (input: FieldChanges): input is NestedChanges => { + // Ensure that the inptu is not undefined and that it doesn't match to a ValueChange + return input !== undefined && !isValueChange(input); +}; + +export const fetchDiffAndAnalyze = async (serviceUrl: string, name: string, fromVersion: string, toVersion: string) => { + const changes = await restClient.fetchDiff(serviceUrl, name, fromVersion, toVersion); + return analyzeChanges(changes); +}; + +export const analyzeChanges = (schemasDiff: DictionaryDiff): ChangeAnalysis => { + const analysis: ChangeAnalysis = { + fields: { + addedFields: [], + renamedFields: [], + deletedFields: [], + }, + isArrayDesignationChanges: [], + restrictionsChanges: { + codeList: { + created: [], + deleted: [], + updated: [], + }, + regex: { + updated: [], + created: [], + deleted: [], + }, + required: { + updated: [], + created: [], + deleted: [], + }, + script: { + updated: [], + created: [], + deleted: [], + }, + range: { + updated: [], + created: [], + deleted: [], + }, + }, + valueTypeChanges: [], + }; + + schemasDiff.forEach((fieldChange, fieldName) => { + if (fieldChange) { + const fieldDiff = fieldChange.diff; + + // if we have type at first level then it's a field add/delete + if (isValueChange(fieldDiff)) { + categorizeFieldChanges(analysis, fieldName, fieldDiff); + } + + if (isNestedChange(fieldDiff)) { + if (fieldDiff.meta) { + categorizeMetaChanges(analysis, fieldName, fieldDiff.meta); + } + + if (fieldDiff.restrictions) { + categorizeRestrictionChanges(analysis, fieldName, fieldDiff.restrictions, fieldChange.right); + } + + if (fieldDiff.isArray) { + categorizeFieldArrayDesignationChange(analysis, fieldName, fieldDiff.isArray); + } + + if (fieldDiff.valueType) { + categorizerValueTypeChange(analysis, fieldName, fieldDiff.valueType); + } + } + } + }); + + return analysis; +}; + +const categorizeFieldArrayDesignationChange = (analysis: ChangeAnalysis, field: string, changes: FieldChanges) => { + // changing isArray designation is a relevant change for all cases except if it is created and set to false + if (!(changes?.type === 'created' && changes.data === false)) { + analysis.isArrayDesignationChanges.push(field); + } +}; + +const categorizerValueTypeChange = (analysis: ChangeAnalysis, field: string, changes: FieldChanges) => { + analysis.valueTypeChanges.push(field); +}; + +const categorizeRestrictionChanges = ( + analysis: ChangeAnalysis, + field: string, + restrictionsChange: FieldChanges, + fieldDefinitionAfter?: SchemaField, +) => { + const restrictionsToCheck: (keyof RestrictionChanges)[] = ['regex', 'script', 'required', 'codeList', 'range']; + + // additions or deletions of a restriction object as whole (i.e. contains 1 or many restrictions within the 'data') + if (isValueChange(restrictionsChange) && restrictionsChange.type !== 'unchanged') { + const createOrAddChange = restrictionsChange; + const restrictionsData = createOrAddChange.data; + + for (const k of restrictionsToCheck) { + if (restrictionsData[k]) { + switch (k) { + case 'codeList': { + analysis.restrictionsChanges[k][restrictionsChange.type].push({ + field: field, + definition: restrictionsData[k], + }); + break; + } + case 'range': { + analysis.restrictionsChanges[k][restrictionsChange.type].push({ + field: field, + definition: restrictionsData[k], + }); + break; + } + case 'regex': { + analysis.restrictionsChanges[k][restrictionsChange.type].push({ + field: field, + definition: restrictionsData[k], + }); + break; + } + case 'required': { + analysis.restrictionsChanges[k][restrictionsChange.type].push({ + field: field, + definition: restrictionsData[k], + }); + break; + } + case 'script': { + analysis.restrictionsChanges[k][restrictionsChange.type].push({ + field: field, + definition: restrictionsData[k], + }); + break; + } + } + } + } + return; + } + + // in case 'restrictions' key was already there but we modified its contents + for (const k of restrictionsToCheck) { + if (restrictionsChange && isNestedChange(restrictionsChange) && k in restrictionsChange) { + const change = restrictionsChange[k]; + if (k === 'range' && change !== undefined && !change.type) { + // if the change is nested (type is at max level) then the boundries were updated only : ex: + /* + change = { + "max" : { + type: "updated" + data: "..." + }, + "exclusiveMin": { + type: "deleted" + data .. + } + } + */ + // TODO: This section breaks from the expected format as defined by the diff types. needs to be evaluated if it is working correctly. + if ( + Object.keys(change).some((k) => k == 'max' || k == 'min' || k == 'exclusiveMin' || k == 'exclusiveMax') && + fieldDefinitionAfter?.restrictions && + 'range' in fieldDefinitionAfter.restrictions + ) { + analysis.restrictionsChanges[k]['updated'].push({ + field: field, + // we push the whole range definition since it doesnt make sense to just + // push one boundary. + definition: fieldDefinitionAfter?.restrictions?.range, + }); + } + return; + } + if (isValueChange(change) && change.type !== 'unchanged') { + const definition = change.data; + switch (k) { + case 'codeList': { + analysis.restrictionsChanges[k][change.type].push({ + field: field, + definition, + }); + break; + } + case 'range': { + analysis.restrictionsChanges[k][change.type].push({ + field: field, + definition, + }); + break; + } + case 'regex': { + analysis.restrictionsChanges[k][change.type].push({ + field: field, + definition, + }); + break; + } + case 'required': { + analysis.restrictionsChanges[k][change.type].push({ + field: field, + definition, + }); + break; + } + case 'script': { + analysis.restrictionsChanges[k][change.type].push({ + field: field, + definition, + }); + break; + } + } + } + } + } +}; + +const categorizeFieldChanges = (analysis: ChangeAnalysis, field: string, changes: ValueChange) => { + const changeType = changes.type; + if (changeType === 'created') { + analysis.fields.addedFields.push({ + name: field, + definition: changes.data, + }); + } else if (changeType === 'deleted') { + analysis.fields.deletedFields.push(field); + } +}; + +const categorizeMetaChanges = (analysis: ChangeAnalysis, field: string, metaChanges: FieldChanges) => { + // **** meta changes - core *** + if (metaChanges?.data?.core === true) { + const changeType = metaChanges.type; + if (changeType === 'created' || changeType === 'updated') { + analysis.metaChanges?.core.changedToCore.push(field); + } else if (changeType === 'deleted') { + analysis.metaChanges?.core.changedFromCore.push(field); + } + } +}; diff --git a/packages/client/src/changeAnalysis/index.ts b/packages/client/src/changeAnalysis/index.ts new file mode 100644 index 0000000..303c367 --- /dev/null +++ b/packages/client/src/changeAnalysis/index.ts @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +export * from './changeAnalysisTypes'; +export * from './changeAnalyzer'; diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts index e7cf786..4b8bd69 100644 --- a/packages/client/src/index.ts +++ b/packages/client/src/index.ts @@ -17,10 +17,9 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import * as entities from './schema-entities'; -import * as analyzer from './change-analyzer'; -import * as functions from './schema-functions'; -import * as parallel from './parallel'; +export * as DictionaryTypes from 'dictionary'; +export * as analyzer from './changeAnalysis'; +export * as functions from './processing'; +export { restClient } from './rest'; -import { restClient } from './schema-rest-client'; -export { entities, analyzer, functions, parallel, restClient }; +export type { DataRecord, DataRecordValue, UnprocessedDataRecord } from './types'; diff --git a/packages/client/src/processing/convertDataValueTypes.ts b/packages/client/src/processing/convertDataValueTypes.ts new file mode 100644 index 0000000..986b728 --- /dev/null +++ b/packages/client/src/processing/convertDataValueTypes.ts @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import _ from 'lodash'; + +import { Singular } from 'common'; +import { Schema, SchemaField, SchemaFieldValueType } from 'dictionary'; + +import { DataRecord, DataRecordValue, UnprocessedDataRecord } from '../types'; +import { convertToArray, isEmpty } from '../utils'; +import { SchemaValidationError, SchemaValidationErrorTypes } from '../validation'; + +/** + * Warning: + * This needs to be provided records that have already had their types validated, there is little to no checking that the values + * this converts will be correct, and no errors are thrown by failed conversions. + * + * @param schemaDef + * @param record + * @param index + * @param recordErrors + * @returns + */ +export const convertFromRawStrings = ( + schemaDef: Schema, + record: UnprocessedDataRecord, + index: number, + recordErrors: ReadonlyArray, +): DataRecord => { + const mutableRecord: DataRecord = { ...record }; + schemaDef.fields.forEach((field) => { + // if there was an error for this field don't convert it. this means a string was passed instead of number or boolean + // this allows us to continue other validations without hiding possible errors downstream. + + if ( + recordErrors.find( + (er) => er.errorType === SchemaValidationErrorTypes.INVALID_FIELD_VALUE_TYPE && er.fieldName === field.name, + ) + ) { + return undefined; + } + + /* + * if the field is missing from the records don't set it to undefined + */ + if (!_.has(record, field.name)) { + return; + } + + // need to check how it behaves for record[field.name] === "" + if (isEmpty(record[field.name])) { + mutableRecord[field.name] = undefined; + return; + } + + const rawValue = record[field.name]; + + if (field.isArray) { + const rawValueAsArray = convertToArray(rawValue); + // TODO: Keeping this type assertion for during the type refactoring process. We need to refactor how values are validated as matching their corresponding types + // refactoring type checking/conversion will result in combining the conversion and type checking code into a single place. Right now its possible to run the converter + // on values that have not been properly validated. + // This type assertion is needed because the code as is results in teh type `(string | number | boolean | undefined)[]` instead of `string[] | number[] | boolean[] | undefined` + mutableRecord[field.name] = rawValueAsArray.map((value) => getTypedValue(field, value)) as DataRecordValue; + } else { + const rawValueAsString = Array.isArray(rawValue) ? rawValue.join('') : rawValue; + mutableRecord[field.name] = getTypedValue(field, rawValueAsString); + } + }); + return mutableRecord; +}; + +const getTypedValue = (field: SchemaField, rawValue: string): Singular => { + switch (field.valueType) { + case SchemaFieldValueType.Values.boolean: { + return Boolean(rawValue.toLowerCase()); + } + case SchemaFieldValueType.Values.integer: { + return Number(rawValue); + } + case SchemaFieldValueType.Values.number: { + return Number(rawValue); + } + case SchemaFieldValueType.Values.string: { + // For string fields with a codeList restriction: + // we want to format the value with the same letter cases as is defined in the codeList + if (field.restrictions?.codeList && Array.isArray(field.restrictions.codeList)) { + const formattedField = field.restrictions.codeList.find( + (codeListOption) => codeListOption.toString().toLowerCase() === rawValue.toString().toLowerCase(), + ); + if (formattedField) { + return formattedField; + } + } + + // Return original string + return rawValue; + } + } +}; diff --git a/packages/client/src/processing/index.ts b/packages/client/src/processing/index.ts new file mode 100644 index 0000000..dfafb5c --- /dev/null +++ b/packages/client/src/processing/index.ts @@ -0,0 +1,241 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { NotFoundError } from 'common'; +import { Dictionary, Schema } from 'dictionary'; +import _ from 'lodash'; + +import { loggerFor } from '../logger'; +import { DataRecord, UnprocessedDataRecord } from '../types/dataRecords'; +import { convertToArray, isEmpty, isNotAbsent, isString, isStringArray, notEmpty } from '../utils'; +import type { SchemaValidationError } from '../validation'; +import * as validation from '../validation'; +import { convertFromRawStrings } from './convertDataValueTypes'; +import { BatchProcessingResult, FieldNamesByPriorityMap, SchemaProcessingResult } from './processingResultTypes'; + +const L = loggerFor(__filename); + +export const processSchemas = ( + dictionary: Dictionary, + schemasData: Record, +): Record => { + const results: Record = {}; + + Object.keys(schemasData).forEach((schemaName) => { + // Run validations at the record level + const recordLevelValidationResults = processRecords(dictionary, schemaName, schemasData[schemaName]); + + // Run cross-schema validations + const schemaDef = getNotNullSchemaDefinitionFromDictionary(dictionary, schemaName); + const crossSchemaLevelValidationResults = validation + .runCrossSchemaValidationPipeline(schemaDef, schemasData, [validation.validateForeignKeys]) + .filter(notEmpty); + + const allErrorsBySchema: validation.SchemaValidationError[] = [ + ...recordLevelValidationResults.validationErrors, + ...crossSchemaLevelValidationResults, + ]; + + results[schemaName] = { + validationErrors: allErrorsBySchema, + processedRecords: recordLevelValidationResults.processedRecords, + }; + }); + + return results; +}; + +export const processRecords = ( + dictionary: Dictionary, + definition: string, + records: UnprocessedDataRecord[], +): BatchProcessingResult => { + const schemaDef = getNotNullSchemaDefinitionFromDictionary(dictionary, definition); + + let validationErrors: SchemaValidationError[] = []; + const processedRecords: DataRecord[] = []; + + records.forEach((dataRecord, index) => { + const result = process(dictionary, definition, dataRecord, index); + validationErrors = validationErrors.concat(result.validationErrors); + processedRecords.push(_.cloneDeep(result.processedRecord)); + }); + // Record set level validations + const newErrors = validateRecordsSet(schemaDef, processedRecords); + validationErrors.push(...newErrors); + L.debug( + `done processing all rows, validationErrors: ${validationErrors.length}, validRecords: ${processedRecords.length}`, + ); + + return { + validationErrors, + processedRecords, + }; +}; + +export const process = ( + dictionary: Dictionary, + schemaName: string, + data: Readonly, + index: number, +): SchemaProcessingResult => { + const schemaDef = dictionary.schemas.find((e) => e.name === schemaName); + + if (!schemaDef) { + throw new Error(`no schema found for : ${schemaName}`); + } + + let validationErrors: SchemaValidationError[] = []; + + const defaultedRecord = populateDefaults(schemaDef, data, index); + L.debug(`done populating defaults for record #${index}`); + const result = validateUnprocessedRecord(schemaDef, defaultedRecord, index); + L.debug(`done validation for record #${index}`); + if (result && result.length > 0) { + L.debug(`${result.length} validation errors for record #${index}`); + validationErrors = validationErrors.concat(result); + } + const convertedRecord = convertFromRawStrings(schemaDef, defaultedRecord, index, result); + L.debug(`converted row #${index} from raw strings`); + const postTypeConversionValidationResult = validateAfterTypeConversion( + schemaDef, + _.cloneDeep(convertedRecord) as DataRecord, + index, + ); + + if (postTypeConversionValidationResult && postTypeConversionValidationResult.length > 0) { + validationErrors = validationErrors.concat(postTypeConversionValidationResult); + } + + L.debug(`done processing all rows, validationErrors: ${validationErrors.length}, validRecords: ${convertedRecord}`); + + return { + validationErrors, + processedRecord: convertedRecord, + }; +}; + +const getNotNullSchemaDefinitionFromDictionary = (dictionary: Dictionary, schemaName: string): Schema => { + const schemaDef = dictionary.schemas.find((e) => e.name === schemaName); + if (!schemaDef) { + throw new Error(`no schema found for : ${schemaName}`); + } + return schemaDef; +}; + +export const getSchemaFieldNamesWithPriority = (schema: Dictionary, definition: string): FieldNamesByPriorityMap => { + const schemaDef = schema.schemas.find((schema) => schema.name === definition); + if (!schemaDef) { + throw new NotFoundError(`no schema found for : ${definition}`); + } + const fieldNamesMapped: FieldNamesByPriorityMap = { required: [], optional: [] }; + schemaDef.fields.forEach((field) => { + if (field.restrictions?.required) { + fieldNamesMapped.required.push(field.name); + } else { + fieldNamesMapped.optional.push(field.name); + } + }); + return fieldNamesMapped; +}; + +/** + * Populate the passed records with the default value based on the field name if the field is + * missing from the records it will NOT be added. + * @param definition the name of the schema definition to use for these records + * @param records the list of records to populate with the default values. + */ +const populateDefaults = (schemaDef: Schema, record: UnprocessedDataRecord, index: number): UnprocessedDataRecord => { + const clonedRecord = _.cloneDeep(record); + schemaDef.fields.forEach((field) => { + const defaultValue = field.meta && field.meta.default; + if (isEmpty(defaultValue)) return undefined; + + const value = record[field.name]; + + // data record value is (or is expected to be) just one string + if (isString(value) && !field.isArray) { + if (isNotAbsent(value) && value.trim() === '') { + L.debug(`populating Default: "${defaultValue}" for "${field.name}" of record at index ${index}`); + clonedRecord[field.name] = `${defaultValue}`; + } + return undefined; + } + + // data record value is (or is expected to be) array of string + if (isStringArray(value) && field.isArray) { + if (notEmpty(value) && value.every((v) => v.trim() === '')) { + L.debug(`populating Default: "${defaultValue}" for ${field.name} of record at index ${index}`); + const arrayDefaultValue = convertToArray(defaultValue); + clonedRecord[field.name] = arrayDefaultValue.map((v) => `${v}`); + } + return undefined; + } + }); + + return _.cloneDeep(clonedRecord); +}; + +/** + * Run schema validation pipeline for a schema defintion on the list of records provided. + * @param definition the schema definition name. + * @param record the records to validate. + */ +const validateUnprocessedRecord = ( + schemaDef: Schema, + record: UnprocessedDataRecord, + index: number, +): ReadonlyArray => { + const majorErrors = validation + .runUnprocessedRecordValidationPipeline(record, index, schemaDef.fields, [ + validation.validateFieldNames, + validation.validateNonArrayFields, + validation.validateRequiredFields, + validation.validateValueTypes, + ]) + .filter(notEmpty); + return [...majorErrors]; +}; + +const validateAfterTypeConversion = ( + schemaDef: Schema, + record: DataRecord, + index: number, +): ReadonlyArray => { + const validationErrors = validation + .runRecordValidationPipeline(record, index, schemaDef.fields, [ + validation.validateRegex, + validation.validateRange, + validation.validateCodeList, + validation.validateScript, + ]) + .filter(notEmpty); + + return [...validationErrors]; +}; + +function validateRecordsSet(schemaDef: Schema, processedRecords: DataRecord[]) { + const validationErrors = validation + .runDatasetValidationPipeline(processedRecords, schemaDef, [ + validation.validateUnique, + validation.validateUniqueKey, + ]) + .filter(notEmpty); + return validationErrors; +} diff --git a/packages/client/src/parallel.ts b/packages/client/src/processing/processingResultTypes.ts similarity index 57% rename from packages/client/src/parallel.ts rename to packages/client/src/processing/processingResultTypes.ts index 603d57f..0a66a4b 100644 --- a/packages/client/src/parallel.ts +++ b/packages/client/src/processing/processingResultTypes.ts @@ -17,33 +17,23 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { DataRecord, SchemasDictionary, SchemaProcessingResult } from './schema-entities'; -import { StaticPool } from 'node-worker-threads-pool'; -import * as os from 'os'; -import { loggerFor } from './logger'; -const L = loggerFor(__filename); +import { Schema } from 'dictionary'; +import { DataRecord } from '../types/dataRecords'; +import { SchemaValidationError } from '../validation/types/validationErrorTypes'; -// check allowed cpus or use available -const cpuCount = os.cpus().length; -L.info(`available cpus: ${cpuCount}`); -const availableCpus = Number(process.env.ALLOWED_CPUS) || cpuCount; -L.info(`using ${availableCpus} cpus`); +export type ProcessingFunction = (schema: Schema, rec: Readonly, index: number) => any; -const pool = new StaticPool({ - size: availableCpus, - task: __dirname + '/schema-worker.js', -}); +export type SchemaProcessingResult = { + validationErrors: SchemaValidationError[]; + processedRecord: DataRecord; +}; -export const processRecord = async ( - dictionary: SchemasDictionary, - schemaName: string, - record: Readonly, - index: number, -): Promise => { - return (await pool.exec({ - dictionary, - schemaName, - record, - index, - })) as Promise; +export type BatchProcessingResult = { + validationErrors: SchemaValidationError[]; + processedRecords: DataRecord[]; }; + +export interface FieldNamesByPriorityMap { + required: string[]; + optional: string[]; +} diff --git a/packages/client/src/schema-rest-client.ts b/packages/client/src/rest/index.ts similarity index 72% rename from packages/client/src/schema-rest-client.ts rename to packages/client/src/rest/index.ts index a9a944c..1d39073 100644 --- a/packages/client/src/schema-rest-client.ts +++ b/packages/client/src/rest/index.ts @@ -17,25 +17,20 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { loggerFor } from './logger'; +import { unknownToString } from 'common'; +import { Dictionary, DictionaryDiff, DictionaryDiffArray, FieldDiff } from 'dictionary'; import fetch from 'node-fetch'; -import { SchemasDictionary, SchemasDictionaryDiffs, FieldChanges, FieldDiff } from './schema-entities'; import promiseTools from 'promise-tools'; -import { unknownToString } from 'common'; +import { loggerFor } from '../logger'; const L = loggerFor(__filename); export interface SchemaServiceRestClient { - fetchSchema(schemaSvcUrl: string, name: string, version: string): Promise; - fetchDiff( - schemaSvcUrl: string, - name: string, - fromVersion: string, - toVersion: string, - ): Promise; + fetchSchema(schemaSvcUrl: string, name: string, version: string): Promise; + fetchDiff(schemaSvcUrl: string, name: string, fromVersion: string, toVersion: string): Promise; } export const restClient: SchemaServiceRestClient = { - fetchSchema: async (schemaSvcUrl: string, name: string, version: string): Promise => { + fetchSchema: async (schemaSvcUrl: string, name: string, version: string): Promise => { // for testing where we need to work against stub schema if (schemaSvcUrl.startsWith('file://')) { return await loadSchemaFromFile(version, schemaSvcUrl, name); @@ -49,7 +44,7 @@ export const restClient: SchemaServiceRestClient = { L.debug(`in fetch live schema ${version}`); const schemaDictionary = await doRequest(url); // todo validate response and map it to a schema - return schemaDictionary[0] as SchemasDictionary; + return schemaDictionary[0] as Dictionary; } catch (error: unknown) { L.error(`failed to fetch schema at url: ${url} - ${unknownToString(error)}`); throw error; @@ -60,25 +55,19 @@ export const restClient: SchemaServiceRestClient = { name: string, fromVersion: string, toVersion: string, - ): Promise => { - // for testing where we need to work against stub schema - let diffResponse: any; - if (schemaSvcBaseUrl.startsWith('file://')) { - diffResponse = await loadDiffFromFile(schemaSvcBaseUrl, name, fromVersion, toVersion); - } else { - const url = `${schemaSvcBaseUrl}/diff?name=${name}&left=${fromVersion}&right=${toVersion}`; - diffResponse = (await doRequest(url)) as any[]; - } - const result: SchemasDictionaryDiffs = {}; - for (const entry of diffResponse) { - const fieldName = entry[0] as string; + ): Promise => { + // TODO: Error handling (return result?) + const url = `${schemaSvcBaseUrl}/diff?name=${name}&left=${fromVersion}&right=${toVersion}`; + const diffResponse = await doRequest(url); + + const diffArray = DictionaryDiffArray.parse(diffResponse); + + const result: DictionaryDiff = new Map(); + for (const entry of diffArray) { + const fieldName = entry[0]; if (entry[1]) { - const fieldDiff: FieldDiff = { - before: entry[1].left, - after: entry[1].right, - diff: entry[1].diff, - }; - result[fieldName] = fieldDiff; + const fieldDiff: FieldDiff = entry[1]; + result.set(fieldName, fieldDiff); } } return result; @@ -96,26 +85,26 @@ const doRequest = async (url: string) => { return await response.json(); } catch (error: unknown) { L.error(`failed to fetch schema at url: ${url} - ${unknownToString(error)}`); - throw response.status == 404 ? new Error('Not Found') : new Error('Request Failed'); + throw response.status === 404 ? new Error('Not Found') : new Error('Request Failed'); } }; async function loadSchemaFromFile(version: string, schemaSvcUrl: string, name: string) { L.debug(`in fetch stub schema ${version}`); - const result = delay(1000); + const result = delay(1000); const dictionary = await result(() => { - const dictionaries: SchemasDictionary[] = require(schemaSvcUrl.substring(7, schemaSvcUrl.length)) - .dictionaries as SchemasDictionary[]; + const dictionaries: Dictionary[] = require(schemaSvcUrl.substring(7, schemaSvcUrl.length)) + .dictionaries as Dictionary[]; if (!dictionaries) { throw new Error('your mock json is not structured correctly, see sampleFiles/sample-schema.json'); } - const dic = dictionaries.find((d: any) => d.version == version && d.name == name); + const dic = dictionaries.find((d: any) => d.version === version && d.name === name); if (!dic) { return undefined; } return dic; }); - if (dictionary == undefined) { + if (dictionary === undefined) { throw new Error("couldn't load stub dictionary with the criteria specified"); } L.debug(`schema found ${dictionary.version}`); @@ -132,14 +121,14 @@ async function loadDiffFromFile(schemaSvcBaseUrl: string, name: string, fromVers } const diff = diffResponse.find( - (d: any) => d.fromVersion == fromVersion && d.toVersion == toVersion && d.name == name, + (d) => d.fromVersion === fromVersion && d.toVersion === toVersion && d.name === name, ); if (!diff) { return undefined; } return diff; }); - if (diff == undefined) { + if (diff === undefined) { throw new Error("couldn't load stub diff with the criteria specified, check your stub file"); } return diff.data; diff --git a/packages/client/src/schema-entities.ts b/packages/client/src/schema-entities.ts deleted file mode 100644 index 810c9c8..0000000 --- a/packages/client/src/schema-entities.ts +++ /dev/null @@ -1,221 +0,0 @@ -/* - * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved - * - * This program and the accompanying materials are made available under the terms of - * the GNU Affero General Public License v3.0. You should have received a copy of the - * GNU Affero General Public License along with this program. - * If not, see . - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT - * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER - * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -import { loggerFor } from './logger'; -import { DeepReadonly } from 'deep-freeze'; -const L = loggerFor(__filename); - -export class DataRecord { - readonly [k: string]: string | string[]; -} - -export class TypedDataRecord { - readonly [k: string]: SchemaTypes; -} - -export type SchemaTypes = string | string[] | boolean | boolean[] | number | number[] | undefined; - -export interface SchemasDictionary { - version: string; - name: string; - schemas: Array; -} - -export interface SchemaDefinition { - readonly name: string; - readonly description: string; - readonly restrictions: SchemaRestriction; - readonly fields: ReadonlyArray; -} - -export interface SchemasDictionaryDiffs { - [fieldName: string]: FieldDiff; -} - -export interface FieldDiff { - before?: FieldDefinition; - after?: FieldDefinition; - diff: FieldChanges; -} - -export type SchemaData = ReadonlyArray; - -// changes can be nested -// in case of created/delete field we get Change -// in case of simple field change we get {"fieldName": {"data":.., "type": ..}} -// in case of nested fields: {"fieldName1": {"fieldName2": {"data":.., "type": ..}}} -export type FieldChanges = { [field: string]: FieldChanges } | Change; - -export enum ChangeTypeName { - CREATED = 'created', - DELETED = 'deleted', - UPDATED = 'updated', -} - -export interface Change { - type: ChangeTypeName; - data: any; -} - -export interface SchemaRestriction { - foreignKey?: { - schema: string; - mappings: { - local: string; - foreign: string; - }[]; - }[]; - uniqueKey?: string[]; -} - -export interface FieldDefinition { - name: string; - valueType: ValueType; - description: string; - meta?: { key?: boolean; default?: SchemaTypes; core?: boolean; examples?: string }; - restrictions?: { - codeList?: CodeListRestriction; - regex?: string; - script?: Array | string; - required?: boolean; - unique?: boolean; - range?: RangeRestriction; - }; - isArray?: boolean; -} - -export type CodeListRestriction = Array; - -export type RangeRestriction = { - min?: number; - max?: number; - exclusiveMin?: number; - exclusiveMax?: number; -}; - -export enum ValueType { - STRING = 'string', - INTEGER = 'integer', - NUMBER = 'number', - BOOLEAN = 'boolean', -} - -export type SchemaProcessingResult = DeepReadonly<{ - validationErrors: SchemaValidationError[]; - processedRecord: TypedDataRecord; -}>; - -export type BatchProcessingResult = DeepReadonly<{ - validationErrors: SchemaValidationError[]; - processedRecords: TypedDataRecord[]; -}>; - -export enum SchemaValidationErrorTypes { - MISSING_REQUIRED_FIELD = 'MISSING_REQUIRED_FIELD', - INVALID_FIELD_VALUE_TYPE = 'INVALID_FIELD_VALUE_TYPE', - INVALID_BY_REGEX = 'INVALID_BY_REGEX', - INVALID_BY_RANGE = 'INVALID_BY_RANGE', - INVALID_BY_SCRIPT = 'INVALID_BY_SCRIPT', - INVALID_ENUM_VALUE = 'INVALID_ENUM_VALUE', - UNRECOGNIZED_FIELD = 'UNRECOGNIZED_FIELD', - INVALID_BY_UNIQUE = 'INVALID_BY_UNIQUE', - INVALID_BY_FOREIGN_KEY = 'INVALID_BY_FOREIGN_KEY', - INVALID_BY_UNIQUE_KEY = 'INVALID_BY_UNIQUE_KEY', -} - -export interface SchemaValidationError { - readonly errorType: SchemaValidationErrorTypes; - readonly index: number; - readonly fieldName: string; - readonly info: Record; - readonly message: string; -} - -export interface FieldNamesByPriorityMap { - required: string[]; - optional: string[]; -} - -export interface ChangeAnalysis { - fields: { - addedFields: AddedFieldChange[]; - renamedFields: string[]; - deletedFields: string[]; - }; - isArrayDesignationChanges: string[]; - restrictionsChanges: RestrictionChanges; - metaChanges?: MetaChanges; - valueTypeChanges: string[]; -} - -export type RestrictionChanges = { - range: { - [key in ChangeTypeName]: ObjectChange[]; - }; - codeList: { - [key in ChangeTypeName]: ObjectChange[]; - }; - regex: RegexChanges; - required: RequiredChanges; - script: ScriptChanges; -}; - -export type MetaChanges = { - core: { - changedToCore: string[]; // fields that are core now - changedFromCore: string[]; // fields that are not core now - }; -}; - -export type RegexChanges = { - [key in ChangeTypeName]: StringAttributeChange[]; -}; - -export type RequiredChanges = { - [key in ChangeTypeName]: BooleanAttributeChange[]; -}; - -export type ScriptChanges = { - [key in ChangeTypeName]: StringAttributeChange[]; -}; - -export interface AddedFieldChange { - name: string; - definition: FieldDefinition; -} - -export interface ObjectChange { - field: string; - definition: any; -} - -export interface CodeListChange { - field: string; - definition: any; -} - -export interface StringAttributeChange { - field: string; - definition: string; -} - -export interface BooleanAttributeChange { - field: string; - definition: boolean; -} diff --git a/packages/client/src/schema-error-messages.ts b/packages/client/src/schema-error-messages.ts deleted file mode 100644 index 3ed08ab..0000000 --- a/packages/client/src/schema-error-messages.ts +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved - * - * This program and the accompanying materials are made available under the terms of - * the GNU Affero General Public License v3.0. You should have received a copy of the - * GNU Affero General Public License along with this program. - * If not, see . - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT - * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER - * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -import { isArray } from 'lodash'; -import { RangeRestriction } from './schema-entities'; - -function getForeignKeyErrorMsg(errorData: any) { - const valueEntries = Object.entries(errorData.info.value); - const formattedKeyValues: string[] = valueEntries.map(([key, value]) => { - if (isArray(value)) { - return `${key}: [${value.join(', ')}]`; - } else { - return `${key}: ${value}`; - } - }); - const valuesAsString = formattedKeyValues.join(', '); - const detail = `Key ${valuesAsString} is not present in schema ${errorData.info.foreignSchema}`; - const msg = `Record violates foreign key restriction defined for field(s) ${errorData.fieldName}. ${detail}.`; - return msg; -} - -function getUniqueKeyErrorMsg(errorData: any) { - const uniqueKeyFields: string[] = errorData.info.uniqueKeyFields; - const formattedKeyValues: string[] = uniqueKeyFields.map((fieldName) => { - const value = errorData.info.value[fieldName]; - if (isArray(value)) { - return `${fieldName}: [${value.join(', ')}]`; - } else { - return `${fieldName}: ${value === '' ? 'null' : value}`; - } - }); - const valuesAsString = formattedKeyValues.join(', '); - const msg = `Key ${valuesAsString} must be unique.`; - return msg; -} - -const INVALID_VALUE_ERROR_MESSAGE = 'The value is not permissible for this field.'; -const ERROR_MESSAGES: { [key: string]: (errorData: any) => string } = { - INVALID_FIELD_VALUE_TYPE: () => INVALID_VALUE_ERROR_MESSAGE, - INVALID_BY_REGEX: (errData) => getRegexErrorMsg(errData.info), - INVALID_BY_RANGE: (errorData) => `Value is out of permissible range, value must be ${rangeToSymbol(errorData.info)}.`, - INVALID_BY_SCRIPT: (error) => error.info.message, - INVALID_ENUM_VALUE: () => INVALID_VALUE_ERROR_MESSAGE, - MISSING_REQUIRED_FIELD: (errorData) => `${errorData.fieldName} is a required field.`, - INVALID_BY_UNIQUE: (errorData) => `Value for ${errorData.fieldName} must be unique.`, - INVALID_BY_FOREIGN_KEY: (errorData) => getForeignKeyErrorMsg(errorData), - INVALID_BY_UNIQUE_KEY: (errorData) => getUniqueKeyErrorMsg(errorData), -}; - -// Returns the formatted message for the given error key, taking any required properties from the info object -// Default value is the errorType itself (so we can identify errorTypes that we are missing messages for and the user could look up the error meaning in our docs) -const schemaErrorMessage = (errorType: string, errorData: any = {}): string => { - return errorType && Object.keys(ERROR_MESSAGES).includes(errorType) - ? ERROR_MESSAGES[errorType](errorData) - : errorType; -}; - -const rangeToSymbol = (range: RangeRestriction): string => { - let minString = ''; - let maxString = ''; - - const hasBothRange = - (range.min !== undefined || range.exclusiveMin !== undefined) && - (range.max != undefined || range.exclusiveMax !== undefined); - - if (range.min !== undefined) { - minString = `>= ${range.min}`; - } - - if (range.exclusiveMin !== undefined) { - minString = `> ${range.exclusiveMin}`; - } - - if (range.max !== undefined) { - maxString = `<= ${range.max}`; - } - - if (range.exclusiveMax !== undefined) { - maxString = `< ${range.exclusiveMax}`; - } - - return hasBothRange ? `${minString} and ${maxString}` : `${minString}${maxString}`; -}; - -function getRegexErrorMsg(info: any) { - let msg = `The value is not a permissible for this field, it must meet the regular expression: "${info.regex}".`; - if (info.examples) { - msg = msg + ` Examples: ${info.examples}`; - } - return msg; -} - -export default schemaErrorMessage; diff --git a/packages/client/src/schema-functions.ts b/packages/client/src/schema-functions.ts deleted file mode 100644 index 52c0d7f..0000000 --- a/packages/client/src/schema-functions.ts +++ /dev/null @@ -1,855 +0,0 @@ -/* - * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved - * - * This program and the accompanying materials are made available under the terms of - * the GNU Affero General Public License v3.0. You should have received a copy of the - * GNU Affero General Public License along with this program. - * If not, see . - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT - * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER - * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -import { - SchemaValidationError, - TypedDataRecord, - SchemaTypes, - SchemaProcessingResult, - FieldNamesByPriorityMap, - BatchProcessingResult, - CodeListRestriction, - RangeRestriction, - SchemaData, -} from './schema-entities'; -import vm from 'vm'; -import { - SchemasDictionary, - SchemaDefinition, - FieldDefinition, - ValueType, - DataRecord, - SchemaValidationErrorTypes, -} from './schema-entities'; - -import { - Checks, - notEmpty, - isEmptyString, - isAbsent, - F, - isNotAbsent, - isStringArray, - isString, - isEmpty, - convertToArray, - isNumberArray, -} from './utils'; -import schemaErrorMessage from './schema-error-messages'; -import { loggerFor } from './logger'; -import { DeepReadonly } from 'deep-freeze'; -import _, { isArray } from 'lodash'; -import { findDuplicateKeys, findMissingForeignKeys } from './records-operations'; -const L = loggerFor(__filename); - -export const getSchemaFieldNamesWithPriority = ( - schema: SchemasDictionary, - definition: string, -): FieldNamesByPriorityMap => { - const schemaDef: SchemaDefinition | undefined = schema.schemas.find((schema) => schema.name === definition); - if (!schemaDef) { - throw new Error(`no schema found for : ${definition}`); - } - const fieldNamesMapped: FieldNamesByPriorityMap = { required: [], optional: [] }; - schemaDef.fields.forEach((field) => { - if (field.restrictions && field.restrictions.required) { - fieldNamesMapped.required.push(field.name); - } else { - fieldNamesMapped.optional.push(field.name); - } - }); - return fieldNamesMapped; -}; - -const getNotNullSchemaDefinitionFromDictionary = ( - dictionary: SchemasDictionary, - schemaName: string, -): SchemaDefinition => { - const schemaDef: SchemaDefinition | undefined = dictionary.schemas.find((e) => e.name === schemaName); - if (!schemaDef) { - throw new Error(`no schema found for : ${schemaName}`); - } - return schemaDef; -}; - -export const processSchemas = ( - dictionary: SchemasDictionary, - schemasData: Record, -): Record => { - Checks.checkNotNull('dictionary', dictionary); - Checks.checkNotNull('schemasData', schemasData); - - const results: Record = {}; - - Object.keys(schemasData).forEach((schemaName) => { - // Run validations at the record level - const recordLevelValidationResults = processRecords(dictionary, schemaName, schemasData[schemaName]); - - // Run cross-schema validations - const schemaDef: SchemaDefinition = getNotNullSchemaDefinitionFromDictionary(dictionary, schemaName); - const crossSchemaLevelValidationResults = validation - .runCrossSchemaValidationPipeline(schemaDef, schemasData, [validation.validateForeignKey]) - .filter(notEmpty); - - const recordLevelErrors = recordLevelValidationResults.validationErrors.map((x) => { - return { - errorType: x.errorType, - index: x.index, - fieldName: x.fieldName, - info: x.info, - message: x.message, - }; - }); - - const crossSchemaLevelErrors = crossSchemaLevelValidationResults.map((x) => { - return { - errorType: x.errorType, - index: x.index, - fieldName: x.fieldName, - info: x.info, - message: x.message, - }; - }); - - const allErrorsBySchema = [...recordLevelErrors, ...crossSchemaLevelErrors]; - - results[schemaName] = F({ - validationErrors: allErrorsBySchema, - processedRecords: recordLevelValidationResults.processedRecords, - }); - }); - - return results; -}; - -export const processRecords = ( - dataSchema: SchemasDictionary, - definition: string, - records: ReadonlyArray, -): BatchProcessingResult => { - Checks.checkNotNull('records', records); - Checks.checkNotNull('dataSchema', dataSchema); - Checks.checkNotNull('definition', definition); - - const schemaDef: SchemaDefinition = getNotNullSchemaDefinitionFromDictionary(dataSchema, definition); - - let validationErrors: SchemaValidationError[] = []; - const processedRecords: TypedDataRecord[] = []; - - records.forEach((r, i) => { - const result = process(dataSchema, definition, r, i); - validationErrors = validationErrors.concat(result.validationErrors); - processedRecords.push(_.cloneDeep(result.processedRecord) as TypedDataRecord); - }); - // Record set level validations - const newErrors = validateRecordsSet(schemaDef, processedRecords); - validationErrors.push(...newErrors); - L.debug( - `done processing all rows, validationErrors: ${validationErrors.length}, validRecords: ${processedRecords.length}`, - ); - - return F({ - validationErrors, - processedRecords, - }); -}; - -export const process = ( - dataSchema: SchemasDictionary, - definition: string, - rec: Readonly, - index: number, -): SchemaProcessingResult => { - Checks.checkNotNull('records', rec); - Checks.checkNotNull('dataSchema', dataSchema); - Checks.checkNotNull('definition', definition); - - const schemaDef: SchemaDefinition | undefined = dataSchema.schemas.find((e) => e.name === definition); - - if (!schemaDef) { - throw new Error(`no schema found for : ${definition}`); - } - - let validationErrors: SchemaValidationError[] = []; - - const defaultedRecord: DataRecord = populateDefaults(schemaDef, F(rec), index); - L.debug(`done populating defaults for record #${index}`); - const result = validate(schemaDef, defaultedRecord, index); - L.debug(`done validation for record #${index}`); - if (result && result.length > 0) { - L.debug(`${result.length} validation errors for record #${index}`); - validationErrors = validationErrors.concat(result); - } - const convertedRecord = convertFromRawStrings(schemaDef, defaultedRecord, index, result); - L.debug(`converted row #${index} from raw strings`); - const postTypeConversionValidationResult = validateAfterTypeConversion( - schemaDef, - _.cloneDeep(convertedRecord) as DataRecord, - index, - ); - - if (postTypeConversionValidationResult && postTypeConversionValidationResult.length > 0) { - validationErrors = validationErrors.concat(postTypeConversionValidationResult); - } - - L.debug(`done processing all rows, validationErrors: ${validationErrors.length}, validRecords: ${convertedRecord}`); - - return F({ - validationErrors, - processedRecord: convertedRecord, - }); -}; - -/** - * Populate the passed records with the default value based on the field name if the field is - * missing from the records it will NOT be added. - * @param definition the name of the schema definition to use for these records - * @param records the list of records to populate with the default values. - */ -const populateDefaults = ( - schemaDef: Readonly, - record: DeepReadonly, - index: number, -): DataRecord => { - Checks.checkNotNull('records', record); - L.debug(`in populateDefaults ${schemaDef.name}, ${record}`); - const mutableRecord: RawMutableRecord = _.cloneDeep(record) as RawMutableRecord; - const x: SchemaDefinition = schemaDef; - schemaDef.fields.forEach((field) => { - const defaultValue = field.meta && field.meta.default; - if (isEmpty(defaultValue)) return undefined; - - const value = record[field.name]; - - // data record value is (or is expected to be) just one string - if (isString(value) && !field.isArray) { - if (isNotAbsent(value) && value.trim() === '') { - L.debug(`populating Default: ${defaultValue} for ${field.name} in record : ${record}`); - mutableRecord[field.name] = `${defaultValue}`; - } - return undefined; - } - - // data record value is (or is expected to be) array of string - if (isStringArray(value) && field.isArray) { - if (notEmpty(value) && value.every((v) => v.trim() === '')) { - L.debug(`populating Default: ${defaultValue} for ${field.name} in record : ${record}`); - const arrayDefaultValue = convertToArray(defaultValue); - mutableRecord[field.name] = arrayDefaultValue.map((v) => `${v}`); - } - return undefined; - } - }); - - return _.cloneDeep(mutableRecord); -}; - -const convertFromRawStrings = ( - schemaDef: SchemaDefinition, - record: DataRecord, - index: number, - recordErrors: ReadonlyArray, -): DeepReadonly => { - const mutableRecord: MutableRecord = { ...record }; - schemaDef.fields.forEach((field) => { - // if there was an error for this field don't convert it. this means a string was passed instead of number or boolean - // this allows us to continue other validations without hiding possible errors down. - if ( - recordErrors.find( - (er) => er.errorType == SchemaValidationErrorTypes.INVALID_FIELD_VALUE_TYPE && er.fieldName == field.name, - ) - ) { - return undefined; - } - - /* - * if the field is missing from the records don't set it to undefined - */ - if (!_.has(record, field.name)) { - return; - } - - // need to check how it behaves for record[field.name] == "" - if (isEmpty(record[field.name])) { - mutableRecord[field.name] = undefined; - return; - } - - const valueType = field.valueType; - const rawValue = record[field.name]; - - if (field.isArray) { - const rawValues = convertToArray(rawValue); - mutableRecord[field.name] = rawValues.map( - (rv) => getTypedValue(field, valueType, rv) as any, // fix type here - ); - } else { - mutableRecord[field.name] = getTypedValue(field, valueType, rawValue as string); - } - }); - return F(mutableRecord); -}; - -const getTypedValue = (field: FieldDefinition, valueType: ValueType, rawValue: string) => { - let formattedFieldValue = rawValue; - // convert field to match corresponding enum from codelist, if possible - if (field.restrictions && field.restrictions.codeList && valueType === ValueType.STRING) { - const formattedField = field.restrictions.codeList.find( - (e) => e.toString().toLowerCase() === rawValue.toString().toLowerCase(), - ); - if (formattedField) { - formattedFieldValue = formattedField as string; - } - } - - let typedValue: SchemaTypes = rawValue; - switch (valueType) { - case ValueType.STRING: - typedValue = formattedFieldValue; - break; - case ValueType.INTEGER: - typedValue = Number(rawValue); - break; - case ValueType.NUMBER: - typedValue = Number(rawValue); - break; - case ValueType.BOOLEAN: - // we have to lower case in case of inconsistent letters (boolean requires all small letters). - typedValue = Boolean(rawValue.toLowerCase()); - break; - } - - return typedValue; -}; - -/** - * A "select" function that retrieves specific fields from the dataset as a record, as well as the numeric position of each row in the dataset. - * @param dataset Dataset to select fields from. - * @param fields Array with names of the fields to select. - * @returns A tuple array. In each tuple, the first element is the index of the row in the dataset, and the second value is the record with the - * selected values. - */ -const selectFieldsFromDataset = ( - dataset: SchemaData, - fields: string[], -): [number, Record][] => { - const records: [number, Record][] = []; - dataset.forEach((row, index) => { - const values: Record = {}; - fields.forEach((field) => { - values[field] = row[field] || ''; - }); - records.push([index, values]); - }); - return records; -}; - -/** - * Run schema validation pipeline for a schema defintion on the list of records provided. - * @param definition the schema definition name. - * @param record the records to validate. - */ -const validate = ( - schemaDef: SchemaDefinition, - record: DataRecord, - index: number, -): ReadonlyArray => { - const majorErrors = validation - .runValidationPipeline(record, index, schemaDef.fields, [ - validation.validateFieldNames, - validation.validateNonArrayFields, - validation.validateRequiredFields, - validation.validateValueTypes, - ]) - .filter(notEmpty); - return [...majorErrors]; -}; - -const validateAfterTypeConversion = ( - schemaDef: SchemaDefinition, - record: TypedDataRecord, - index: number, -): ReadonlyArray => { - const validationErrors = validation - .runValidationPipeline(record, index, schemaDef.fields, [ - validation.validateRegex, - validation.validateRange, - validation.validateEnum, - validation.validateScript, - ]) - .filter(notEmpty); - - return [...validationErrors]; -}; -export type ProcessingFunction = (schema: SchemaDefinition, rec: Readonly, index: number) => any; - -type MutableRecord = { [key: string]: SchemaTypes }; -type RawMutableRecord = { [key: string]: string | string[] }; - -namespace validation { - // these validation functions run AFTER the record has been converted to the correct types from raw strings - export type TypedValidationFunction = ( - rec: TypedDataRecord, - index: number, - fields: Array, - ) => Array; - - // these validation functions run BEFORE the record has been converted to the correct types from raw strings - export type ValidationFunction = ( - rec: DataRecord, - index: number, - fields: Array, - ) => Array; - - // these validation functions run AFTER the records has been converted to the correct types from raw strings, and apply to a dataset instead of - // individual records - export type TypedDatasetValidationFunction = ( - dataset: Array, - schemaDef: SchemaDefinition, - ) => Array; - - export type CrossSchemaValidationFunction = ( - schemaDef: SchemaDefinition, - schemasData: Record, - ) => Array; - - export const runValidationPipeline = ( - rec: DataRecord | TypedDataRecord, - index: number, - fields: ReadonlyArray, - funs: Array, - ) => { - let result: Array = []; - for (const fun of funs) { - if (rec instanceof DataRecord) { - const typedFunc = fun as ValidationFunction; - result = result.concat(typedFunc(rec as DataRecord, index, getValidFields(result, fields))); - } else { - const typedFunc = fun as TypedValidationFunction; - result = result.concat(typedFunc(rec as TypedDataRecord, index, getValidFields(result, fields))); - } - } - return result; - }; - - export const runDatasetValidationPipeline = ( - dataset: Array, - schemaDef: SchemaDefinition, - funs: Array, - ) => { - let result: Array = []; - for (const fun of funs) { - const typedFunc = fun as TypedDatasetValidationFunction; - result = result.concat(typedFunc(dataset, schemaDef)); - } - return result; - }; - - export const runCrossSchemaValidationPipeline = ( - schemaDef: SchemaDefinition, - schemasData: Record, - funs: Array, - ) => { - let result: Array = []; - for (const fun of funs) { - const typedFunc = fun as CrossSchemaValidationFunction; - result = result.concat(typedFunc(schemaDef, schemasData)); - } - return result; - }; - - export const validateRegex: TypedValidationFunction = ( - rec: TypedDataRecord, - index: number, - fields: ReadonlyArray, - ) => { - return fields - .map((field) => { - const recordFieldValues = convertToArray(rec[field.name]); - if (!isStringArray(recordFieldValues)) return undefined; - - const regex = field.restrictions?.regex; - if (isEmpty(regex)) return undefined; - - const invalidValues = recordFieldValues.filter((v) => isInvalidRegexValue(regex, v)); - if (invalidValues.length !== 0) { - const examples = field.meta?.examples; - const info = { value: invalidValues, regex, examples }; - return buildError(SchemaValidationErrorTypes.INVALID_BY_REGEX, field.name, index, info); - } - return undefined; - }) - .filter(notEmpty); - }; - - export const validateRange: TypedValidationFunction = ( - rec: TypedDataRecord, - index: number, - fields: ReadonlyArray, - ) => { - return fields - .map((field) => { - const recordFieldValues = convertToArray(rec[field.name]); - if (!isNumberArray(recordFieldValues)) return undefined; - - const range = field.restrictions?.range; - if (isEmpty(range)) return undefined; - - const invalidValues = recordFieldValues.filter((v) => isOutOfRange(range, v)); - if (invalidValues.length !== 0) { - const info = { value: invalidValues, ...range }; - return buildError(SchemaValidationErrorTypes.INVALID_BY_RANGE, field.name, index, info); - } - return undefined; - }) - .filter(notEmpty); - }; - - export const validateScript: TypedValidationFunction = ( - rec: TypedDataRecord, - index: number, - fields: Array, - ) => { - return fields - .map((field) => { - if (field.restrictions && field.restrictions.script) { - const scriptResult = validateWithScript(field, rec); - if (!scriptResult.valid) { - return buildError(SchemaValidationErrorTypes.INVALID_BY_SCRIPT, field.name, index, { - message: scriptResult.message, - value: rec[field.name], - }); - } - } - return undefined; - }) - .filter(notEmpty); - }; - - export const validateEnum: TypedValidationFunction = ( - rec: TypedDataRecord, - index: number, - fields: Array, - ) => { - return fields - .map((field) => { - const codeList = field.restrictions?.codeList || undefined; - if (isEmpty(codeList)) return undefined; - - const recordFieldValues = convertToArray(rec[field.name]); // put all values into array for easier validation - const invalidValues = recordFieldValues.filter((val) => isInvalidEnumValue(codeList, val)); - - if (invalidValues.length !== 0) { - const info = { value: invalidValues }; - return buildError(SchemaValidationErrorTypes.INVALID_ENUM_VALUE, field.name, index, info); - } - return undefined; - }) - .filter(notEmpty); - }; - - export const validateUnique: TypedDatasetValidationFunction = ( - dataset: Array, - schemaDef: SchemaDefinition, - ) => { - const errors: Array = []; - schemaDef.fields.forEach((field) => { - const unique = field.restrictions?.unique || undefined; - if (!unique) return undefined; - const keysToValidate = selectFieldsFromDataset(dataset as DataRecord[], [field.name]); - const duplicateKeys = findDuplicateKeys(keysToValidate); - - duplicateKeys.forEach(([index, record]) => { - const info = { value: record[field.name] }; - errors.push(buildError(SchemaValidationErrorTypes.INVALID_BY_UNIQUE, field.name, index, info)); - }); - }); - return errors; - }; - - export const validateUniqueKey: TypedDatasetValidationFunction = ( - dataset: Array, - schemaDef: SchemaDefinition, - ) => { - const errors: Array = []; - const uniqueKeyRestriction = schemaDef?.restrictions?.uniqueKey; - if (uniqueKeyRestriction) { - const uniqueKeyFields: string[] = uniqueKeyRestriction; - const keysToValidate = selectFieldsFromDataset(dataset as SchemaData, uniqueKeyFields); - const duplicateKeys = findDuplicateKeys(keysToValidate); - - duplicateKeys.forEach(([index, record]) => { - const info = { value: record, uniqueKeyFields: uniqueKeyFields }; - errors.push( - buildError(SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, uniqueKeyFields.join(', '), index, info), - ); - }); - } - return errors; - }; - - export const validateValueTypes: ValidationFunction = ( - rec: DataRecord, - index: number, - fields: Array, - ) => { - return fields - .map((field) => { - if (isEmpty(rec[field.name])) return undefined; - - const recordFieldValues = convertToArray(rec[field.name]); // put all values into array - const invalidValues = recordFieldValues.filter((v) => isInvalidFieldType(field.valueType, v)); - const info = { value: invalidValues }; - - if (invalidValues.length !== 0) { - return buildError(SchemaValidationErrorTypes.INVALID_FIELD_VALUE_TYPE, field.name, index, info); - } - return undefined; - }) - .filter(notEmpty); - }; - - export const validateRequiredFields = (rec: DataRecord, index: number, fields: Array) => { - return fields - .map((field) => { - if (isRequiredMissing(field, rec)) { - return buildError(SchemaValidationErrorTypes.MISSING_REQUIRED_FIELD, field.name, index); - } - return undefined; - }) - .filter(notEmpty); - }; - - export const validateFieldNames: ValidationFunction = ( - record: Readonly, - index: number, - fields: Array, - ) => { - const expectedFields = new Set(fields.map((field) => field.name)); - return Object.keys(record) - .map((recFieldName) => { - if (!expectedFields.has(recFieldName)) { - return buildError(SchemaValidationErrorTypes.UNRECOGNIZED_FIELD, recFieldName, index); - } - return undefined; - }) - .filter(notEmpty); - }; - - export const validateNonArrayFields: ValidationFunction = ( - record: Readonly, - index: number, - fields: Array, - ) => { - return fields - .map((field) => { - if (!field.isArray && isStringArray(record[field.name])) { - return buildError(SchemaValidationErrorTypes.INVALID_FIELD_VALUE_TYPE, field.name, index); - } - return undefined; - }) - .filter(notEmpty); - }; - - export const validateForeignKey: CrossSchemaValidationFunction = ( - schemaDef: SchemaDefinition, - schemasData: Record, - ) => { - const errors: Array = []; - const foreignKeyDefinitions = schemaDef?.restrictions?.foreignKey; - if (foreignKeyDefinitions) { - foreignKeyDefinitions.forEach((foreignKeyDefinition) => { - const localSchemaData = schemasData[schemaDef.name] || []; - const foreignSchemaData = schemasData[foreignKeyDefinition.schema] || []; - - // A foreign key can have more than one field, in which case is a composite foreign key. - const localFields = foreignKeyDefinition.mappings.map((x) => x.local); - const foreignFields = foreignKeyDefinition.mappings.map((x) => x.foreign); - - const fieldsMappings = new Map(foreignKeyDefinition.mappings.map((x) => [x.foreign, x.local])); - - // Select the keys of the datasets to compare. The keys are records to support the scenario where the fk is composite. - const localValues: [number, Record][] = selectFieldsFromDataset( - localSchemaData, - localFields, - ); - const foreignValues: [number, Record][] = selectFieldsFromDataset( - foreignSchemaData, - foreignFields, - ); - - // This artificial record in foreignValues allows null references in localValues to be valid. - const emptyRow: Record = {}; - foreignFields.forEach((field) => (emptyRow[field] = '')); - foreignValues.push([-1, emptyRow]); - - const missingForeignKeys = findMissingForeignKeys(localValues, foreignValues, fieldsMappings); - - missingForeignKeys.forEach((record) => { - const index = record[0]; - const info = { - value: record[1], - foreignSchema: foreignKeyDefinition.schema, - }; - - errors.push( - buildError(SchemaValidationErrorTypes.INVALID_BY_FOREIGN_KEY, localFields.join(', '), index, info), - ); - }); - }); - } - return errors; - }; - - export const getValidFields = ( - errs: ReadonlyArray, - fields: ReadonlyArray, - ) => { - return fields.filter((field) => { - return !errs.find((e) => e.fieldName == field.name); - }); - }; - - // return false if the record value is a valid type - export const isInvalidFieldType = (valueType: ValueType, value: string) => { - // optional field if the value is absent at this point - if (isAbsent(value) || isEmptyString(value)) return false; - switch (valueType) { - case ValueType.STRING: - return false; - case ValueType.INTEGER: - return isNaN(Number(value)) || !Number.isInteger(Number(value)); - case ValueType.NUMBER: - return isNaN(Number(value)); - case ValueType.BOOLEAN: - return !(value.toLowerCase() === 'true' || value.toLowerCase() === 'false'); - } - }; - - export const isRequiredMissing = (field: FieldDefinition, record: DataRecord) => { - const isRequired = field.restrictions && field.restrictions.required; - if (!isRequired) return false; - - const recordFieldValues = convertToArray(record[field.name]); - return recordFieldValues.every(isEmptyString); - }; - - const isOutOfRange = (range: RangeRestriction, value: number | undefined) => { - if (value == undefined) return false; - const invalidRange = - // less than the min if defined ? - (range.min !== undefined && value < range.min) || - (range.exclusiveMin !== undefined && value <= range.exclusiveMin) || - // bigger than max if defined ? - (range.max !== undefined && value > range.max) || - (range.exclusiveMax !== undefined && value >= range.exclusiveMax); - return invalidRange; - }; - - const isInvalidEnumValue = (codeList: CodeListRestriction, value: string | boolean | number | undefined) => { - // optional field if the value is absent at this point - if (isAbsent(value) || isEmptyString(value as string)) return false; - return !codeList.find((e) => e === value); - }; - - const isInvalidRegexValue = (regex: string, value: string) => { - // optional field if the value is absent at this point - if (isAbsent(value) || isEmptyString(value)) return false; - const regexPattern = new RegExp(regex); - return !regexPattern.test(value); - }; - - const ctx = vm.createContext(); - - const validateWithScript = ( - field: FieldDefinition, - record: TypedDataRecord, - ): { - valid: boolean; - message: string; - } => { - try { - const args = { - $row: record, - $field: record[field.name], - $name: field.name, - }; - - if (!field.restrictions || !field.restrictions.script) { - throw new Error('called validation by script without script provided'); - } - - // scripts should already be strings inside arrays, but ensure that they are to help transition between lectern versions - // checking for this can be removed in future versions of lectern (feb 2020) - const scripts = - typeof field.restrictions.script === 'string' ? [field.restrictions.script] : field.restrictions.script; - - let result: { - valid: boolean; - message: string; - } = { - valid: false, - message: '', - }; - - for (const scriptString of scripts) { - const script = getScript(scriptString); - const valFunc = script.runInContext(ctx); - if (!valFunc) throw new Error('Invalid script'); - result = valFunc(args); - /* Return the first script that's invalid. Otherwise result will be valid with message: 'ok'*/ - if (!result.valid) break; - } - - return result; - } catch (err) { - console.error( - `failed running validation script ${field.name} for record: ${JSON.stringify(record)}. Error message: ${err}`, - ); - return { - valid: false, - message: 'failed to run script validation, check script and the input', - }; - } - }; - - const getScript = (scriptString: string) => { - const script = new vm.Script(scriptString); - return script; - }; - - const buildError = ( - errorType: SchemaValidationErrorTypes, - fieldName: string, - index: number, - info: object = {}, - ): SchemaValidationError => { - const errorData = { errorType, fieldName, index, info }; - return { ...errorData, message: schemaErrorMessage(errorType, errorData) }; - }; -} -function validateRecordsSet(schemaDef: SchemaDefinition, processedRecords: TypedDataRecord[]) { - const validationErrors = validation - .runDatasetValidationPipeline(processedRecords, schemaDef, [ - validation.validateUnique, - validation.validateUniqueKey, - ]) - .filter(notEmpty); - return validationErrors; -} diff --git a/packages/client/src/schema-worker.js b/packages/client/src/schema-worker.js deleted file mode 100644 index cdb44c0..0000000 --- a/packages/client/src/schema-worker.js +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2020 The Ontario Institute for Cancer Research. All rights reserved - * - * This program and the accompanying materials are made available under the terms of - * the GNU Affero General Public License v3.0. You should have received a copy of the - * GNU Affero General Public License along with this program. - * If not, see . - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT - * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER - * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -const workerThreads = require('worker_threads'); -const parentPort = workerThreads.parentPort; -// ts node is required to import ts modules like "schema-functions.ts" in this case since -// worker threads run in their own V8 instance -const tsNode = require('ts-node'); - -/** - * when we run the app with node directly like this: - * node -r ts-node/register server.ts - * we will have a registered ts node instance, registering another one will result in wierd behaviour - * however when we run with mocha: - * mocha -r ts-node/register .ts - * (same applies if we run with ts node directly: ts-node server.ts) - * the worker thread won't have an instance of ts node transpiler - * unlike node for some reason which seem to attach the isntance to the worker thread process. - * - * so we had to add this work around to avoid double registry in different run modes. - * root cause of why mocha acts different than node is not found yet. - */ -if (!process[tsNode.REGISTER_INSTANCE]) { - tsNode.register(); -} -const service = require('./schema-functions'); - -function processProxy(args) { - return service.process(args.dictionary, args.schemaName, args.record, args.index); -} - -parentPort.on('message', (args) => { - const result = processProxy(args); - parentPort.postMessage(result); -}); diff --git a/packages/client/src/types/dataRecords.ts b/packages/client/src/types/dataRecords.ts new file mode 100644 index 0000000..43adfda --- /dev/null +++ b/packages/client/src/types/dataRecords.ts @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * Represents a data record as taken from an input file. All values are the original strings and have not been validated into + * numbers/bools or split into arrays. + */ +export type UnprocessedDataRecord = { + [k: string]: string | string[]; +}; + +/** + * The available data types for a field in a Lectern Schema. + */ +export type DataRecordValue = string | string[] | number | number[] | boolean | boolean[] | undefined; + +/** + * Represents a data record after processing, with the data checked to be a valid type for a Lectern schema. + * The type of data should match the expected type for the given field. + */ +export type DataRecord = { + [key: string]: DataRecordValue; +}; diff --git a/packages/client/src/types/index.ts b/packages/client/src/types/index.ts new file mode 100644 index 0000000..f35338d --- /dev/null +++ b/packages/client/src/types/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +export * from './dataRecords'; diff --git a/packages/client/src/utils.ts b/packages/client/src/utils.ts index 391b266..bb4d2aa 100644 --- a/packages/client/src/utils.ts +++ b/packages/client/src/utils.ts @@ -17,40 +17,8 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import fs from 'fs'; import deepFreeze from 'deep-freeze'; import _ from 'lodash'; -import { isArray } from 'util'; - -const fsPromises = fs.promises; - -export namespace Checks { - export const checkNotNull = (argName: string, arg: any) => { - if (!arg) { - throw new Errors.InvalidArgument(argName); - } - }; -} - -export namespace Errors { - export class InvalidArgument extends Error { - constructor(argumentName: string) { - super(`Invalid argument : ${argumentName}`); - } - } - - export class NotFound extends Error { - constructor(msg: string) { - super(msg); - } - } - - export class StateConflict extends Error { - constructor(msg: string) { - super(msg); - } - } -} // type gaurd to filter out undefined and null // https://stackoverflow.com/questions/43118692/typescript-filter-out-nulls-from-an-array @@ -108,7 +76,9 @@ export const isAbsent = (value: string | number | boolean | undefined): value is return !isNotAbsent(value); }; -export const isNotAbsent = (value: string | number | boolean | undefined): value is string | number | boolean => { +export const isNotAbsent = ( + value: string | number | boolean | undefined | null, +): value is string | number | boolean => { return value !== null && value !== undefined; }; @@ -131,7 +101,7 @@ export function toString(obj: any) { } export function isValueEqual(value: any, other: any) { - if (isArray(value) && isArray(other)) { + if (Array.isArray(value) && Array.isArray(other)) { return _.difference(value, other).length === 0; // check equal, ignore order } diff --git a/packages/client/src/validation/dataRecordValidation/fieldNamesValidation.ts b/packages/client/src/validation/dataRecordValidation/fieldNamesValidation.ts new file mode 100644 index 0000000..2df6bbb --- /dev/null +++ b/packages/client/src/validation/dataRecordValidation/fieldNamesValidation.ts @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { + BaseSchemaValidationError, + SchemaValidationErrorTypes, + UnrecognizedFieldValidationError, +} from '../types/validationErrorTypes'; +import { UnprocessedRecordValidationFunction } from '../types/validationFunctionTypes'; + +export const validateFieldNames: UnprocessedRecordValidationFunction = ( + record, + index, + fields, +): UnrecognizedFieldValidationError[] => { + const expectedFields = new Set(fields.map((field) => field.name)); + return Object.keys(record) + .filter((fieldName) => !expectedFields.has(fieldName)) + .map((fieldName) => buildUnrecognizedFieldError({ fieldName, index })); +}; + +export const buildUnrecognizedFieldError = (errorData: BaseSchemaValidationError): UnrecognizedFieldValidationError => { + const message = `${errorData.fieldName} is not an allowed field for this schema.`; + const info = {}; + + return { + ...errorData, + errorType: SchemaValidationErrorTypes.UNRECOGNIZED_FIELD, + info, + message, + }; +}; diff --git a/packages/client/src/validation/dataRecordValidation/valueTypeValidation.ts b/packages/client/src/validation/dataRecordValidation/valueTypeValidation.ts new file mode 100644 index 0000000..b42304d --- /dev/null +++ b/packages/client/src/validation/dataRecordValidation/valueTypeValidation.ts @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { SchemaFieldValueType } from 'dictionary'; + +import { convertToArray, isEmpty, isEmptyString, isStringArray, notEmpty } from '../../utils'; +import { + INVALID_VALUE_ERROR_MESSAGE, + SchemaValidationErrorTypes, + type BaseSchemaValidationError, + type ValueTypeValidationError, +} from '../types/validationErrorTypes'; +import type { UnprocessedRecordValidationFunction, ValidationFunction } from '../types/validationFunctionTypes'; + +/** + * Test the values provided to every field in a DataRecord to find any non-array fields that + * have array values. + * @param record Data Record with original string values provided in data file + * @param index + * @param schemaFields + * @returns + */ +export const validateNonArrayFields: UnprocessedRecordValidationFunction = ( + record, + index, + schemaFields, +): ValueTypeValidationError[] => { + return schemaFields + .map((field) => { + const value = record[field.name]; + if (!field.isArray && isStringArray(value)) { + return buildFieldValueTypeError({ fieldName: field.name, index }, { value }); + } + return undefined; + }) + .filter(notEmpty); +}; + +/** + * Test the values provided to every field in a DataRecord to find any values that cannot be + * converted to the required type defined in the field schema + * @param record Data Record with original string values provided in data file + * @param index + * @param schemaFields + * @returns + */ +export const validateValueTypes: UnprocessedRecordValidationFunction = ( + record, + index, + schemaFields, +): ValueTypeValidationError[] => { + return schemaFields + .map((field) => { + if (isEmpty(record[field.name])) { + return undefined; + } + + const recordFieldValues = convertToArray(record[field.name]); // put all values into array + const invalidValues = recordFieldValues.filter((v) => v !== undefined && isInvalidFieldType(field.valueType, v)); + const info = { value: invalidValues }; + + if (invalidValues.length !== 0) { + return buildFieldValueTypeError({ fieldName: field.name, index }, info); + } + return undefined; + }) + .filter(notEmpty); +}; + +/** + * Check a value is valid for a given schema value type. + * @param valueType + * @param value + * @returns + */ +const isInvalidFieldType = (valueType: SchemaFieldValueType, value: string) => { + // optional field if the value is absent at this point + if (isEmptyString(value)) return false; + switch (valueType) { + case SchemaFieldValueType.Values.string: + return false; + case SchemaFieldValueType.Values.integer: + return !Number.isSafeInteger(Number(value)); + case SchemaFieldValueType.Values.number: + return isNaN(Number(value)); + case SchemaFieldValueType.Values.boolean: + return !(value.toLowerCase() === 'true' || value.toLowerCase() === 'false'); + } +}; + +const buildFieldValueTypeError = ( + errorData: BaseSchemaValidationError, + info: ValueTypeValidationError['info'], +): ValueTypeValidationError => { + const message = INVALID_VALUE_ERROR_MESSAGE; + return { + ...errorData, + errorType: SchemaValidationErrorTypes.INVALID_FIELD_VALUE_TYPE, + info, + message, + }; +}; diff --git a/packages/client/src/validation/fieldRestrictions/codeListValidation.ts b/packages/client/src/validation/fieldRestrictions/codeListValidation.ts new file mode 100644 index 0000000..50d9a56 --- /dev/null +++ b/packages/client/src/validation/fieldRestrictions/codeListValidation.ts @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { convertToArray, isAbsent, isEmptyString, notEmpty } from '../../utils'; +import { + BaseSchemaValidationError, + EnumValueValidationError, + INVALID_VALUE_ERROR_MESSAGE, + SchemaValidationErrorTypes, +} from '../types/validationErrorTypes'; +import { ValidationFunction } from '../types/validationFunctionTypes'; + +/** + * Check all values of a DataRecord pass codeList restrictions in their schema. + * @param rec + * @param index + * @param fields + * @returns + */ +export const validateCodeList: ValidationFunction = (rec, index, fields): EnumValueValidationError[] => { + return fields + .map((field) => { + if (field.restrictions && 'codeList' in field.restrictions && field.restrictions.codeList !== undefined) { + const codeList = field.restrictions.codeList; + if (!Array.isArray(codeList)) { + // codeList restriction is a string, not array. This happens when the references have not been replaced. + // We cannot proceed without the final array so we will return undefined. + return undefined; + } + + // put all values into array to standardize validation for array and non array fields + const recordFieldValues = convertToArray(rec[field.name]); + const invalidValues = recordFieldValues.filter((val) => isInvalidEnumValue(codeList, val)); + + if (invalidValues.length !== 0) { + return buildCodeListError({ fieldName: field.name, index }, { value: invalidValues }); + } + } + return undefined; + }) + .filter(notEmpty); +}; + +const buildCodeListError = ( + errorData: BaseSchemaValidationError, + info: EnumValueValidationError['info'], +): EnumValueValidationError => { + const message = INVALID_VALUE_ERROR_MESSAGE; + + return { + ...errorData, + errorType: SchemaValidationErrorTypes.INVALID_ENUM_VALUE, + info, + message, + }; +}; + +const isInvalidEnumValue = (codeList: string[] | number[], value: string | boolean | number | undefined) => { + // only validate existing values + if (isAbsent(value) || (typeof value === 'string' && isEmptyString(value))) { + return false; + } + + return !codeList.some((allowedValue) => allowedValue === value); +}; diff --git a/packages/client/src/validation/fieldRestrictions/rangeValidation.ts b/packages/client/src/validation/fieldRestrictions/rangeValidation.ts new file mode 100644 index 0000000..83d4408 --- /dev/null +++ b/packages/client/src/validation/fieldRestrictions/rangeValidation.ts @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { RestrictionRange } from 'dictionary'; +import { convertToArray, isEmpty, isNumberArray, notEmpty } from '../../utils'; +import { rangeToSymbol } from '../utils/rangeToSymbol'; +import { + BaseSchemaValidationError, + RangeValidationError, + SchemaValidationErrorTypes, +} from '../types/validationErrorTypes'; +import { ValidationFunction } from '../types/validationFunctionTypes'; + +/** + * Check all values of a DataRecord pass range restrictions in their schema. + * @param record + * @param index + * @param schemaFields + * @returns + */ +export const validateRange: ValidationFunction = (record, index, schemaFields): RangeValidationError[] => { + return schemaFields + .map((field) => { + const recordFieldValues = convertToArray(record[field.name]); + if (!isNumberArray(recordFieldValues)) { + return undefined; + } + + const range = field.restrictions && 'range' in field.restrictions ? field.restrictions.range : undefined; + if (isEmpty(range)) { + return undefined; + } + + const invalidValues = recordFieldValues.filter((value) => isOutOfRange(range, value)); + if (invalidValues.length !== 0) { + const info = { value: invalidValues, ...range }; + return buildRangeError({ fieldName: field.name, index }, info); + } + return undefined; + }) + .filter(notEmpty); +}; + +const buildRangeError = ( + errorData: BaseSchemaValidationError, + info: RangeValidationError['info'], +): RangeValidationError => { + const message = `Value is out of permissible range, it must be ${rangeToSymbol(info)}.`; + + return { + ...errorData, + errorType: SchemaValidationErrorTypes.INVALID_BY_RANGE, + info, + message, + }; +}; + +const isOutOfRange = (range: RestrictionRange, value: number | undefined) => { + if (value === undefined) return false; + const invalidRange = + // less than the min if defined ? + (range.min !== undefined && value < range.min) || + (range.exclusiveMin !== undefined && value <= range.exclusiveMin) || + // bigger than max if defined ? + (range.max !== undefined && value > range.max) || + (range.exclusiveMax !== undefined && value >= range.exclusiveMax); + return invalidRange; +}; diff --git a/packages/client/src/validation/fieldRestrictions/regexValidation.ts b/packages/client/src/validation/fieldRestrictions/regexValidation.ts new file mode 100644 index 0000000..a501bdd --- /dev/null +++ b/packages/client/src/validation/fieldRestrictions/regexValidation.ts @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +import { SchemaFieldValueType } from 'dictionary'; +import { convertToArray, isEmpty, isEmptyString, isStringArray, notEmpty } from '../../utils'; +import { + BaseSchemaValidationError, + RegexValidationError, + SchemaValidationErrorTypes, +} from '../types/validationErrorTypes'; +import { ValidationFunction } from '../types/validationFunctionTypes'; + +/** + * Check all values of a DataRecord pass regex restrictions in their schema. + * @param record + * @param index + * @param fields + * @returns + */ +export const validateRegex: ValidationFunction = (record, index, fields): RegexValidationError[] => { + return fields + .map((field) => { + if ( + field.valueType === SchemaFieldValueType.Values.string && + field.restrictions && + !isEmpty(field.restrictions.regex) + ) { + const regex = field.restrictions.regex; + const recordFieldValues = convertToArray(record[field.name]); + if (!isStringArray(recordFieldValues)) { + // This field value should be string or string array, we will skip validation if the type is wrong. + return undefined; + } + + const invalidValues = recordFieldValues.filter((v) => isInvalidRegexValue(regex, v)); + if (invalidValues.length !== 0) { + const examples = typeof field.meta?.examples === 'string' ? field.meta.examples : undefined; + + return buildRegexError({ fieldName: field.name, index }, { value: invalidValues, regex, examples }); + } + } + + // Field does not have regex validation + return undefined; + }) + .filter(notEmpty); +}; + +const isInvalidRegexValue = (regex: string, value: string) => { + // optional field if the value is absent at this point + if (isEmptyString(value)) return false; + const regexPattern = new RegExp(regex); + return !regexPattern.test(value); +}; + +const buildRegexError = ( + errorData: BaseSchemaValidationError, + info: RegexValidationError['info'], +): RegexValidationError => { + const examplesMessage = info.examples ? ` Examples: ${info.examples}` : ''; + const message = `The value is not a permissible for this field, it must meet the regular expression: "${info.regex}".${examplesMessage}`; + + return { + ...errorData, + errorType: SchemaValidationErrorTypes.INVALID_BY_REGEX, + info, + message, + }; +}; diff --git a/packages/client/src/validation/fieldRestrictions/requiredValidation.ts b/packages/client/src/validation/fieldRestrictions/requiredValidation.ts new file mode 100644 index 0000000..82bbc48 --- /dev/null +++ b/packages/client/src/validation/fieldRestrictions/requiredValidation.ts @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { SchemaField } from 'dictionary'; +import { convertToArray, isEmpty, notEmpty } from '../../utils'; +import { + BaseSchemaValidationError, + MissingRequiredFieldValidationError, + SchemaValidationErrorTypes, +} from '../types/validationErrorTypes'; +import { ValidationFunction } from '../types/validationFunctionTypes'; +import { DataRecord } from '../../types/dataRecords'; + +/** + * Check all values of a DataRecord pass required restrictions in their schema. + * @param record + * @param index + * @param fields + * @returns + */ +export const validateRequiredFields: ValidationFunction = ( + record, + index, + fields, +): MissingRequiredFieldValidationError[] => { + return fields + .map((field) => { + if (isRequiredMissing(field, record)) { + return buildRequiredError({ fieldName: field.name, index }, {}); + } + return undefined; + }) + .filter(notEmpty); +}; + +const buildRequiredError = ( + errorData: BaseSchemaValidationError, + info: MissingRequiredFieldValidationError['info'], +): MissingRequiredFieldValidationError => { + const message = `${errorData.fieldName} is a required field.`; + + return { + ...errorData, + errorType: SchemaValidationErrorTypes.MISSING_REQUIRED_FIELD, + info, + message, + }; +}; + +const isRequiredMissing = (field: SchemaField, record: DataRecord) => { + const isRequired = field.restrictions && field.restrictions.required; + if (!isRequired) return false; + + const recordFieldValues = convertToArray(record[field.name]); + return recordFieldValues.every(isEmpty); +}; diff --git a/packages/client/src/validation/fieldRestrictions/scriptValidation.ts b/packages/client/src/validation/fieldRestrictions/scriptValidation.ts new file mode 100644 index 0000000..5f6ba0c --- /dev/null +++ b/packages/client/src/validation/fieldRestrictions/scriptValidation.ts @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { SchemaField } from 'dictionary'; +import vm from 'vm'; +import { DataRecord } from '../../types/dataRecords'; +import { notEmpty } from '../../utils'; +import { + BaseSchemaValidationError, + SchemaValidationErrorTypes, + ScriptValidationError, +} from '../types/validationErrorTypes'; +import { ValidationFunction } from '../types/validationFunctionTypes'; + +const ctx = vm.createContext(); + +/** + * Check all values of a DataRecord pass all script restrictions in their schema. + * This will run all script restrictions from the provided inside a Node VM context. + * + * Running code in teh VM context will protect the global Node data context from interactions + * with the schema script, either being read or written. + * + * @param record + * @param index + * @param fields + * @returns + */ +export const validateScript: ValidationFunction = (record, index, fields) => { + return fields + .map((field) => { + if (field.restrictions && field.restrictions.script) { + const scriptResult = validateWithScript(field, record); + if (!scriptResult.valid) { + return buildScriptError( + { fieldName: field.name, index }, + { + message: scriptResult.message, + value: record[field.name], + }, + ); + } + } + return undefined; + }) + .filter(notEmpty); +}; + +const buildScriptError = ( + errorData: BaseSchemaValidationError, + info: ScriptValidationError['info'], +): ScriptValidationError => { + const message = info.message || `${errorData.fieldName} was invalid based on a script restriction.`; + + return { + ...errorData, + errorType: SchemaValidationErrorTypes.INVALID_BY_SCRIPT, + info, + message, + }; +}; + +const getScript = (scriptString: string) => { + const script = new vm.Script(scriptString); + return script; +}; + +const validateWithScript = ( + field: SchemaField, + record: DataRecord, +): { + valid: boolean; + message: string; +} => { + try { + const args = { + $row: record, + $field: record[field.name], + $name: field.name, + }; + + if (!field.restrictions || !field.restrictions.script) { + throw new Error('called validation by script without script provided'); + } + + // scripts should already be strings inside arrays, but ensure that they are to help transition between lectern versions + // checking for this can be removed in future versions of lectern (feb 2020) + const scripts = + typeof field.restrictions.script === 'string' ? [field.restrictions.script] : field.restrictions.script; + + let result: { + valid: boolean; + message: string; + } = { + valid: false, + message: '', + }; + + for (const scriptString of scripts) { + const script = getScript(scriptString); + const valFunc = script.runInContext(ctx); + if (!valFunc) throw new Error('Invalid script'); + result = valFunc(args); + /* Return the first script that's invalid. Otherwise result will be valid with message: 'ok'*/ + if (!result.valid) break; + } + + return result; + } catch (err) { + console.error( + `failed running validation script ${field.name} for record: ${JSON.stringify(record)}. Error message: ${err}`, + ); + return { + valid: false, + message: 'failed to run script validation, check script and the input', + }; + } +}; diff --git a/packages/client/src/validation/index.ts b/packages/client/src/validation/index.ts new file mode 100644 index 0000000..28f5820 --- /dev/null +++ b/packages/client/src/validation/index.ts @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +export * from './dataRecordValidation/fieldNamesValidation'; +export * from './dataRecordValidation/valueTypeValidation'; +export * from './fieldRestrictions/codeListValidation'; +export * from './fieldRestrictions/rangeValidation'; +export * from './fieldRestrictions/regexValidation'; +export * from './fieldRestrictions/requiredValidation'; +export * from './fieldRestrictions/scriptValidation'; +export * from './schemaRestrictions/foreignKeysValidation'; +export * from './schemaRestrictions/uniqueKeyValidation'; +export * from './schemaRestrictions/uniqueValidation'; +export * from './types'; +export * from './validationPipelines'; diff --git a/packages/client/src/validation/schemaRestrictions/foreignKeysValidation.ts b/packages/client/src/validation/schemaRestrictions/foreignKeysValidation.ts new file mode 100644 index 0000000..9687e99 --- /dev/null +++ b/packages/client/src/validation/schemaRestrictions/foreignKeysValidation.ts @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { differenceWith, isEqual } from 'lodash'; +import { DataRecord } from '../../types/dataRecords'; +import { ForeignKeyValidationError, SchemaValidationErrorTypes } from '../types/validationErrorTypes'; +import { CrossSchemaValidationFunction } from '../types/validationFunctionTypes'; +import { selectFieldsFromDataset } from '../utils/datasetUtils'; + +/** + * Validate all foreign key restrictions in a Schema + * @param schema + * @param data + * @returns + */ +export const validateForeignKeys: CrossSchemaValidationFunction = (schema, data): ForeignKeyValidationError[] => { + const errors: Array = []; + const foreignKeyDefinitions = schema?.restrictions?.foreignKey; + if (foreignKeyDefinitions) { + foreignKeyDefinitions.forEach((foreignKeyDefinition) => { + const localSchemaData = data[schema.name] || []; + const foreignSchemaData = data[foreignKeyDefinition.schema] || []; + + // A foreign key can have more than one field, in which case is a composite foreign key. + const localFields = foreignKeyDefinition.mappings.map((x) => x.local); + const foreignFields = foreignKeyDefinition.mappings.map((x) => x.foreign); + + const fieldsMappings = new Map(foreignKeyDefinition.mappings.map((x) => [x.foreign, x.local])); + + // Select the keys of the datasets to compare. The keys are records to support the scenario where the fk is composite. + const localValues: [number, DataRecord][] = selectFieldsFromDataset(localSchemaData, localFields); + const foreignValues: [number, DataRecord][] = selectFieldsFromDataset(foreignSchemaData, foreignFields); + + // This artificial record in foreignValues allows null references in localValues to be valid. + const emptyRow: Record = {}; + foreignFields.forEach((field) => (emptyRow[field] = '')); + foreignValues.push([-1, emptyRow]); + + const missingForeignKeys = findMissingForeignKeys(localValues, foreignValues, fieldsMappings); + + missingForeignKeys.forEach((record) => { + const index = record[0]; + const info: ForeignKeyValidationError['info'] = { + value: record[1], + foreignSchema: foreignKeyDefinition.schema, + }; + const errorFieldName = localFields.join(', '); + errors.push({ + errorType: SchemaValidationErrorTypes.INVALID_BY_FOREIGN_KEY, + fieldName: errorFieldName, + index, + info, + message: getForeignKeyErrorMessage({ + fieldName: errorFieldName, + foreignSchema: foreignKeyDefinition.schema, + value: record[1], + }), + }); + }); + }); + } + return errors; +}; + +function getForeignKeyErrorMessage(errorData: { value: DataRecord; foreignSchema: string; fieldName: string }) { + const valueEntries = Object.entries(errorData.value); + const formattedKeyValues: string[] = valueEntries.map(([key, value]) => { + if (Array.isArray(value)) { + return `${key}: [${value.join(', ')}]`; + } else { + return `${key}: ${value}`; + } + }); + const valuesAsString = formattedKeyValues.join(', '); + const detail = `Key ${valuesAsString} is not present in schema ${errorData.foreignSchema}`; + const msg = `Record violates foreign key restriction defined for field(s) ${errorData.fieldName}. ${detail}.`; + return msg; +} + +/** + * Find missing foreign keys by calculating the difference between 2 dataset keys (similar to a set difference). + * Returns rows in `dataKeysA` which are not present in `dataKeysB`. + * @param datasetKeysA Keys of the dataset A. The returned value of this function is a subset of this array. + * @param datasetKeysB Keys of the dataset B. Elements to be substracted from `datasetKeysA`. + * @param fieldsMapping Mapping of the field names so the keys can be compared correctly. + */ +const findMissingForeignKeys = ( + datasetKeysA: [number, DataRecord][], + datasetKeysB: [number, DataRecord][], + fieldsMapping: Map, +): [number, DataRecord][] => { + const diff = differenceWith(datasetKeysA, datasetKeysB, (a, b) => + isEqual(a[1], renameProperties(b[1], fieldsMapping)), + ); + return diff; +}; + +/** + * Renames properties in a record using a mapping between current and new names. + * @param record The record whose properties should be renamed. + * @param fieldsMapping A mapping of current property names to new property names. + * @returns A new record with the properties' names changed according to the mapping. + */ +const renameProperties = (record: DataRecord, fieldsMapping: Map): DataRecord => { + const renamed: DataRecord = {}; + Object.entries(record).forEach(([propertyName, propertyValue]) => { + const newName = fieldsMapping.get(propertyName) ?? propertyName; + renamed[newName] = propertyValue; + }); + return renamed; +}; diff --git a/packages/client/src/validation/schemaRestrictions/uniqueKeyValidation.ts b/packages/client/src/validation/schemaRestrictions/uniqueKeyValidation.ts new file mode 100644 index 0000000..e841425 --- /dev/null +++ b/packages/client/src/validation/schemaRestrictions/uniqueKeyValidation.ts @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { + BaseSchemaValidationError, + SchemaValidationErrorTypes, + UniqueKeyValidationError, +} from '../types/validationErrorTypes'; +import { DatasetValidationFunction } from '../types/validationFunctionTypes'; +import { findDuplicateKeys, selectFieldsFromDataset } from '../utils/datasetUtils'; + +export const validateUniqueKey: DatasetValidationFunction = (dataset, schema): UniqueKeyValidationError[] => { + const errors: Array = []; + const uniqueKeyRestriction = schema?.restrictions?.uniqueKey; + if (uniqueKeyRestriction) { + const uniqueKeyFields: string[] = uniqueKeyRestriction; + const keysToValidate = selectFieldsFromDataset(dataset, uniqueKeyFields); + const duplicateKeys = findDuplicateKeys(keysToValidate); + + duplicateKeys.forEach(([index, record]) => { + const info = { uniqueKeyFields: uniqueKeyFields, value: record }; + errors.push(buildUniqueKeyError({ fieldName: uniqueKeyFields.join(', '), index }, info)); + }); + } + return errors; +}; + +const buildUniqueKeyError = ( + errorData: BaseSchemaValidationError, + info: UniqueKeyValidationError['info'], +): UniqueKeyValidationError => { + const uniqueKeyFields = info.uniqueKeyFields; + const record = info.value; + const formattedKeyValues = uniqueKeyFields.map((fieldName) => { + if (fieldName in record) { + const value = record[fieldName]; + if (Array.isArray(value)) { + return `${fieldName}: [${value.join(', ')}]`; + } else { + return `${fieldName}: ${value === '' ? 'null' : value}`; + } + } + }); + const valuesAsString = formattedKeyValues.join(', '); + const message = `UniqueKey field values "${valuesAsString}" must be unique.`; + + return { + ...errorData, + errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, + info, + message, + }; +}; diff --git a/packages/client/src/validation/schemaRestrictions/uniqueValidation.ts b/packages/client/src/validation/schemaRestrictions/uniqueValidation.ts new file mode 100644 index 0000000..5443d66 --- /dev/null +++ b/packages/client/src/validation/schemaRestrictions/uniqueValidation.ts @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { + BaseSchemaValidationError, + SchemaValidationErrorTypes, + UniqueValidationError, +} from '../types/validationErrorTypes'; +import { DatasetValidationFunction } from '../types/validationFunctionTypes'; +import { findDuplicateKeys, selectFieldsFromDataset } from '../utils/datasetUtils'; + +/** + * Validate all unique field restrictions in a schema. This will find all records that have duplicate + * values for fields that are restricted to being unique. + * @param data + * @param schema + * @returns + */ +export const validateUnique: DatasetValidationFunction = (data, schema): UniqueValidationError[] => { + const errors: Array = []; + schema.fields.forEach((field) => { + const unique = field.restrictions?.unique || undefined; + if (!unique) return undefined; + const keysToValidate = selectFieldsFromDataset(data, [field.name]); + const duplicateKeys = findDuplicateKeys(keysToValidate); + + duplicateKeys.forEach(([index, record]) => { + const info = { value: record[field.name] }; + errors.push(buildUniqueError({ fieldName: field.name, index }, info)); + }); + }); + return errors; +}; + +const buildUniqueError = ( + errorData: BaseSchemaValidationError, + info: UniqueValidationError['info'], +): UniqueValidationError => { + const message = `Values for column "${errorData.fieldName}" must be unique.`; + + return { + ...errorData, + errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE, + info, + message, + }; +}; diff --git a/packages/client/src/validation/types/index.ts b/packages/client/src/validation/types/index.ts new file mode 100644 index 0000000..ba34998 --- /dev/null +++ b/packages/client/src/validation/types/index.ts @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +export * from './validationErrorTypes'; +export * from './validationFunctionTypes'; diff --git a/packages/client/src/validation/types/validationErrorTypes.ts b/packages/client/src/validation/types/validationErrorTypes.ts new file mode 100644 index 0000000..33abed1 --- /dev/null +++ b/packages/client/src/validation/types/validationErrorTypes.ts @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import type { Values, Singular } from 'common'; +import type { DataRecord, DataRecordValue } from '../../types/dataRecords'; +import type { RestrictionRange } from 'dictionary'; + +/** + * Represents the common structure of a validation error without the custom content provided by specific error types. + * The `message` property is not included here as this allows the rest of the error content to be passed to a message building function + * that will produce the final typed SchemaValidationError + */ +export type BaseSchemaValidationError = { + index: number; + fieldName: string; +}; + +type GenericSchemaValidationError< + ErrorType extends SchemaValidationErrorType, + Info extends object, +> = BaseSchemaValidationError & { + errorType: ErrorType; + info: Info; + message: string; +}; + +// Common string for invalid value errors +export const INVALID_VALUE_ERROR_MESSAGE = 'The value is not permissible for this field.'; + +export const SchemaValidationErrorTypes = { + INVALID_FIELD_VALUE_TYPE: 'INVALID_FIELD_VALUE_TYPE', + INVALID_BY_REGEX: 'INVALID_BY_REGEX', + INVALID_BY_RANGE: 'INVALID_BY_RANGE', + INVALID_BY_SCRIPT: 'INVALID_BY_SCRIPT', + INVALID_ENUM_VALUE: 'INVALID_ENUM_VALUE', + INVALID_BY_UNIQUE: 'INVALID_BY_UNIQUE', + INVALID_BY_FOREIGN_KEY: 'INVALID_BY_FOREIGN_KEY', + INVALID_BY_UNIQUE_KEY: 'INVALID_BY_UNIQUE_KEY', + MISSING_REQUIRED_FIELD: 'MISSING_REQUIRED_FIELD', + UNRECOGNIZED_FIELD: 'UNRECOGNIZED_FIELD', +} as const; +export type SchemaValidationErrorType = Values; + +export type EnumValueValidationError = GenericSchemaValidationError< + typeof SchemaValidationErrorTypes.INVALID_ENUM_VALUE, + { value: DataRecordValue[] } +>; +export type ForeignKeyValidationError = GenericSchemaValidationError< + typeof SchemaValidationErrorTypes.INVALID_BY_FOREIGN_KEY, + { value: DataRecord; foreignSchema: string } +>; +export type MissingRequiredFieldValidationError = GenericSchemaValidationError< + typeof SchemaValidationErrorTypes.MISSING_REQUIRED_FIELD, + {} +>; +export type RangeValidationError = GenericSchemaValidationError< + typeof SchemaValidationErrorTypes.INVALID_BY_RANGE, + { value: number[] } & RestrictionRange +>; +export type RegexValidationError = GenericSchemaValidationError< + typeof SchemaValidationErrorTypes.INVALID_BY_REGEX, + { value: string[]; regex: string; examples?: string } +>; +export type ScriptValidationError = GenericSchemaValidationError< + typeof SchemaValidationErrorTypes.INVALID_BY_SCRIPT, + { message: string; value: DataRecordValue } +>; +export type UniqueValidationError = GenericSchemaValidationError< + typeof SchemaValidationErrorTypes.INVALID_BY_UNIQUE, + { value: DataRecordValue } +>; +export type UniqueKeyValidationError = GenericSchemaValidationError< + typeof SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, + { uniqueKeyFields: string[]; value: DataRecord } +>; +export type UnrecognizedFieldValidationError = GenericSchemaValidationError< + typeof SchemaValidationErrorTypes.UNRECOGNIZED_FIELD, + {} +>; +export type ValueTypeValidationError = GenericSchemaValidationError< + typeof SchemaValidationErrorTypes.INVALID_FIELD_VALUE_TYPE, + { value: Singular[] } +>; +export type SchemaValidationError = + | EnumValueValidationError + | ValueTypeValidationError + | ForeignKeyValidationError + | MissingRequiredFieldValidationError + | RangeValidationError + | RegexValidationError + | ScriptValidationError + | UniqueValidationError + | UniqueKeyValidationError + | UnrecognizedFieldValidationError; diff --git a/packages/client/src/validation/types/validationFunctionTypes.ts b/packages/client/src/validation/types/validationFunctionTypes.ts new file mode 100644 index 0000000..2709c9c --- /dev/null +++ b/packages/client/src/validation/types/validationFunctionTypes.ts @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { Schema, SchemaField } from 'dictionary'; +import { DataRecord, UnprocessedDataRecord } from '../../types/dataRecords'; +import { SchemaValidationError } from './validationErrorTypes'; + +// these validation functions run AFTER the record has been converted to the correct types from raw strings +export type UnprocessedRecordValidationFunction = ( + record: UnprocessedDataRecord, + index: number, + schemaFields: Schema['fields'], +) => Array; + +// these validation functions run BEFORE the record has been converted to the correct types from raw strings +export type ValidationFunction = ( + record: DataRecord, + index: number, + schemaFields: Schema['fields'], +) => Array; + +// these validation functions run AFTER the records has been converted to the correct types from raw strings, and apply to a dataset instead of +// individual records +export type DatasetValidationFunction = (data: Array, schema: Schema) => Array; + +export type CrossSchemaValidationFunction = ( + schema: Schema, + data: Record, +) => Array; diff --git a/packages/client/src/records-operations.ts b/packages/client/src/validation/utils/datasetUtils.ts similarity index 53% rename from packages/client/src/records-operations.ts rename to packages/client/src/validation/utils/datasetUtils.ts index 7b964f3..e07f4c3 100644 --- a/packages/client/src/records-operations.ts +++ b/packages/client/src/validation/utils/datasetUtils.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 The Ontario Institute for Cancer Research. All rights reserved + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved * * This program and the accompanying materials are made available under the terms of * the GNU Affero General Public License v3.0. You should have received a copy of the @@ -17,25 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { differenceWith, isEqual } from 'lodash'; - -/** - * Renames properties in a record using a mapping between current and new names. - * @param record The record whose properties should be renamed. - * @param fieldsMapping A mapping of current property names to new property names. - * @returns A new record with the properties' names changed according to the mapping. - */ -const renameProperties = ( - record: Record, - fieldsMapping: Map, -): Record => { - const renamed: Record = {}; - Object.entries(record).forEach(([propertyName, propertyValue]) => { - const newName = fieldsMapping.get(propertyName) ?? propertyName; - renamed[newName] = propertyValue; - }); - return renamed; -}; +import { DataRecord } from '../../types/dataRecords'; /** * Returns a string representation of a record. The record is sorted by its properties so @@ -44,43 +26,23 @@ const renameProperties = ( * @param record Record to be processed. * @returns String representation of the record sorted by its properties. */ -const getSortedRecordKey = (record: Record): string => { +const getSortedRecordKey = (record: DataRecord): string => { const sortedKeys = Object.keys(record).sort(); - const sortedRecord: Record = {}; + const sortedRecord: DataRecord = {}; for (const key of sortedKeys) { sortedRecord[key] = record[key]; } return JSON.stringify(sortedRecord); }; -/** - * Find missing foreign keys by calculating the difference between 2 dataset keys (similar to a set difference). - * Returns rows in `dataKeysA` which are not present in `dataKeysB`. - * @param datasetKeysA Keys of the dataset A. The returned value of this function is a subset of this array. - * @param datasetKeysB Keys of the dataset B. Elements to be substracted from `datasetKeysA`. - * @param fieldsMapping Mapping of the field names so the keys can be compared correctly. - */ -export const findMissingForeignKeys = ( - datasetKeysA: [number, Record][], - datasetKeysB: [number, Record][], - fieldsMapping: Map, -): [number, Record][] => { - const diff = differenceWith(datasetKeysA, datasetKeysB, (a, b) => - isEqual(a[1], renameProperties(b[1], fieldsMapping)), - ); - return diff; -}; - /** * Find duplicate keys in a dataset. * @param datasetKeys Array with the keys to evaluate. * @returns An Array with all the values that appear more than once in the dataset. */ -export const findDuplicateKeys = ( - datasetKeys: [number, Record][], -): [number, Record][] => { - const duplicateKeys: [number, Record][] = []; - const recordKeysMap: Map<[number, Record], string> = new Map(); +export const findDuplicateKeys = (datasetKeys: [number, DataRecord][]): [number, DataRecord][] => { + const duplicateKeys: [number, DataRecord][] = []; + const recordKeysMap: Map<[number, DataRecord], string> = new Map(); const keyCount: Map = new Map(); // Calculate a key per record, which is a string representation that allows to compare records even if their properties @@ -101,3 +63,19 @@ export const findDuplicateKeys = ( }); return duplicateKeys; }; + +/** + * A "select" function that retrieves specific fields from the dataset as a record, as well as the numeric position of each row in the dataset. + * @param dataset Dataset to select fields from. + * @param fields Array with names of the fields to select. + * @returns An array of tuples tuple where the first element is the index of the row in the dataset, and the second value is the record with the + * selected values. + */ +export const selectFieldsFromDataset = (dataset: DataRecord[], fields: string[]): [number, DataRecord][] => + dataset.map((row, index) => { + const filteredRecord = fields.reduce((acc, field) => { + acc[field] = row[field] || ''; + return acc; + }, {}); + return [index, filteredRecord]; + }); diff --git a/packages/client/src/validation/utils/fieldTypeUtils.ts b/packages/client/src/validation/utils/fieldTypeUtils.ts new file mode 100644 index 0000000..405d3d8 --- /dev/null +++ b/packages/client/src/validation/utils/fieldTypeUtils.ts @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { SchemaFieldValueType } from 'dictionary'; + +import { isEmptyString } from '../../utils'; + +/** + * Check a value is valid for a given schema value type. + * @param valueType + * @param value + * @returns + */ +export const isInvalidFieldType = (valueType: SchemaFieldValueType, value: string) => { + // optional field if the value is absent at this point + if (isEmptyString(value)) return false; + switch (valueType) { + case SchemaFieldValueType.Values.string: + return false; + case SchemaFieldValueType.Values.integer: + return !Number.isSafeInteger(Number(value)); + case SchemaFieldValueType.Values.number: + return isNaN(Number(value)); + case SchemaFieldValueType.Values.boolean: + return !(value.toLowerCase() === 'true' || value.toLowerCase() === 'false'); + } +}; diff --git a/packages/client/src/validation/utils/rangeToSymbol.ts b/packages/client/src/validation/utils/rangeToSymbol.ts new file mode 100644 index 0000000..1616ccd --- /dev/null +++ b/packages/client/src/validation/utils/rangeToSymbol.ts @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { RestrictionRange } from 'dictionary'; + +export const rangeToSymbol = (range: RestrictionRange): string => { + let minString = ''; + let maxString = ''; + + const hasBothRange = + (range.min !== undefined || range.exclusiveMin !== undefined) && + (range.max !== undefined || range.exclusiveMax !== undefined); + + if (range.min !== undefined) { + minString = `>= ${range.min}`; + } + + if (range.exclusiveMin !== undefined) { + minString = `> ${range.exclusiveMin}`; + } + + if (range.max !== undefined) { + maxString = `<= ${range.max}`; + } + + if (range.exclusiveMax !== undefined) { + maxString = `< ${range.exclusiveMax}`; + } + + return hasBothRange ? `${minString} and ${maxString}` : `${minString}${maxString}`; +}; diff --git a/packages/client/src/validation/validationPipelines.ts b/packages/client/src/validation/validationPipelines.ts new file mode 100644 index 0000000..f11a5fc --- /dev/null +++ b/packages/client/src/validation/validationPipelines.ts @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { Schema, SchemaField } from 'dictionary'; + +import { DataRecord, UnprocessedDataRecord } from '../types/dataRecords'; +import { SchemaValidationError } from './types/validationErrorTypes'; +import { + CrossSchemaValidationFunction, + DatasetValidationFunction, + UnprocessedRecordValidationFunction, + ValidationFunction, +} from './types/validationFunctionTypes'; + +export const runUnprocessedRecordValidationPipeline = ( + record: UnprocessedDataRecord, + index: number, + fields: ReadonlyArray, + validationFunctions: Array, +) => { + let result: Array = []; + for (const validationFunction of validationFunctions) { + result = result.concat(validationFunction(record, index, getValidFields(result, fields))); + } + return result; +}; + +export const runRecordValidationPipeline = ( + record: DataRecord, + index: number, + fields: ReadonlyArray, + validationFunctions: Array, +) => { + let result: Array = []; + for (const validationFunction of validationFunctions) { + result = result.concat(validationFunction(record, index, getValidFields(result, fields))); + } + return result; +}; + +export const runDatasetValidationPipeline = ( + data: DataRecord[], + schema: Schema, + validationFunctions: Array, +) => validationFunctions.flatMap((validationFunction) => validationFunction(data, schema)); + +export const runCrossSchemaValidationPipeline = ( + schema: Schema, + data: Record, + validationFunctions: Array, +) => { + let result: Array = []; + for (const validationFunction of validationFunctions) { + result = result.concat(validationFunction(schema, data)); + } + return result; +}; + +const getValidFields = (errs: ReadonlyArray, fields: ReadonlyArray) => { + return fields.filter((field) => { + return !errs.find((e) => e.fieldName === field.name); + }); +}; diff --git a/packages/client/test/change-analyzer.spec.ts b/packages/client/test/changeAnalyzer.spec.ts similarity index 77% rename from packages/client/test/change-analyzer.spec.ts rename to packages/client/test/changeAnalyzer.spec.ts index b0fb319..c2bbff0 100644 --- a/packages/client/test/change-analyzer.spec.ts +++ b/packages/client/test/changeAnalyzer.spec.ts @@ -18,23 +18,13 @@ */ import chai from 'chai'; -import * as analyzer from '../src/change-analyzer'; -import { SchemasDictionaryDiffs, FieldDiff, ChangeAnalysis } from '../src/schema-entities'; -import _ from 'lodash'; +import { DiffUtils } from 'dictionary'; +import { analyzer } from '../src'; +import { ChangeAnalysis } from '../src/changeAnalysis'; +import diffResponse from './fixtures/diffResponse'; chai.should(); -const diffResponse: any = require('./schema-diff.json'); -const schemaDiff: SchemasDictionaryDiffs = {}; -for (const entry of diffResponse) { - const fieldName = entry[0] as string; - if (entry[1]) { - const fieldDiff: FieldDiff = { - before: entry[1].left, - after: entry[1].right, - diff: entry[1].diff, - }; - schemaDiff[fieldName] = fieldDiff; - } -} + +const diffFixture = DiffUtils.diffArrayToMap(diffResponse); const expectedResult: ChangeAnalysis = { fields: { @@ -42,12 +32,6 @@ const expectedResult: ChangeAnalysis = { renamedFields: [], deletedFields: ['primary_diagnosis.menopause_status'], }, - metaChanges: { - core: { - changedToCore: [], - changedFromCore: [], - }, - }, restrictionsChanges: { codeList: { created: [], @@ -91,16 +75,6 @@ const expectedResult: ChangeAnalysis = { created: [], deleted: [], }, - script: { - updated: [], - created: [ - { - field: 'donor.survival_time', - definition: ' $field / 2 == 0 ', - }, - ], - deleted: [], - }, range: { updated: [ { @@ -121,14 +95,19 @@ const expectedResult: ChangeAnalysis = { ], deleted: [], }, + script: { + created: [], + deleted: [], + updated: [], + }, }, isArrayDesignationChanges: ['primary_diagnosis.presenting_symptoms'], valueTypeChanges: ['sample_registration.program_id'], }; -describe('change-analyzer', () => { +describe('changeAnalyzer', () => { it('categorize changes correctly', () => { - const result = analyzer.analyzeChanges(schemaDiff); + const result = analyzer.analyzeChanges(diffFixture); result.should.deep.eq(expectedResult); }); }); diff --git a/packages/client/test/fixtures/diffResponse.ts b/packages/client/test/fixtures/diffResponse.ts new file mode 100644 index 0000000..cf16490 --- /dev/null +++ b/packages/client/test/fixtures/diffResponse.ts @@ -0,0 +1,317 @@ +import { DictionaryDiffArray } from 'dictionary'; + +const diffResponse = [ + [ + 'donor.submitter_donor_id', + { + left: { + description: 'Unique identifier of the donor, assigned by the data provider.', + name: 'submitter_donor_id', + restrictions: { + required: true, + regex: '[A-Za-z0-9\\-\\._]{1,64}', + }, + valueType: 'string', + }, + right: { + description: 'Unique identifier of the donor, assigned by the data provider.', + name: 'submitter_donor_id', + restrictions: { + required: true, + regex: '[A-Za-z0-9\\-\\._]{3,64}', + }, + valueType: 'string', + }, + diff: { + restrictions: { + regex: { + type: 'updated', + data: '[A-Za-z0-9\\-\\._]{3,64}', + }, + }, + }, + }, + ], + [ + 'donor.vital_status', + { + left: { + description: 'Donors last known state of living or deceased.', + name: 'vital_status', + restrictions: { + codeList: ['Alive', 'Deceased', 'Not reported', 'Unknown'], + required: true, + }, + valueType: 'string', + }, + right: { + description: 'Donors last known state of living or deceased.', + name: 'vital_status', + restrictions: { + regex: '[A-Z]{3,100}', + required: true, + }, + valueType: 'string', + }, + diff: { + restrictions: { + codeList: { + type: 'deleted', + data: ['Alive', 'Deceased', 'Not reported', 'Unknown'], + }, + regex: { + type: 'created', + data: '[A-Z]{3,100}', + }, + }, + }, + }, + ], + [ + 'donor.cause_of_death', + { + left: { + description: "Description of the cause of a donor's death.", + name: 'cause_of_death', + restrictions: { + codeList: ['Died of cancer', 'Died of other reasons', 'Not reported', 'Unknown'], + }, + valueType: 'string', + }, + right: { + description: "Description of the cause of a donor's death.", + name: 'cause_of_death', + restrictions: { + codeList: ['Died of other reasons', 'Not reported', 'N/A'], + }, + valueType: 'string', + }, + diff: { + restrictions: { + codeList: { + type: 'updated', + data: { + added: ['N/A'], + deleted: ['Died of cancer', 'Unknown'], + }, + }, + }, + }, + }, + ], + [ + 'donor.survival_time', + { + left: { + description: 'Interval of how long the donor has survived since primary diagnosis, in days.', + meta: { + units: 'days', + }, + name: 'survival_time', + valueType: 'integer', + }, + right: { + description: 'Interval of how long the donor has survived since primary diagnosis, in days.', + meta: { + units: 'days', + }, + name: 'survival_time', + valueType: 'integer', + restrictions: { + range: { + min: 0, + max: 200000, + }, + }, + }, + diff: { + restrictions: { + type: 'created', + data: { + range: { + min: 0, + max: 200000, + }, + }, + }, + }, + }, + ], + [ + 'primary_diagnosis.cancer_type_code', + { + left: { + name: 'cancer_type_code', + valueType: 'string', + description: + 'The code to represent the cancer type using the WHO ICD-10 code (https://icd.who.int/browse10/2016/en#/) classification.', + restrictions: { + required: true, + regex: '[A-Z]{1}[0-9]{2}.[0-9]{0,3}[A-Z]{0,1}$', + }, + }, + right: { + name: 'cancer_type_code', + valueType: 'string', + description: + 'The code to represent the cancer type using the WHO ICD-10 code (https://icd.who.int/browse10/2016/en#/) classification.', + restrictions: { + required: true, + regex: '[A-Z]{1}[0-9]{2}.[0-9]{0,3}[A-Z]{2,3}$', + }, + }, + diff: { + restrictions: { + regex: { + type: 'updated', + data: '[A-Z]{1}[0-9]{2}.[0-9]{0,3}[A-Z]{2,3}$', + }, + }, + }, + }, + ], + [ + 'primary_diagnosis.menopause_status', + { + left: { + name: 'menopause_status', + description: 'Indicate the menopause status of the patient at the time of primary diagnosis.', + valueType: 'string', + restrictions: { + codeList: ['Perimenopausal', 'Postmenopausal', 'Premenopausal', 'Unknown'], + }, + }, + diff: { + type: 'deleted', + data: { + name: 'menopause_status', + description: 'Indicate the menopause status of the patient at the time of primary diagnosis.', + valueType: 'string', + restrictions: { + codeList: ['Perimenopausal', 'Postmenopausal', 'Premenopausal', 'Unknown'], + }, + }, + }, + }, + ], + [ + 'primary_diagnosis.presenting_symptoms', + { + left: { + name: 'presenting_symptoms', + description: 'Indicate presenting symptoms at time of primary diagnosis.', + valueType: 'string', + restrictions: { + codeList: ['Abdominal Pain', 'Anemia', 'Diabetes', 'Diarrhea', 'Nausea', 'None'], + }, + }, + right: { + name: 'presenting_symptoms', + description: 'Indicate presenting symptoms at time of primary diagnosis.', + valueType: 'string', + isArray: true, + restrictions: { + codeList: ['Abdominal Pain', 'Anemia', 'Diabetes', 'Diarrhea', 'Nausea', 'None'], + }, + }, + diff: { + isArray: { + type: 'created', + data: true, + }, + }, + }, + ], + [ + 'sample_registration.program_id', + { + left: { + name: 'program_id', + valueType: 'string', + description: 'Unique identifier of the ARGO program.', + meta: { + validationDependency: true, + primaryId: true, + examples: 'PACA-AU,BR-CA', + displayName: 'Program ID', + }, + restrictions: { + required: true, + }, + }, + right: { + name: 'program_id', + valueType: 'integer', + description: 'Unique identifier of the ARGO program.', + meta: { + validationDependency: true, + primaryId: true, + examples: 'PACA-AU,BR-CA', + displayName: 'Program ID', + }, + restrictions: { + required: true, + }, + }, + diff: { + valueType: { + type: 'updated', + data: 'integer', + }, + }, + }, + ], + [ + 'specimen.percent_stromal_cells', + { + left: { + name: 'percent_stromal_cells', + description: + 'Indicate a value, in decimals, that represents the percentage of reactive cells that are present in a malignant tumour specimen but are not malignant such as fibroblasts, vascular structures, etc.', + valueType: 'number', + meta: { + dependsOn: 'sample_registration.tumour_normal_designation', + notes: '', + displayName: 'Percent Stromal Cells', + }, + restrictions: { + range: { + min: 0, + max: 0.1, + }, + }, + }, + right: { + name: 'percent_stromal_cells', + description: + 'Indicate a value, in decimals, that represents the percentage of reactive cells that are present in a malignant tumour specimen but are not malignant such as fibroblasts, vascular structures, etc.', + valueType: 'number', + meta: { + dependsOn: 'sample_registration.tumour_normal_designation', + notes: '', + displayName: 'Percent Stromal Cells', + }, + restrictions: { + range: { + max: 1, + }, + }, + }, + diff: { + restrictions: { + range: { + min: { + type: 'deleted', + data: 0, + }, + max: { + type: 'updated', + data: 1, + }, + }, + }, + }, + }, + ], +] satisfies DictionaryDiffArray; +export default diffResponse; diff --git a/packages/client/test/fixtures/registrationSchema.ts b/packages/client/test/fixtures/registrationSchema.ts new file mode 100644 index 0000000..4d57053 --- /dev/null +++ b/packages/client/test/fixtures/registrationSchema.ts @@ -0,0 +1,519 @@ +import { Dictionary } from 'dictionary'; + +const dictionary: Dictionary = { + schemas: [ + { + name: 'registration', + description: 'TSV for Registration of Donor-Specimen-Sample', + fields: [ + { + name: 'program_id', + valueType: 'string', + description: 'Unique identifier for program', + meta: { + key: true, + examples: 'PACA-CA, BASHAR-LA', + }, + restrictions: { + required: true, + regex: '^[A-Z1-9][-_A-Z1-9]{2,7}(-[A-Z][A-Z])$', + }, + }, + { + name: 'submitter_donor_id', + valueType: 'string', + description: 'Unique identifier for donor, assigned by the data provider.', + meta: { + key: true, + }, + restrictions: { + required: true, + regex: '^(?!(DO|do)).+', + }, + }, + { + name: 'gender', + valueType: 'string', + description: 'The gender of the patient', + meta: { + default: 'Other', + }, + restrictions: { + required: true, + codeList: ['Male', 'Female', 'Other'], + }, + }, + { + name: 'submitter_specimen_id', + valueType: 'string', + description: 'Submitter assigned specimen id', + meta: { + key: true, + }, + restrictions: { + required: true, + regex: '^(?!(SP|sp)).+', + }, + }, + { + name: 'specimen_type', + valueType: 'string', + description: 'Indicate the tissue source of the biospecimen', + meta: { + default: 'Other', + }, + restrictions: { + required: true, + codeList: [ + 'Blood derived', + 'Blood derived - bone marrow', + 'Blood derived - peripheral blood', + 'Bone marrow', + 'Buccal cell', + 'Lymph node', + 'Solid tissue', + 'Plasma', + 'Serum', + 'Urine', + 'Cerebrospinal fluid', + 'Sputum', + 'NOS (Not otherwise specified)', + 'Other', + 'FFPE', + 'Pleural effusion', + 'Mononuclear cells from bone marrow', + 'Saliva', + 'Skin', + ], + }, + }, + { + name: 'tumour_normal_designation', + valueType: 'string', + description: 'Indicate whether specimen is tumour or normal type', + restrictions: { + required: true, + codeList: [ + 'Normal', + 'Normal - tissue adjacent to primary tumour', + 'Primary tumour', + 'Primary tumour - adjacent to normal', + 'Primary tumour - additional new primary', + 'Recurrent tumour', + 'Metastatic tumour', + 'Metastatic tumour - metastasis local to lymph node', + 'Metastatic tumour - metastasis to distant location', + 'Metastatic tumour - additional metastatic', + 'Xenograft - derived from primary tumour', + 'Xenograft - derived from tumour cell line', + 'Cell line - derived from xenograft tissue', + 'Cell line - derived from tumour', + 'Cell line - derived from normal', + ], + }, + }, + { + name: 'submitter_sample_id', + valueType: 'string', + description: 'Submitter assigned sample id', + restrictions: { + required: true, + regex: '^(?!(SA|sa)).+', + }, + }, + { + name: 'sample_type', + valueType: 'string', + description: 'Specimen Type', + restrictions: { + required: true, + codeList: [ + 'Total DNA', + 'Amplified DNA', + 'ctDNA', + 'other DNA enrichments', + 'Total RNA', + 'Ribo-Zero RNA', + 'polyA+ RNA', + 'other RNA fractions', + ], + }, + }, + ], + }, + { + name: 'address', + description: 'adderss schema', + fields: [ + { + name: 'postal_code', + valueType: 'string', + description: 'postal code', + restrictions: { + required: true, + script: + '/** important to return the result object here here */\r\n(function validate(inputs) {\r\n const {$row, $field, $name} = inputs; var person = $row;\r\nvar postalCode = $field; var result = { valid: true, message: "ok"};\r\n\r\n /* custom logic start */\r\n if (person.country === "US") {\r\n var valid = /^[0-9]{5}(?:-[0-9]{4})?$/.test(postalCode);\r\n if (!valid) {\r\n result.valid = false;\r\n result.message = "invalid postal code for US";\r\n }\r\n } else if (person.country === "CANADA") {\r\n var valid = /^[A-Za-z]\\d[A-Za-z][ -]?\\d[A-Za-z]\\d$/.test(postalCode);\r\n if (!valid) {\r\n result.valid = false;\r\n result.message = "invalid postal code for CANADA";\r\n }\r\n }\r\n /* custom logic end */\r\n\r\n return result;\r\n})\r\n\r\n', + }, + }, + { + name: 'unit_number', + valueType: 'integer', + description: 'unit number', + restrictions: { + range: { + min: 0, + exclusiveMax: 999, + }, + }, + }, + { + name: 'country', + valueType: 'string', + description: 'Country', + restrictions: { + required: true, + codeList: ['US', 'CANADA'], + }, + }, + ], + }, + { + name: 'donor', + description: 'TSV for donor', + fields: [ + { + name: 'program_id', + valueType: 'string', + description: 'Unique identifier for program', + meta: { + key: true, + }, + restrictions: { + required: true, + regex: '^[A-Z1-9][-_A-Z1-9]{2,7}(-[A-Z][A-Z])$', + }, + }, + { + name: 'submitter_donor_id', + valueType: 'string', + description: 'Unique identifier for donor, assigned by the data provider.', + meta: { + key: true, + }, + restrictions: { + required: true, + regex: '^(?!(DO|do)).+', + }, + }, + { + name: 'gender', + valueType: 'string', + description: 'The gender of the patient', + meta: { + default: 'Other', + }, + restrictions: { + required: true, + codeList: ['Male', 'Female', 'Other'], + }, + }, + { + name: 'ethnicity', + valueType: 'string', + description: 'The ethnicity of the patient', + restrictions: { + required: true, + codeList: ['asian', 'black or african american', 'caucasian', 'not reported'], + }, + }, + { + name: 'vital_status', + valueType: 'string', + description: 'Indicate the vital status of the patient', + restrictions: { + required: true, + codeList: ['alive', 'deceased'], + }, + }, + { + name: 'cause_of_death', + valueType: 'string', + description: 'Indicate the cause of death of patient', + restrictions: { + required: false, + codeList: ['died of cancer', 'died of other reasons', 'N/A'], + }, + }, + { + name: 'survival_time', + valueType: 'integer', + description: 'Survival time', + restrictions: { + required: false, + }, + }, + ], + }, + { + name: 'favorite_things', + description: 'favorite things listed', + fields: [ + { + name: 'id', + valueType: 'string', + description: 'Favourite id values', + restrictions: { + required: true, + regex: '^[A-Z1-9][-_A-Z1-9]{2,7}$', + }, + }, + { + name: 'qWords', + valueType: 'string', + description: 'Words starting with q', + restrictions: { + required: false, + regex: '^q.*$', + }, + isArray: true, + }, + { + name: 'qWord', + valueType: 'string', + description: 'Word starting with q', + restrictions: { + required: false, + regex: '^q.*$', + }, + isArray: false, + }, + { + name: 'fruit', + valueType: 'string', + description: 'fruit', + restrictions: { + required: false, + codeList: ['Mango', 'Orange', 'None'], + }, + isArray: true, + }, + { + name: 'fruit_single_value', + valueType: 'string', + description: 'fruit', + restrictions: { + required: false, + codeList: ['Mango', 'Orange', 'None'], + }, + isArray: false, + }, + { + name: 'animal', + valueType: 'string', + description: 'animal', + restrictions: { + required: false, + codeList: ['Dog', 'Cat', 'None'], + }, + isArray: true, + }, + { + name: 'fraction', + valueType: 'number', + description: 'numbers between 0 and 1 exclusive', + restrictions: { required: false, range: { max: 1, exclusiveMin: 0 } }, + isArray: true, + }, + { + name: 'integers', + valueType: 'integer', + description: 'integers between -10 and 10', + restrictions: { required: false, range: { max: 10, min: -10 } }, + isArray: true, + }, + { + name: 'unique_value', + valueType: 'string', + description: 'unique value', + restrictions: { required: false, unique: true }, + isArray: true, + }, + ], + }, + { + name: 'parent_schema_1', + description: 'Parent schema 1. Used to test relational validations', + fields: [ + { + name: 'id', + valueType: 'string', + description: 'Id', + }, + { + name: 'external_id', + valueType: 'string', + description: 'External Id', + }, + { + name: 'name', + valueType: 'string', + description: 'Name', + }, + ], + }, + { + name: 'parent_schema_2', + description: 'Parent schema 1. Used to test relational validations', + fields: [ + { + name: 'id1', + valueType: 'string', + description: 'Id 1', + isArray: true, + }, + { + name: 'id2', + valueType: 'string', + description: 'Id 2', + isArray: true, + }, + ], + }, + { + name: 'child_schema_simple_fk', + description: 'Child schema referencing a field in a foreign schema', + restrictions: { + foreignKey: [ + { + schema: 'parent_schema_1', + mappings: [ + { + local: 'parent_schema_1_id', + foreign: 'id', + }, + ], + }, + ], + }, + fields: [ + { + name: 'id', + valueType: 'number', + description: 'Id', + }, + { + name: 'parent_schema_1_id', + valueType: 'string', + description: 'Reference to id in schema parent_schema_1', + }, + ], + }, + { + name: 'child_schema_composite_fk', + description: 'Child schema referencing several fields in a foreign schema', + restrictions: { + foreignKey: [ + { + schema: 'parent_schema_1', + mappings: [ + { + local: 'parent_schema_1_id', + foreign: 'id', + }, + { + local: 'parent_schema_1_external_id', + foreign: 'external_id', + }, + ], + }, + ], + }, + fields: [ + { + name: 'id', + valueType: 'number', + description: 'Id', + }, + { + name: 'parent_schema_1_id', + valueType: 'string', + description: 'Reference to id in schema parent_schema_1', + }, + { + name: 'parent_schema_1_external_id', + valueType: 'string', + description: 'Reference to external id in schema parent_schema_1', + }, + ], + }, + { + name: 'child_schema_composite_array_values_fk', + description: 'Child schema referencing several fields in a foreign schema', + restrictions: { + foreignKey: [ + { + schema: 'parent_schema_2', + mappings: [ + { + local: 'parent_schema_2_id1', + foreign: 'id1', + }, + { + local: 'parent_schema_2_id12', + foreign: 'id2', + }, + ], + }, + ], + }, + fields: [ + { + name: 'id', + valueType: 'number', + description: 'Id', + }, + { + name: 'parent_schema_2_id1', + valueType: 'string', + description: 'Reference to id1 in schema parent_schema_2', + isArray: true, + }, + { + name: 'parent_schema_2_id12', + valueType: 'string', + description: 'Reference to external id2 in schema parent_schema_2', + isArray: true, + }, + ], + }, + { + name: 'unique_key_schema', + description: 'Schema to test uniqueKey restriction', + restrictions: { + uniqueKey: ['numeric_id_1', 'string_id_2', 'array_string_id_3'], + }, + fields: [ + { + name: 'numeric_id_1', + valueType: 'number', + description: 'Id 1. Numeric value as part of a composite unique key', + }, + { + name: 'string_id_2', + valueType: 'string', + description: 'Id 2. String value as part of a composite unique key', + }, + { + name: 'array_string_id_3', + valueType: 'string', + description: 'Id 3. String array as part of a composite unique key', + isArray: true, + }, + ], + }, + ], + name: 'ARGO Clinical Submission', + version: '1.0', +}; + +export default dictionary; diff --git a/packages/client/test/processing.spec.ts b/packages/client/test/processing.spec.ts new file mode 100644 index 0000000..503239b --- /dev/null +++ b/packages/client/test/processing.spec.ts @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import chai from 'chai'; +import { functions as schemaService } from '../src'; +import { loggerFor } from '../src/logger'; +import { SchemaValidationErrorTypes } from '../src/validation'; +import dictionary from './fixtures/registrationSchema'; +const L = loggerFor(__filename); + +chai.should(); + +const VALUE_NOT_ALLOWED = 'The value is not permissible for this field.'; +const PROGRAM_ID_REQ = 'program_id is a required field.'; + +describe('processing', () => { + it('should populate records based on default value ', () => { + const result = schemaService.processRecords(dictionary, 'registration', [ + { + program_id: 'PEME-CA', + submitter_donor_id: 'OD1234', + gender: '', + submitter_specimen_id: '87813', + specimen_type: 'Skin', + tumour_normal_designation: 'Normal', + submitter_sample_id: 'MAS123', + sample_type: 'ctDNA', + }, + { + program_id: 'PEME-CA', + submitter_donor_id: 'OD1234', + gender: '', + submitter_specimen_id: '87812', + specimen_type: 'Skin', + tumour_normal_designation: 'Normal', + submitter_sample_id: 'MAS1234', + sample_type: 'ctDNA', + }, + ]); + chai.expect(result.processedRecords[0].gender).to.eq('Other'); + chai.expect(result.processedRecords[1].gender).to.eq('Other'); + }); + + it('should NOT populate missing columns based on default value ', () => { + const result = schemaService.processRecords(dictionary, 'registration', [ + { + program_id: 'PEME-CA', + submitter_donor_id: 'OD1234', + gendr: '', + submitter_specimen_id: '87813', + specimen_type: 'Skin', + tumour_normal_designation: 'Normal', + submitter_sample_id: 'MAS123', + sample_type: 'ctDNA', + }, + { + program_id: 'PEME-CA', + submitter_donor_id: 'OD1234', + gender: '', + submitter_specimen_id: '87812', + specimen_type: 'Skin', + tumour_normal_designation: 'Normal', + submitter_sample_id: 'MAS1234', + sample_type: 'ctDNA', + }, + ]); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.MISSING_REQUIRED_FIELD, + fieldName: 'gender', + index: 0, + info: {}, + message: 'gender is a required field.', + }); + }); +}); diff --git a/packages/client/test/schema-diff.json b/packages/client/test/schema-diff.json deleted file mode 100644 index e4f7a4d..0000000 --- a/packages/client/test/schema-diff.json +++ /dev/null @@ -1,312 +0,0 @@ -[ - [ - "donor.submitter_donor_id", - { - "left": { - "description": "Unique identifier of the donor, assigned by the data provider.", - "name": "submitter_donor_id", - "restrictions": { - "required": true, - "regex": "[A-Za-z0-9\\-\\._]{1,64}" - }, - "valueType": "string" - }, - "right": { - "description": "Unique identifier of the donor, assigned by the data provider.", - "name": "submitter_donor_id", - "restrictions": { - "required": true, - "regex": "[A-Za-z0-9\\-\\._]{3,64}" - }, - "valueType": "string" - }, - "diff": { - "restrictions": { - "regex": { - "type": "updated", - "data": "[A-Za-z0-9\\-\\._]{3,64}" - } - } - } - } - ], - [ - "donor.vital_status", - { - "left": { - "description": "Donors last known state of living or deceased.", - "name": "vital_status", - "restrictions": { - "codeList": ["Alive", "Deceased", "Not reported", "Unknown"], - "required": true - }, - "valueType": "string" - }, - "right": { - "description": "Donors last known state of living or deceased.", - "name": "vital_status", - "restrictions": { - "regex": "[A-Z]{3,100}", - "required": true - }, - "valueType": "string" - }, - "diff": { - "restrictions": { - "codeList": { - "type": "deleted", - "data": ["Alive", "Deceased", "Not reported", "Unknown"] - }, - "regex": { - "type": "created", - "data": "[A-Z]{3,100}" - } - } - } - } - ], - [ - "donor.cause_of_death", - { - "left": { - "description": "Description of the cause of a donor's death.", - "name": "cause_of_death", - "restrictions": { - "codeList": ["Died of cancer", "Died of other reasons", "Not reported", "Unknown"] - }, - "valueType": "string" - }, - "right": { - "description": "Description of the cause of a donor's death.", - "name": "cause_of_death", - "restrictions": { - "codeList": ["Died of other reasons", "Not reported", "N/A"] - }, - "valueType": "string" - }, - "diff": { - "restrictions": { - "codeList": { - "type": "updated", - "data": { - "added": ["N/A"], - "deleted": ["Died of cancer", "Unknown"] - } - } - } - } - } - ], - [ - "donor.survival_time", - { - "left": { - "description": "Interval of how long the donor has survived since primary diagnosis, in days.", - "meta": { - "units": "days" - }, - "name": "survival_time", - "valueType": "integer" - }, - "right": { - "description": "Interval of how long the donor has survived since primary diagnosis, in days.", - "meta": { - "units": "days" - }, - "name": "survival_time", - "valueType": "integer", - "restrictions": { - "script": " $field / 2 == 0 ", - "range": { - "min": 0, - "max": 200000 - } - } - }, - "diff": { - "restrictions": { - "type": "created", - "data": { - "range": { - "min": 0, - "max": 200000 - }, - "script": " $field / 2 == 0 " - } - } - } - } - ], - [ - "primary_diagnosis.cancer_type_code", - { - "left": { - "name": "cancer_type_code", - "valueType": "string", - "description": "The code to represent the cancer type using the WHO ICD-10 code (https://icd.who.int/browse10/2016/en#/) classification.", - "restrictions": { - "required": true, - "regex": "[A-Z]{1}[0-9]{2}.[0-9]{0,3}[A-Z]{0,1}$" - } - }, - "right": { - "name": "cancer_type_code", - "valueType": "string", - "description": "The code to represent the cancer type using the WHO ICD-10 code (https://icd.who.int/browse10/2016/en#/) classification.", - "restrictions": { - "required": true, - "regex": "[A-Z]{1}[0-9]{2}.[0-9]{0,3}[A-Z]{2,3}$" - } - }, - "diff": { - "restrictions": { - "regex": { - "type": "updated", - "data": "[A-Z]{1}[0-9]{2}.[0-9]{0,3}[A-Z]{2,3}$" - } - } - } - } - ], - [ - "primary_diagnosis.menopause_status", - { - "left": { - "name": "menopause_status", - "description": "Indicate the menopause status of the patient at the time of primary diagnosis.", - "valueType": "string", - "restrictions": { - "codeList": ["Perimenopausal", "Postmenopausal", "Premenopausal", "Unknown"] - } - }, - "diff": { - "type": "deleted", - "data": { - "name": "menopause_status", - "description": "Indicate the menopause status of the patient at the time of primary diagnosis.", - "valueType": "string", - "restrictions": { - "codeList": ["Perimenopausal", "Postmenopausal", "Premenopausal", "Unknown"] - } - } - } - } - ], - [ - "primary_diagnosis.presenting_symptoms", - { - "left": { - "name": "presenting_symptoms", - "description": "Indicate presenting symptoms at time of primary diagnosis.", - "valueType": "string", - "restrictions": { - "codeList": ["Abdominal Pain", "Anemia", "Diabetes", "Diarrhea", "Nausea", "None"] - } - }, - "right": { - "name": "presenting_symptoms", - "description": "Indicate presenting symptoms at time of primary diagnosis.", - "valueType": "string", - "isArray": true, - "restrictions": { - "codeList": ["Abdominal Pain", "Anemia", "Diabetes", "Diarrhea", "Nausea", "None"] - } - }, - "diff": { - "isArray": { - "type": "created", - "data": true - } - } - } - ], - [ - "sample_registration.program_id", - { - "left": { - "name": "program_id", - "valueType": "string", - "description": "Unique identifier of the ARGO program.", - "meta": { - "validationDependency": true, - "primaryId": true, - "examples": "PACA-AU,BR-CA", - "displayName": "Program ID" - }, - "restrictions": { - "required": true - } - }, - "right": { - "name": "program_id", - "valueType": "integer", - "description": "Unique identifier of the ARGO program.", - "meta": { - "validationDependency": true, - "primaryId": true, - "examples": "PACA-AU,BR-CA", - "displayName": "Program ID" - }, - "restrictions": { - "required": true - } - }, - "diff": { - "valueType": { - "type": "updated", - "data": "integer" - } - } - } - ], - [ - "specimen.percent_stromal_cells", - { - "left": { - "name": "percent_stromal_cells", - "description": "Indicate a value, in decimals, that represents the percentage of reactive cells that are present in a malignant tumour specimen but are not malignant such as fibroblasts, vascular structures, etc.", - "valueType": "number", - "meta": { - "dependsOn": "sample_registration.tumour_normal_designation", - "notes": "", - "displayName": "Percent Stromal Cells" - }, - "restrictions": { - "range": { - "min": 0, - "max": 0.1 - } - } - }, - "right": { - "name": "percent_stromal_cells", - "description": "Indicate a value, in decimals, that represents the percentage of reactive cells that are present in a malignant tumour specimen but are not malignant such as fibroblasts, vascular structures, etc.", - "valueType": "number", - "meta": { - "dependsOn": "sample_registration.tumour_normal_designation", - "notes": "", - "displayName": "Percent Stromal Cells" - }, - "restrictions": { - "range": { - "max": 1 - } - } - }, - "diff": { - "restrictions": { - "range": { - "min": { - "type": "deleted", - "data": 0 - }, - "max": { - "type": "updated", - "data": 1 - } - } - } - } - } - ] -] diff --git a/packages/client/test/schema-functions.spec.ts b/packages/client/test/schema-functions.spec.ts deleted file mode 100644 index 315fa81..0000000 --- a/packages/client/test/schema-functions.spec.ts +++ /dev/null @@ -1,9295 +0,0 @@ -/* - * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved - * - * This program and the accompanying materials are made available under the terms of - * the GNU Affero General Public License v3.0. You should have received a copy of the - * GNU Affero General Public License along with this program. - * If not, see . - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT - * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER - * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -import chai from 'chai'; -import * as schemaService from '../src/schema-functions'; -import { SchemasDictionary, SchemaValidationErrorTypes } from '../src/schema-entities'; -import schemaErrorMessage from '../src/schema-error-messages'; -import { loggerFor } from '../src/logger'; -const L = loggerFor(__filename); - -chai.should(); -const schema: SchemasDictionary = require('./schema.json')[0]; - -const VALUE_NOT_ALLOWED = 'The value is not permissible for this field.'; -const PROGRAM_ID_REQ = 'program_id is a required field.'; - -describe('schema-functions', () => { - it('should populate records based on default value ', () => { - const result = schemaService.processRecords(schema, 'registration', [ - { - program_id: 'PEME-CA', - submitter_donor_id: 'OD1234', - gender: '', - submitter_specimen_id: '87813', - specimen_type: 'Skin', - tumour_normal_designation: 'Normal', - submitter_sample_id: 'MAS123', - sample_type: 'ctDNA', - }, - { - program_id: 'PEME-CA', - submitter_donor_id: 'OD1234', - gender: '', - submitter_specimen_id: '87812', - specimen_type: 'Skin', - tumour_normal_designation: 'Normal', - submitter_sample_id: 'MAS1234', - sample_type: 'ctDNA', - }, - ]); - chai.expect(result.processedRecords[0].gender).to.eq('Other'); - chai.expect(result.processedRecords[1].gender).to.eq('Other'); - }); - - it('should NOT populate missing columns based on default value ', () => { - const result = schemaService.processRecords(schema, 'registration', [ - { - program_id: 'PEME-CA', - submitter_donor_id: 'OD1234', - gendr: '', - submitter_specimen_id: '87813', - specimen_type: 'Skin', - tumour_normal_designation: 'Normal', - submitter_sample_id: 'MAS123', - sample_type: 'ctDNA', - }, - { - program_id: 'PEME-CA', - submitter_donor_id: 'OD1234', - gender: '', - submitter_specimen_id: '87812', - specimen_type: 'Skin', - tumour_normal_designation: 'Normal', - submitter_sample_id: 'MAS1234', - sample_type: 'ctDNA', - }, - ]); - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.MISSING_REQUIRED_FIELD, - fieldName: 'gender', - index: 0, - info: {}, - message: 'gender is a required field.', - }); - }); - - it('should validate required', () => { - const result = schemaService.processRecords(schema, 'registration', [ - { - submitter_donor_id: 'OD1234', - gender: 'Female', - submitter_specimen_id: '87813', - specimen_type: 'Skin', - tumour_normal_designation: 'Normal', - submitter_sample_id: 'MAS123', - sample_type: 'ctDNA', - }, - ]); - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.MISSING_REQUIRED_FIELD, - fieldName: 'program_id', - index: 0, - info: {}, - message: PROGRAM_ID_REQ, - }); - }); - - it('should validate value types', () => { - const result = schemaService.processRecords(schema, 'address', [ - { - country: 'US', - unit_number: 'abc', - postal_code: '12345', - }, - ]); - - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.INVALID_FIELD_VALUE_TYPE, - fieldName: 'unit_number', - index: 0, - info: { value: ['abc'] }, - message: VALUE_NOT_ALLOWED, - }); - }); - - it('should convert string to integer after processing', () => { - const result = schemaService.processRecords(schema, 'address', [ - { - country: 'US', - unit_number: '123', - postal_code: '12345', - }, - ]); - chai.expect(result.processedRecords).to.deep.include({ - country: 'US', - unit_number: 123, - postal_code: '12345', - }); - }); - - it('should validate regex', () => { - const result = schemaService.processRecords(schema, 'registration', [ - { - program_id: 'PEME-CAA', - submitter_donor_id: 'OD1234', - gender: 'Female', - submitter_specimen_id: '87813', - specimen_type: 'Skin', - tumour_normal_designation: 'Normal', - submitter_sample_id: 'MAS123', - sample_type: 'ctDNA', - }, - ]); - chai.expect(result.validationErrors[0]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_REGEX, - fieldName: 'program_id', - index: 0, - info: { - examples: 'PACA-CA, BASHAR-LA', - regex: '^[A-Z1-9][-_A-Z1-9]{2,7}(-[A-Z][A-Z])$', - value: ['PEME-CAA'], - }, - message: - 'The value is not a permissible for this field, it must meet the regular expression: "^[A-Z1-9][-_A-Z1-9]{2,7}(-[A-Z][A-Z])$". Examples: PACA-CA, BASHAR-LA', - }); - }); - - it('should validate range', () => { - const result = schemaService.processRecords(schema, 'address', [ - { - country: 'US', - postal_code: '12345', - unit_number: '-1', - }, - { - country: 'US', - postal_code: '12345', - unit_number: '223', - }, - { - country: 'US', - postal_code: '12345', - unit_number: '500000', - }, - ]); - - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.INVALID_BY_RANGE, - fieldName: 'unit_number', - index: 0, - info: { - exclusiveMax: 999, - min: 0, - value: [-1], - }, - message: schemaErrorMessage(SchemaValidationErrorTypes.INVALID_BY_RANGE, { - info: { - exclusiveMax: 999, - min: 0, - }, - }), - }); - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.INVALID_BY_RANGE, - fieldName: 'unit_number', - index: 2, - info: { - exclusiveMax: 999, - min: 0, - value: [500000], - }, - message: schemaErrorMessage(SchemaValidationErrorTypes.INVALID_BY_RANGE, { - info: { - exclusiveMax: 999, - min: 0, - }, - }), - }); - }); - - it('should validate script', () => { - const result = schemaService.processRecords(schema, 'address', [ - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - ]); - - chai.expect(result.validationErrors.length).to.eq(2); - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.INVALID_BY_SCRIPT, - fieldName: 'postal_code', - index: 0, - info: { message: 'invalid postal code for US', value: '12' }, - message: 'invalid postal code for US', - }); - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.INVALID_BY_SCRIPT, - fieldName: 'postal_code', - index: 1, - info: { message: 'invalid postal code for CANADA', value: 'ABC' }, - message: 'invalid postal code for CANADA', - }); - }); - - it('should validate if non-required feilds are not provided', () => { - const result = schemaService.processRecords(schema, 'donor', [ - // optional enum field not provided - { - program_id: 'PACA-AU', - submitter_donor_id: 'ICGC_0004', - gender: 'Female', - ethnicity: 'black or african american', - vital_status: 'alive', - }, - // optional enum field provided with proper value - { - program_id: 'PACA-AU', - submitter_donor_id: 'ICGC_0002', - gender: 'Male', - ethnicity: 'asian', - vital_status: 'deceased', - cause_of_death: 'died of cancer', - survival_time: '124', - }, - // optional enum field provided with no value - { - program_id: 'PACA-AU', - submitter_donor_id: 'ICGC_0002', - gender: 'Male', - ethnicity: 'asian', - vital_status: 'deceased', - cause_of_death: '', - survival_time: '124', - }, - ]); - chai.expect(result.validationErrors.length).to.eq(0); - }); - - it('should error if integer fields are not valid', () => { - const result = schemaService.processRecords(schema, 'donor', [ - { - program_id: 'PACA-AU', - submitter_donor_id: 'ICGC_0002', - gender: 'Other', - ethnicity: 'asian', - vital_status: 'deceased', - cause_of_death: 'died of cancer', - survival_time: '0.5', - }, - ]); - - chai.expect(result.validationErrors.length).to.eq(1); - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.INVALID_FIELD_VALUE_TYPE, - fieldName: 'survival_time', - index: 0, - info: { value: ['0.5'] }, - message: VALUE_NOT_ALLOWED, - }); - }); - - it('should validate case insensitive enums, return proper format', () => { - const result = schemaService.processRecords(schema, 'registration', [ - { - program_id: 'PACA-AU', - submitter_donor_id: 'OD1234', - gender: 'feMale', - submitter_specimen_id: '87813', - specimen_type: 'sKiN', - tumour_normal_designation: 'Normal', - submitter_sample_id: 'MAS123', - sample_type: 'CTdna', - }, - ]); - chai.expect(result.validationErrors.length).to.eq(0); - chai.expect(result.processedRecords[0]).to.deep.eq({ - program_id: 'PACA-AU', - submitter_donor_id: 'OD1234', - gender: 'Female', - submitter_specimen_id: '87813', - specimen_type: 'Skin', - tumour_normal_designation: 'Normal', - submitter_sample_id: 'MAS123', - sample_type: 'ctDNA', - }); - }); - - it('should not validate if unrecognized fields are provided', () => { - const result = schemaService.processRecords(schema, 'donor', [ - { - program_id: 'PACA-AU', - submitter_donor_id: 'ICGC_0002', - gender: 'Other', - ethnicity: 'asian', - vital_status: 'deceased', - cause_of_death: 'died of cancer', - survival_time: '5', - hackField: 'muchHack', - }, - ]); - chai.expect(result.validationErrors.length).to.eq(1); - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.UNRECOGNIZED_FIELD, - message: SchemaValidationErrorTypes.UNRECOGNIZED_FIELD, - fieldName: 'hackField', - index: 0, - info: {}, - }); - }); - - it('should validate number/integer array with field defined ranges', () => { - const result = schemaService.processRecords(schema, 'favorite_things', [ - { - id: 'TH-ING', - fraction: ['0.2', '2', '3'], - integers: ['-100', '-2'], - }, - ]); - - chai.expect(result.validationErrors.length).to.eq(2); - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.INVALID_BY_RANGE, - message: 'Value is out of permissible range, value must be > 0 and <= 1.', - index: 0, - fieldName: 'fraction', - info: { value: [2, 3], max: 1, exclusiveMin: 0 }, - }); - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.INVALID_BY_RANGE, - message: 'Value is out of permissible range, value must be >= -10 and <= 10.', - index: 0, - fieldName: 'integers', - info: { value: [-100], max: 10, min: -10 }, - }); - }); - - it('should validate string array with field defined codelist', () => { - const result = schemaService.processRecords(schema, 'favorite_things', [ - { - id: 'TH-ING', - fruit: ['Mango', '2'], - }, - ]); - chai.expect(result.validationErrors.length).to.eq(1); - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.INVALID_ENUM_VALUE, - message: 'The value is not permissible for this field.', - fieldName: 'fruit', - index: 0, - info: { value: ['2'] }, - }); - }); - - it('should validate string with field defined codelist', () => { - const result = schemaService.processRecords(schema, 'favorite_things', [ - { - id: 'TH-ING', - fruit_single_value: 'Banana', - }, - ]); - chai.expect(result.validationErrors.length).to.eq(1); - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.INVALID_ENUM_VALUE, - message: 'The value is not permissible for this field.', - fieldName: 'fruit_single_value', - index: 0, - info: { value: ['Banana'] }, - }); - }); - - it('should validate string array with field defined regex', () => { - const result = schemaService.processRecords(schema, 'favorite_things', [ - { - id: 'TH-ING', - qWords: ['que', 'not_q'], - }, - ]); - chai.expect(result.validationErrors.length).to.eq(1); - chai.expect(result.validationErrors[0]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_REGEX, - message: 'The value is not a permissible for this field, it must meet the regular expression: "^q.*$".', - fieldName: 'qWords', - index: 0, - info: { value: ['not_q'], regex: '^q.*$', examples: undefined }, - }); - }); - - it('should validate string with field defined regex', () => { - const result = schemaService.processRecords(schema, 'favorite_things', [ - { - id: 'TH-ING', - qWord: 'not_q', - }, - ]); - chai.expect(result.validationErrors.length).to.eq(1); - chai.expect(result.validationErrors[0]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_REGEX, - message: 'The value is not a permissible for this field, it must meet the regular expression: "^q.*$".', - fieldName: 'qWord', - index: 0, - info: { value: ['not_q'], regex: '^q.*$', examples: undefined }, - }); - }); - - it('should pass unique restriction validation when only null values exists', () => { - const result = schemaService.processRecords(schema, 'favorite_things', [ - { - id: 'TH-ING', - unique_value: '', - }, - ]); - chai.expect(result.validationErrors.length).to.eq(0); - }); - - it('should pass unique restriction validation when only one record exists', () => { - const result = schemaService.processRecords(schema, 'favorite_things', [ - { - id: 'TH-ING', - unique_value: 'unique_value_1', - }, - ]); - chai.expect(result.validationErrors.length).to.eq(0); - }); - - it('should fail unique restriction validation when duplicate values exist (scalar)', () => { - const result = schemaService.processRecords(schema, 'favorite_things', [ - { - id: 'ID-1', - unique_value: 'unique_value_1', - }, - { - id: 'ID-2', - unique_value: 'unique_value_1', - }, - ]); - - chai.expect(result.validationErrors.length).to.eq(2); - chai.expect(result.validationErrors[0]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE, - message: 'Value for unique_value must be unique.', - fieldName: 'unique_value', - index: 0, - info: { value: ['unique_value_1'] }, - }); - chai.expect(result.validationErrors[1]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE, - message: 'Value for unique_value must be unique.', - fieldName: 'unique_value', - index: 1, - info: { value: ['unique_value_1'] }, - }); - }); - it('should fail unique restriction validation when duplicate values exist (array)', () => { - const result = schemaService.processRecords(schema, 'favorite_things', [ - { - id: 'ID-1', - unique_value: ['unique_value_1', 'unique_value_2'], - }, - { - id: 'ID-2', - unique_value: ['unique_value_1', 'unique_value_2'], - }, - ]); - - chai.expect(result.validationErrors.length).to.eq(2); - chai.expect(result.validationErrors[0]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE, - message: 'Value for unique_value must be unique.', - fieldName: 'unique_value', - index: 0, - info: { value: ['unique_value_1', 'unique_value_2'] }, - }); - chai.expect(result.validationErrors[1]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE, - message: 'Value for unique_value must be unique.', - fieldName: 'unique_value', - index: 1, - info: { value: ['unique_value_1', 'unique_value_2'] }, - }); - }); - - it('should pass foreignKey restriction validation when values exist in foreign schema', () => { - const parent_schema_1_data = [ - { - id: 'parent_schema_1_id_1', - name: 'parent_schema_1_name_1', - }, - { - id: 'parent_schema_1_id_2', - name: 'parent_schema_1_name_2', - }, - ]; - - const child_schema_simple_fk_data = [ - { - id: '1', - parent_schema_1_id: 'parent_schema_1_id_1', - }, - { - id: '2', - parent_schema_1_id: 'parent_schema_1_id_2', - }, - ]; - const schemaData = { - parent_schema_1: parent_schema_1_data, - child_schema_simple_fk: child_schema_simple_fk_data, - }; - - const result = schemaService.processSchemas(schema, schemaData); - - chai.expect(result['parent_schema_1'].validationErrors.length).to.eq(0); - chai.expect(result['child_schema_simple_fk'].validationErrors.length).to.eq(0); - }); - - it('should pass foreignKey restriction validation when local schema has null values', () => { - const parent_schema_1_data = [ - { - id: 'parent_schema_1_id_1', - name: 'parent_schema_1_name_1', - }, - { - id: 'parent_schema_1_id_2', - name: 'parent_schema_1_name_2', - }, - ]; - - const child_schema_simple_fk_data = [ - { - id: '1', - parent_schema_1_id: 'parent_schema_1_id_1', - }, - { - id: '2', - parent_schema_1_id: '', - }, - ]; - const schemaData = { - parent_schema_1: parent_schema_1_data, - child_schema_simple_fk: child_schema_simple_fk_data, - }; - - const result = schemaService.processSchemas(schema, schemaData); - - chai.expect(result['parent_schema_1'].validationErrors.length).to.eq(0); - chai.expect(result['child_schema_simple_fk'].validationErrors.length).to.eq(0); - }); - - it('should pass foreignKey restriction validation when values exist in foreign schema (composite fk)', () => { - const parent_schema_1_data = [ - { - id: 'parent_schema_1_id_1', - external_id: 'parent_schema_1_external_id_1', - name: 'parent_schema_1_name_1', - }, - { - id: 'parent_schema_1_id_2', - external_id: 'parent_schema_1_external_id_2', - name: 'parent_schema_1_name_2', - }, - ]; - - const child_schema_composite_fk_data = [ - { - id: '1', - parent_schema_1_id: 'parent_schema_1_id_1', - parent_schema_1_external_id: 'parent_schema_1_external_id_1', - }, - { - id: '2', - parent_schema_1_id: 'parent_schema_1_id_2', - parent_schema_1_external_id: 'parent_schema_1_external_id_2', - }, - ]; - const schemaData = { - parent_schema_1: parent_schema_1_data, - child_schema_composite_fk: child_schema_composite_fk_data, - }; - - const result = schemaService.processSchemas(schema, schemaData); - - chai.expect(result['parent_schema_1'].validationErrors.length).to.eq(0); - chai.expect(result['child_schema_composite_fk'].validationErrors.length).to.eq(0); - }); - - it('should fail foreignKey restriction validation when value does not exist in foreign schema', () => { - const parent_schema_1_data = [ - { - id: 'parent_schema_1_id_1', - name: 'parent_schema_1_name_1', - }, - { - id: 'parent_schema_1_id_2', - name: 'parent_schema_1_name_2', - }, - ]; - - const child_schema_simple_fk_data = [ - { - id: '1', - parent_schema_1_id: 'parent_schema_1_id_1', - }, - { - id: '2', - parent_schema_1_id: 'non_existing_value_in_foreign_schema', - }, - ]; - const schemaData = { - parent_schema_1: parent_schema_1_data, - child_schema_simple_fk: child_schema_simple_fk_data, - }; - - const result = schemaService.processSchemas(schema, schemaData); - const childSchemaErrors = result['child_schema_simple_fk'].validationErrors; - - chai.expect(childSchemaErrors.length).to.eq(1); - chai.expect(childSchemaErrors[0]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_FOREIGN_KEY, - message: - 'Record violates foreign key restriction defined for field(s) parent_schema_1_id. Key parent_schema_1_id: non_existing_value_in_foreign_schema is not present in schema parent_schema_1.', - fieldName: 'parent_schema_1_id', - index: 1, - info: { foreignSchema: 'parent_schema_1', value: { parent_schema_1_id: 'non_existing_value_in_foreign_schema' } }, - }); - }); - - it('should fail foreignKey restriction validation when values do not exist in foreign schema (composite fk)', () => { - const parent_schema_1_data = [ - { - id: 'parent_schema_1_id_1', - external_id: 'parent_schema_1_external_id_1', - name: 'parent_schema_1_name_1', - }, - { - id: 'parent_schema_1_id_2', - external_id: 'parent_schema_1_external_id_2', - name: 'parent_schema_1_name_2', - }, - ]; - - const child_schema_composite_fk_data = [ - { - id: '1', - parent_schema_1_id: 'parent_schema_1_id_1', - parent_schema_1_external_id: 'parent_schema_1_external_id_1', - }, - { - id: '2', - parent_schema_1_id: 'parent_schema_1_id_2', - parent_schema_1_external_id: 'non_existing_value_in_foreign_schema', - }, - ]; - const schemaData = { - parent_schema_1: parent_schema_1_data, - child_schema_composite_fk: child_schema_composite_fk_data, - }; - - const result = schemaService.processSchemas(schema, schemaData); - const childSchemaErrors = result['child_schema_composite_fk'].validationErrors; - - chai.expect(childSchemaErrors.length).to.eq(1); - chai.expect(childSchemaErrors[0]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_FOREIGN_KEY, - message: - 'Record violates foreign key restriction defined for field(s) parent_schema_1_id, parent_schema_1_external_id. Key parent_schema_1_id: parent_schema_1_id_2, parent_schema_1_external_id: non_existing_value_in_foreign_schema is not present in schema parent_schema_1.', - fieldName: 'parent_schema_1_id, parent_schema_1_external_id', - index: 1, - info: { - foreignSchema: 'parent_schema_1', - value: { - parent_schema_1_external_id: 'non_existing_value_in_foreign_schema', - parent_schema_1_id: 'parent_schema_1_id_2', - }, - }, - }); - }); - - it('should fail foreignKey restriction validation when values (array) do not match in foreign schema (composite fk)', () => { - const parent_schema_2_data = [ - { - id1: ['id1_1', 'id1_2'], - id2: ['id2_1'], - }, - ]; - - const child_schema_composite_array_values_fk_data = [ - { - id: '1', - parent_schema_2_id1: ['id1_1'], - parent_schema_2_id12: ['id1_2', 'id2_1'], - }, - ]; - const schemaData = { - parent_schema_2: parent_schema_2_data, - child_schema_composite_array_values_fk: child_schema_composite_array_values_fk_data, - }; - - const result = schemaService.processSchemas(schema, schemaData); - const childSchemaErrors = result['child_schema_composite_array_values_fk'].validationErrors; - - chai.expect(childSchemaErrors.length).to.eq(1); - chai.expect(childSchemaErrors[0]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_FOREIGN_KEY, - message: - 'Record violates foreign key restriction defined for field(s) parent_schema_2_id1, parent_schema_2_id12. Key parent_schema_2_id1: [id1_1], parent_schema_2_id12: [id1_2, id2_1] is not present in schema parent_schema_2.', - fieldName: 'parent_schema_2_id1, parent_schema_2_id12', - index: 0, - info: { - foreignSchema: 'parent_schema_2', - value: { - parent_schema_2_id1: ['id1_1'], - parent_schema_2_id12: ['id1_2', 'id2_1'], - }, - }, - }); - }); - - it('should pass uniqueKey restriction validation when only a record exists', () => { - const result = schemaService.processRecords(schema, 'unique_key_schema', [ - { - numeric_id_1: '1', - string_id_2: 'string_value', - array_string_id_3: ['array_element_1', 'array_element_2'], - }, - ]); - - chai.expect(result.validationErrors.length).to.eq(0); - }); - - it('should pass uniqueKey restriction validation when values are unique', () => { - const result = schemaService.processRecords(schema, 'unique_key_schema', [ - { - numeric_id_1: '1', - string_id_2: 'string_value', - array_string_id_3: ['array_element_1', 'array_element_2'], - }, - { - numeric_id_1: '1', - string_id_2: 'string_value', - array_string_id_3: ['array_element_1', 'array_element_x'], - }, - ]); - - chai.expect(result.validationErrors.length).to.eq(0); - }); - - it('should fail uniqueKey restriction validation when missing values are part of the key and they are not unique', () => { - const result = schemaService.processRecords(schema, 'unique_key_schema', [ - { - numeric_id_1: '', - string_id_2: '', - array_string_id_3: [], - }, - { - numeric_id_1: '', - string_id_2: '', - array_string_id_3: [], - }, - ]); - - chai.expect(result.validationErrors.length).to.eq(2); - chai.expect(result.validationErrors[0]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, - message: 'Key numeric_id_1: null, string_id_2: null, array_string_id_3: null must be unique.', - fieldName: 'numeric_id_1, string_id_2, array_string_id_3', - index: 0, - info: { - uniqueKeyFields: ['numeric_id_1', 'string_id_2', 'array_string_id_3'], - value: { - numeric_id_1: '', - string_id_2: '', - array_string_id_3: '', - }, - }, - }); - chai.expect(result.validationErrors[1]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, - message: 'Key numeric_id_1: null, string_id_2: null, array_string_id_3: null must be unique.', - fieldName: 'numeric_id_1, string_id_2, array_string_id_3', - index: 1, - info: { - uniqueKeyFields: ['numeric_id_1', 'string_id_2', 'array_string_id_3'], - value: { - numeric_id_1: '', - string_id_2: '', - array_string_id_3: '', - }, - }, - }); - }); - - it('should fail uniqueKey restriction validation when values are not unique', () => { - const result = schemaService.processRecords(schema, 'unique_key_schema', [ - { - numeric_id_1: '1', - string_id_2: 'string_value', - array_string_id_3: ['array_element_1', 'array_element_2'], - }, - { - numeric_id_1: '1', - string_id_2: 'string_value', - array_string_id_3: ['array_element_1', 'array_element_2'], - }, - ]); - - chai.expect(result.validationErrors.length).to.eq(2); - chai.expect(result.validationErrors[0]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, - message: - 'Key numeric_id_1: 1, string_id_2: string_value, array_string_id_3: [array_element_1, array_element_2] must be unique.', - fieldName: 'numeric_id_1, string_id_2, array_string_id_3', - index: 0, - info: { - uniqueKeyFields: ['numeric_id_1', 'string_id_2', 'array_string_id_3'], - value: { - numeric_id_1: 1, - string_id_2: 'string_value', - array_string_id_3: ['array_element_1', 'array_element_2'], - }, - }, - }); - chai.expect(result.validationErrors[1]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, - message: - 'Key numeric_id_1: 1, string_id_2: string_value, array_string_id_3: [array_element_1, array_element_2] must be unique.', - fieldName: 'numeric_id_1, string_id_2, array_string_id_3', - index: 1, - info: { - uniqueKeyFields: ['numeric_id_1', 'string_id_2', 'array_string_id_3'], - value: { - numeric_id_1: 1, - string_id_2: 'string_value', - array_string_id_3: ['array_element_1', 'array_element_2'], - }, - }, - }); - }); -}); - -const records = [ - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, -]; diff --git a/packages/client/test/schema.json b/packages/client/test/schema.json deleted file mode 100644 index 19411a6..0000000 --- a/packages/client/test/schema.json +++ /dev/null @@ -1,522 +0,0 @@ -[ - { - "schemas": [ - { - "name": "registration", - "description": "TSV for Registration of Donor-Specimen-Sample", - "key": "submitter_donor_id", - "fields": [ - { - "name": "program_id", - "valueType": "string", - "description": "Unique identifier for program", - "meta": { - "key": true, - "examples": "PACA-CA, BASHAR-LA" - }, - "restrictions": { - "required": true, - "regex": "^[A-Z1-9][-_A-Z1-9]{2,7}(-[A-Z][A-Z])$" - } - }, - { - "name": "submitter_donor_id", - "valueType": "string", - "description": "Unique identifier for donor, assigned by the data provider.", - "meta": { - "key": true - }, - "restrictions": { - "required": true, - "regex": "^(?!(DO|do)).+" - } - }, - { - "name": "gender", - "valueType": "string", - "description": "The gender of the patient", - "meta": { - "default": "Other" - }, - "restrictions": { - "required": true, - "codeList": ["Male", "Female", "Other"] - } - }, - { - "name": "submitter_specimen_id", - "valueType": "string", - "description": "Submitter assigned specimen id", - "meta": { - "key": true - }, - "restrictions": { - "required": true, - "regex": "^(?!(SP|sp)).+" - } - }, - { - "name": "specimen_type", - "valueType": "string", - "description": "Indicate the tissue source of the biospecimen", - "meta": { - "default": "Other" - }, - "restrictions": { - "required": true, - "codeList": [ - "Blood derived", - "Blood derived - bone marrow", - "Blood derived - peripheral blood", - "Bone marrow", - "Buccal cell", - "Lymph node", - "Solid tissue", - "Plasma", - "Serum", - "Urine", - "Cerebrospinal fluid", - "Sputum", - "NOS (Not otherwise specified)", - "Other", - "FFPE", - "Pleural effusion", - "Mononuclear cells from bone marrow", - "Saliva", - "Skin" - ] - } - }, - { - "name": "tumour_normal_designation", - "valueType": "string", - "description": "Indicate whether specimen is tumour or normal type", - "restrictions": { - "required": true, - "codeList": [ - "Normal", - "Normal - tissue adjacent to primary tumour", - "Primary tumour", - "Primary tumour - adjacent to normal", - "Primary tumour - additional new primary", - "Recurrent tumour", - "Metastatic tumour", - "Metastatic tumour - metastasis local to lymph node", - "Metastatic tumour - metastasis to distant location", - "Metastatic tumour - additional metastatic", - "Xenograft - derived from primary tumour", - "Xenograft - derived from tumour cell line", - "Cell line - derived from xenograft tissue", - "Cell line - derived from tumour", - "Cell line - derived from normal" - ] - } - }, - { - "name": "submitter_sample_id", - "valueType": "string", - "description": "Submitter assigned sample id", - "restrictions": { - "required": true, - "regex": "^(?!(SA|sa)).+" - } - }, - { - "name": "sample_type", - "valueType": "string", - "description": "Specimen Type", - "restrictions": { - "required": true, - "codeList": [ - "Total DNA", - "Amplified DNA", - "ctDNA", - "other DNA enrichments", - "Total RNA", - "Ribo-Zero RNA", - "polyA+ RNA", - "other RNA fractions" - ] - } - } - ] - }, - { - "name": "address", - "description": "adderss schema", - "fields": [ - { - "name": "postal_code", - "valueType": "string", - "description": "postal code", - "restrictions": { - "required": true, - "script": "/** important to return the result object here here */\r\n(function validate(inputs) {\r\n const {$row, $field, $name} = inputs; var person = $row;\r\nvar postalCode = $field; var result = { valid: true, message: \"ok\"};\r\n\r\n /* custom logic start */\r\n if (person.country === \"US\") {\r\n var valid = /^[0-9]{5}(?:-[0-9]{4})?$/.test(postalCode);\r\n if (!valid) {\r\n result.valid = false;\r\n result.message = \"invalid postal code for US\";\r\n }\r\n } else if (person.country === \"CANADA\") {\r\n var valid = /^[A-Za-z]\\d[A-Za-z][ -]?\\d[A-Za-z]\\d$/.test(postalCode);\r\n if (!valid) {\r\n result.valid = false;\r\n result.message = \"invalid postal code for CANADA\";\r\n }\r\n }\r\n /* custom logic end */\r\n\r\n return result;\r\n})\r\n\r\n" - } - }, - { - "name": "unit_number", - "valueType": "integer", - "description": "unit number", - "restrictions": { - "range": { - "min": 0, - "exclusiveMax": 999 - } - } - }, - { - "name": "country", - "valueType": "string", - "description": "Country", - "restrictions": { - "required": true, - "codeList": ["US", "CANADA"] - } - } - ] - }, - { - "name": "donor", - "description": "TSV for donor", - "key": "submitter_donor_id", - "fields": [ - { - "name": "program_id", - "valueType": "string", - "description": "Unique identifier for program", - "meta": { - "key": true - }, - "restrictions": { - "required": true, - "regex": "^[A-Z1-9][-_A-Z1-9]{2,7}(-[A-Z][A-Z])$" - } - }, - { - "name": "submitter_donor_id", - "valueType": "string", - "description": "Unique identifier for donor, assigned by the data provider.", - "meta": { - "key": true - }, - "restrictions": { - "required": true, - "regex": "^(?!(DO|do)).+" - } - }, - { - "name": "gender", - "valueType": "string", - "description": "The gender of the patient", - "meta": { - "default": "Other" - }, - "restrictions": { - "required": true, - "codeList": ["Male", "Female", "Other"] - } - }, - { - "name": "ethnicity", - "valueType": "string", - "description": "The ethnicity of the patient", - "restrictions": { - "required": true, - "codeList": ["asian", "black or african american", "caucasian", "not reported"] - } - }, - { - "name": "vital_status", - "valueType": "string", - "description": "Indicate the vital status of the patient", - "restrictions": { - "required": true, - "codeList": ["alive", "deceased"] - } - }, - { - "name": "cause_of_death", - "valueType": "string", - "description": "Indicate the cause of death of patient", - "restrictions": { - "required": false, - "codeList": ["died of cancer", "died of other reasons", "N/A"] - } - }, - { - "name": "survival_time", - "valueType": "integer", - "description": "Survival time", - "restrictions": { - "required": false - } - } - ] - }, - { - "name": "favorite_things", - "description": "favorite things listed", - "fields": [ - { - "name": "id", - "valueType": "string", - "description": "Favourite id values", - "restrictions": { - "required": true, - "regex": "^[A-Z1-9][-_A-Z1-9]{2,7}$" - } - }, - { - "name": "qWords", - "valueType": "string", - "description": "Words starting with q", - "restrictions": { - "required": false, - "regex": "^q.*$" - }, - "isArray": true - }, - { - "name": "qWord", - "valueType": "string", - "description": "Word starting with q", - "restrictions": { - "required": false, - "regex": "^q.*$" - }, - "isArray": false - }, - { - "name": "fruit", - "valueType": "string", - "description": "fruit", - "restrictions": { - "required": false, - "codeList": ["Mango", "Orange", "None"] - }, - "isArray": true - }, - { - "name": "fruit_single_value", - "valueType": "string", - "description": "fruit", - "restrictions": { - "required": false, - "codeList": ["Mango", "Orange", "None"] - }, - "isArray": false - }, - { - "name": "animal", - "valueType": "string", - "description": "animal", - "restrictions": { - "required": false, - "codeList": ["Dog", "Cat", "None"] - }, - "isArray": true - }, - { - "name": "fraction", - "valueType": "number", - "description": "numbers between 0 and 1 exclusive", - "restrictions": { "required": false, "range": { "max": 1, "exclusiveMin": 0 } }, - "isArray": true - }, - { - "name": "integers", - "valueType": "integer", - "description": "integers between -10 and 10", - "restrictions": { "required": false, "range": { "max": 10, "min": -10 } }, - "isArray": true - }, - { - "name": "unique_value", - "valueType": "string", - "description": "unique value", - "restrictions": { "required": false, "unique": true }, - "isArray": true - } - ] - }, - { - "name": "parent_schema_1", - "description": "Parent schema 1. Used to test relational validations", - "fields": [ - { - "name": "id", - "valueType": "string", - "description": "Id" - }, - { - "name": "external_id", - "valueType": "string", - "description": "External Id" - }, - { - "name": "name", - "valueType": "string", - "description": "Name" - } - ] - }, - { - "name": "parent_schema_2", - "description": "Parent schema 1. Used to test relational validations", - "fields": [ - { - "name": "id1", - "valueType": "string", - "description": "Id 1", - "isArray": true - }, - { - "name": "id2", - "valueType": "string", - "description": "Id 2", - "isArray": true - } - ] - }, - { - "name": "child_schema_simple_fk", - "description": "Child schema referencing a field in a foreign schema", - "restrictions": { - "foreignKey": [ - { - "schema": "parent_schema_1", - "mappings": [ - { - "local": "parent_schema_1_id", - "foreign": "id" - } - ] - } - ] - }, - "fields": [ - { - "name": "id", - "valueType": "number", - "description": "Id" - }, - { - "name": "parent_schema_1_id", - "valueType": "string", - "description": "Reference to id in schema parent_schema_1" - } - ] - }, - { - "name": "child_schema_composite_fk", - "description": "Child schema referencing several fields in a foreign schema", - "restrictions": { - "foreignKey": [ - { - "schema": "parent_schema_1", - "mappings": [ - { - "local": "parent_schema_1_id", - "foreign": "id" - }, - { - "local": "parent_schema_1_external_id", - "foreign": "external_id" - } - ] - } - ] - }, - "fields": [ - { - "name": "id", - "valueType": "number", - "description": "Id" - }, - { - "name": "parent_schema_1_id", - "valueType": "string", - "description": "Reference to id in schema parent_schema_1" - }, - { - "name": "parent_schema_1_external_id", - "valueType": "string", - "description": "Reference to external id in schema parent_schema_1" - } - ] - }, - { - "name": "child_schema_composite_array_values_fk", - "description": "Child schema referencing several fields in a foreign schema", - "restrictions": { - "foreignKey": [ - { - "schema": "parent_schema_2", - "mappings": [ - { - "local": "parent_schema_2_id1", - "foreign": "id1" - }, - { - "local": "parent_schema_2_id12", - "foreign": "id2" - } - ] - } - ] - }, - "fields": [ - { - "name": "id", - "valueType": "number", - "description": "Id" - }, - { - "name": "parent_schema_2_id1", - "valueType": "string", - "description": "Reference to id1 in schema parent_schema_2", - "isArray": true - }, - { - "name": "parent_schema_2_id12", - "valueType": "string", - "description": "Reference to external id2 in schema parent_schema_2", - "isArray": true - } - ] - }, - { - "name": "unique_key_schema", - "description": "Schema to test uniqueKey restriction", - "restrictions": { - "uniqueKey": ["numeric_id_1", "string_id_2", "array_string_id_3"] - }, - "fields": [ - { - "name": "numeric_id_1", - "valueType": "number", - "description": "Id 1. Numeric value as part of a composite unique key" - }, - { - "name": "string_id_2", - "valueType": "string", - "description": "Id 2. String value as part of a composite unique key" - }, - { - "name": "array_string_id_3", - "valueType": "string", - "description": "Id 3. String array as part of a composite unique key", - "isArray": true - } - ] - } - ], - "_id": "5d250369f38d1f0d9376fd38", - "name": "ARGO Clinical Submission", - "version": "1.0", - "createdAt": "2019-07-09T21:13:13.683Z", - "updatedAt": "2019-07-09T21:13:13.683Z", - "__v": 0 - } -] diff --git a/packages/client/test/validation.spec.ts b/packages/client/test/validation.spec.ts new file mode 100644 index 0000000..74e6b29 --- /dev/null +++ b/packages/client/test/validation.spec.ts @@ -0,0 +1,828 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import chai from 'chai'; +import { functions as schemaService } from '../src'; +import { loggerFor } from '../src/logger'; +import { SchemaValidationErrorTypes } from '../src/validation'; +import { rangeToSymbol } from '../src/validation/utils/rangeToSymbol'; +import dictionary from './fixtures/registrationSchema'; +const L = loggerFor(__filename); + +chai.should(); + +const VALUE_NOT_ALLOWED = 'The value is not permissible for this field.'; +const PROGRAM_ID_REQ = 'program_id is a required field.'; + +describe('validation', () => { + it('should validate required', () => { + const result = schemaService.processRecords(dictionary, 'registration', [ + { + submitter_donor_id: 'OD1234', + gender: 'Female', + submitter_specimen_id: '87813', + specimen_type: 'Skin', + tumour_normal_designation: 'Normal', + submitter_sample_id: 'MAS123', + sample_type: 'ctDNA', + }, + ]); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.MISSING_REQUIRED_FIELD, + fieldName: 'program_id', + index: 0, + info: {}, + message: PROGRAM_ID_REQ, + }); + }); + + it('should validate value types', () => { + const result = schemaService.processRecords(dictionary, 'address', [ + { + country: 'US', + unit_number: 'abc', + postal_code: '12345', + }, + ]); + + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_FIELD_VALUE_TYPE, + fieldName: 'unit_number', + index: 0, + info: { value: ['abc'] }, + message: VALUE_NOT_ALLOWED, + }); + }); + + it('should convert string to integer after processing', () => { + const result = schemaService.processRecords(dictionary, 'address', [ + { + country: 'US', + unit_number: '123', + postal_code: '12345', + }, + ]); + chai.expect(result.processedRecords).to.deep.include({ + country: 'US', + unit_number: 123, + postal_code: '12345', + }); + }); + + it('should validate regex', () => { + const result = schemaService.processRecords(dictionary, 'registration', [ + { + program_id: 'PEME-CAA', + submitter_donor_id: 'OD1234', + gender: 'Female', + submitter_specimen_id: '87813', + specimen_type: 'Skin', + tumour_normal_designation: 'Normal', + submitter_sample_id: 'MAS123', + sample_type: 'ctDNA', + }, + ]); + chai.expect(result.validationErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_REGEX, + fieldName: 'program_id', + index: 0, + info: { + examples: 'PACA-CA, BASHAR-LA', + regex: '^[A-Z1-9][-_A-Z1-9]{2,7}(-[A-Z][A-Z])$', + value: ['PEME-CAA'], + }, + message: + 'The value is not a permissible for this field, it must meet the regular expression: "^[A-Z1-9][-_A-Z1-9]{2,7}(-[A-Z][A-Z])$". Examples: PACA-CA, BASHAR-LA', + }); + }); + + it('should validate range', () => { + const result = schemaService.processRecords(dictionary, 'address', [ + { + country: 'US', + postal_code: '12345', + unit_number: '-1', + }, + { + country: 'US', + postal_code: '12345', + unit_number: '223', + }, + { + country: 'US', + postal_code: '12345', + unit_number: '500000', + }, + ]); + + const info1 = { + exclusiveMax: 999, + min: 0, + value: [-1], + }; + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_BY_RANGE, + fieldName: 'unit_number', + index: 0, + info: info1, + message: `Value is out of permissible range, it must be ${rangeToSymbol(info1)}.`, + }); + + const info2 = { + exclusiveMax: 999, + min: 0, + value: [500000], + }; + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_BY_RANGE, + fieldName: 'unit_number', + index: 2, + info: info2, + message: `Value is out of permissible range, it must be ${rangeToSymbol(info2)}.`, + }); + }); + + it('should validate script', () => { + const result = schemaService.processRecords(dictionary, 'address', [ + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + ]); + + chai.expect(result.validationErrors.length).to.eq(2); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_BY_SCRIPT, + fieldName: 'postal_code', + index: 0, + info: { message: 'invalid postal code for US', value: '12' }, + message: 'invalid postal code for US', + }); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_BY_SCRIPT, + fieldName: 'postal_code', + index: 1, + info: { message: 'invalid postal code for CANADA', value: 'ABC' }, + message: 'invalid postal code for CANADA', + }); + }); + + it('should validate if non-required feilds are not provided', () => { + const result = schemaService.processRecords(dictionary, 'donor', [ + // optional enum field not provided + { + program_id: 'PACA-AU', + submitter_donor_id: 'ICGC_0004', + gender: 'Female', + ethnicity: 'black or african american', + vital_status: 'alive', + }, + // optional enum field provided with proper value + { + program_id: 'PACA-AU', + submitter_donor_id: 'ICGC_0002', + gender: 'Male', + ethnicity: 'asian', + vital_status: 'deceased', + cause_of_death: 'died of cancer', + survival_time: '124', + }, + // optional enum field provided with no value + { + program_id: 'PACA-AU', + submitter_donor_id: 'ICGC_0002', + gender: 'Male', + ethnicity: 'asian', + vital_status: 'deceased', + cause_of_death: '', + survival_time: '124', + }, + ]); + chai.expect(result.validationErrors.length).to.eq(0); + }); + + it('should error if integer fields are not valid', () => { + const result = schemaService.processRecords(dictionary, 'donor', [ + { + program_id: 'PACA-AU', + submitter_donor_id: 'ICGC_0002', + gender: 'Other', + ethnicity: 'asian', + vital_status: 'deceased', + cause_of_death: 'died of cancer', + survival_time: '0.5', + }, + ]); + + chai.expect(result.validationErrors.length).to.eq(1); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_FIELD_VALUE_TYPE, + fieldName: 'survival_time', + index: 0, + info: { value: ['0.5'] }, + message: VALUE_NOT_ALLOWED, + }); + }); + + it('should validate case insensitive enums, return proper format', () => { + const result = schemaService.processRecords(dictionary, 'registration', [ + { + program_id: 'PACA-AU', + submitter_donor_id: 'OD1234', + gender: 'feMale', + submitter_specimen_id: '87813', + specimen_type: 'sKiN', + tumour_normal_designation: 'Normal', + submitter_sample_id: 'MAS123', + sample_type: 'CTdna', + }, + ]); + chai.expect(result.validationErrors.length).to.eq(0); + chai.expect(result.processedRecords[0]).to.deep.eq({ + program_id: 'PACA-AU', + submitter_donor_id: 'OD1234', + gender: 'Female', + submitter_specimen_id: '87813', + specimen_type: 'Skin', + tumour_normal_designation: 'Normal', + submitter_sample_id: 'MAS123', + sample_type: 'ctDNA', + }); + }); + + it('should not validate if unrecognized fields are provided', () => { + const result = schemaService.processRecords(dictionary, 'donor', [ + { + program_id: 'PACA-AU', + submitter_donor_id: 'ICGC_0002', + gender: 'Other', + ethnicity: 'asian', + vital_status: 'deceased', + cause_of_death: 'died of cancer', + survival_time: '5', + hackField: 'muchHack', + }, + ]); + chai.expect(result.validationErrors.length).to.eq(1); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.UNRECOGNIZED_FIELD, + message: `hackField is not an allowed field for this schema.`, + fieldName: 'hackField', + index: 0, + info: {}, + }); + }); + + it('should validate number/integer array with field defined ranges', () => { + const result = schemaService.processRecords(dictionary, 'favorite_things', [ + { + id: 'TH-ING', + fraction: ['0.2', '2', '3'], + integers: ['-100', '-2'], + }, + ]); + + chai.expect(result.validationErrors.length).to.eq(2); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_BY_RANGE, + message: 'Value is out of permissible range, it must be > 0 and <= 1.', + index: 0, + fieldName: 'fraction', + info: { value: [2, 3], max: 1, exclusiveMin: 0 }, + }); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_BY_RANGE, + message: 'Value is out of permissible range, it must be >= -10 and <= 10.', + index: 0, + fieldName: 'integers', + info: { value: [-100], max: 10, min: -10 }, + }); + }); + + it('should validate string array with field defined codelist', () => { + const result = schemaService.processRecords(dictionary, 'favorite_things', [ + { + id: 'TH-ING', + fruit: ['Mango', '2'], + }, + ]); + chai.expect(result.validationErrors.length).to.eq(1); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_ENUM_VALUE, + message: 'The value is not permissible for this field.', + fieldName: 'fruit', + index: 0, + info: { value: ['2'] }, + }); + }); + + it('should validate string with field defined codelist', () => { + const result = schemaService.processRecords(dictionary, 'favorite_things', [ + { + id: 'TH-ING', + fruit_single_value: 'Banana', + }, + ]); + chai.expect(result.validationErrors.length).to.eq(1); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_ENUM_VALUE, + message: 'The value is not permissible for this field.', + fieldName: 'fruit_single_value', + index: 0, + info: { value: ['Banana'] }, + }); + }); + + it('should validate string array with field defined regex', () => { + const result = schemaService.processRecords(dictionary, 'favorite_things', [ + { + id: 'TH-ING', + qWords: ['que', 'not_q'], + }, + ]); + chai.expect(result.validationErrors.length).to.eq(1); + chai.expect(result.validationErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_REGEX, + message: 'The value is not a permissible for this field, it must meet the regular expression: "^q.*$".', + fieldName: 'qWords', + index: 0, + info: { value: ['not_q'], regex: '^q.*$', examples: undefined }, + }); + }); + + it('should validate string with field defined regex', () => { + const result = schemaService.processRecords(dictionary, 'favorite_things', [ + { + id: 'TH-ING', + qWord: 'not_q', + }, + ]); + chai.expect(result.validationErrors.length).to.eq(1); + chai.expect(result.validationErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_REGEX, + message: 'The value is not a permissible for this field, it must meet the regular expression: "^q.*$".', + fieldName: 'qWord', + index: 0, + info: { value: ['not_q'], regex: '^q.*$', examples: undefined }, + }); + }); + + it('should pass unique restriction validation when only null values exists', () => { + const result = schemaService.processRecords(dictionary, 'favorite_things', [ + { + id: 'TH-ING', + unique_value: '', + }, + ]); + chai.expect(result.validationErrors.length).to.eq(0); + }); + + it('should pass unique restriction validation when only one record exists', () => { + const result = schemaService.processRecords(dictionary, 'favorite_things', [ + { + id: 'TH-ING', + unique_value: 'unique_value_1', + }, + ]); + chai.expect(result.validationErrors.length).to.eq(0); + }); + + it('should fail unique restriction validation when duplicate values exist (scalar)', () => { + const result = schemaService.processRecords(dictionary, 'favorite_things', [ + { + id: 'ID-1', + unique_value: 'unique_value_1', + }, + { + id: 'ID-2', + unique_value: 'unique_value_1', + }, + ]); + + chai.expect(result.validationErrors.length).to.eq(2); + chai.expect(result.validationErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE, + message: 'Values for column "unique_value" must be unique.', + fieldName: 'unique_value', + index: 0, + info: { value: ['unique_value_1'] }, + }); + chai.expect(result.validationErrors[1]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE, + message: 'Values for column "unique_value" must be unique.', + fieldName: 'unique_value', + index: 1, + info: { value: ['unique_value_1'] }, + }); + }); + it('should fail unique restriction validation when duplicate values exist (array)', () => { + const result = schemaService.processRecords(dictionary, 'favorite_things', [ + { + id: 'ID-1', + unique_value: ['unique_value_1', 'unique_value_2'], + }, + { + id: 'ID-2', + unique_value: ['unique_value_1', 'unique_value_2'], + }, + ]); + + chai.expect(result.validationErrors.length).to.eq(2); + chai.expect(result.validationErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE, + message: 'Values for column "unique_value" must be unique.', + fieldName: 'unique_value', + index: 0, + info: { value: ['unique_value_1', 'unique_value_2'] }, + }); + chai.expect(result.validationErrors[1]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE, + message: 'Values for column "unique_value" must be unique.', + fieldName: 'unique_value', + index: 1, + info: { value: ['unique_value_1', 'unique_value_2'] }, + }); + }); + + it('should pass foreignKey restriction validation when values exist in foreign schema', () => { + const parent_schema_1_data = [ + { + id: 'parent_schema_1_id_1', + name: 'parent_schema_1_name_1', + }, + { + id: 'parent_schema_1_id_2', + name: 'parent_schema_1_name_2', + }, + ]; + + const child_schema_simple_fk_data = [ + { + id: '1', + parent_schema_1_id: 'parent_schema_1_id_1', + }, + { + id: '2', + parent_schema_1_id: 'parent_schema_1_id_2', + }, + ]; + const schemaData = { + parent_schema_1: parent_schema_1_data, + child_schema_simple_fk: child_schema_simple_fk_data, + }; + + const result = schemaService.processSchemas(dictionary, schemaData); + + chai.expect(result['parent_schema_1'].validationErrors.length).to.eq(0); + chai.expect(result['child_schema_simple_fk'].validationErrors.length).to.eq(0); + }); + + it('should pass foreignKey restriction validation when local schema has null values', () => { + const parent_schema_1_data = [ + { + id: 'parent_schema_1_id_1', + name: 'parent_schema_1_name_1', + }, + { + id: 'parent_schema_1_id_2', + name: 'parent_schema_1_name_2', + }, + ]; + + const child_schema_simple_fk_data = [ + { + id: '1', + parent_schema_1_id: 'parent_schema_1_id_1', + }, + { + id: '2', + parent_schema_1_id: '', + }, + ]; + const schemaData = { + parent_schema_1: parent_schema_1_data, + child_schema_simple_fk: child_schema_simple_fk_data, + }; + + const result = schemaService.processSchemas(dictionary, schemaData); + + chai.expect(result['parent_schema_1'].validationErrors.length).to.eq(0); + chai.expect(result['child_schema_simple_fk'].validationErrors.length).to.eq(0); + }); + + it('should pass foreignKey restriction validation when values exist in foreign schema (composite fk)', () => { + const parent_schema_1_data = [ + { + id: 'parent_schema_1_id_1', + external_id: 'parent_schema_1_external_id_1', + name: 'parent_schema_1_name_1', + }, + { + id: 'parent_schema_1_id_2', + external_id: 'parent_schema_1_external_id_2', + name: 'parent_schema_1_name_2', + }, + ]; + + const child_schema_composite_fk_data = [ + { + id: '1', + parent_schema_1_id: 'parent_schema_1_id_1', + parent_schema_1_external_id: 'parent_schema_1_external_id_1', + }, + { + id: '2', + parent_schema_1_id: 'parent_schema_1_id_2', + parent_schema_1_external_id: 'parent_schema_1_external_id_2', + }, + ]; + const schemaData = { + parent_schema_1: parent_schema_1_data, + child_schema_composite_fk: child_schema_composite_fk_data, + }; + + const result = schemaService.processSchemas(dictionary, schemaData); + + chai.expect(result['parent_schema_1'].validationErrors.length).to.eq(0); + chai.expect(result['child_schema_composite_fk'].validationErrors.length).to.eq(0); + }); + + it('should fail foreignKey restriction validation when value does not exist in foreign schema', () => { + const parent_schema_1_data = [ + { + id: 'parent_schema_1_id_1', + name: 'parent_schema_1_name_1', + }, + { + id: 'parent_schema_1_id_2', + name: 'parent_schema_1_name_2', + }, + ]; + + const child_schema_simple_fk_data = [ + { + id: '1', + parent_schema_1_id: 'parent_schema_1_id_1', + }, + { + id: '2', + parent_schema_1_id: 'non_existing_value_in_foreign_schema', + }, + ]; + const schemaData = { + parent_schema_1: parent_schema_1_data, + child_schema_simple_fk: child_schema_simple_fk_data, + }; + + const result = schemaService.processSchemas(dictionary, schemaData); + const childSchemaErrors = result['child_schema_simple_fk'].validationErrors; + + chai.expect(childSchemaErrors.length).to.eq(1); + chai.expect(childSchemaErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_FOREIGN_KEY, + message: + 'Record violates foreign key restriction defined for field(s) parent_schema_1_id. Key parent_schema_1_id: non_existing_value_in_foreign_schema is not present in schema parent_schema_1.', + fieldName: 'parent_schema_1_id', + index: 1, + info: { foreignSchema: 'parent_schema_1', value: { parent_schema_1_id: 'non_existing_value_in_foreign_schema' } }, + }); + }); + + it('should fail foreignKey restriction validation when values do not exist in foreign schema (composite fk)', () => { + const parent_schema_1_data = [ + { + id: 'parent_schema_1_id_1', + external_id: 'parent_schema_1_external_id_1', + name: 'parent_schema_1_name_1', + }, + { + id: 'parent_schema_1_id_2', + external_id: 'parent_schema_1_external_id_2', + name: 'parent_schema_1_name_2', + }, + ]; + + const child_schema_composite_fk_data = [ + { + id: '1', + parent_schema_1_id: 'parent_schema_1_id_1', + parent_schema_1_external_id: 'parent_schema_1_external_id_1', + }, + { + id: '2', + parent_schema_1_id: 'parent_schema_1_id_2', + parent_schema_1_external_id: 'non_existing_value_in_foreign_schema', + }, + ]; + const schemaData = { + parent_schema_1: parent_schema_1_data, + child_schema_composite_fk: child_schema_composite_fk_data, + }; + + const result = schemaService.processSchemas(dictionary, schemaData); + const childSchemaErrors = result['child_schema_composite_fk'].validationErrors; + + chai.expect(childSchemaErrors.length).to.eq(1); + chai.expect(childSchemaErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_FOREIGN_KEY, + message: + 'Record violates foreign key restriction defined for field(s) parent_schema_1_id, parent_schema_1_external_id. Key parent_schema_1_id: parent_schema_1_id_2, parent_schema_1_external_id: non_existing_value_in_foreign_schema is not present in schema parent_schema_1.', + fieldName: 'parent_schema_1_id, parent_schema_1_external_id', + index: 1, + info: { + foreignSchema: 'parent_schema_1', + value: { + parent_schema_1_external_id: 'non_existing_value_in_foreign_schema', + parent_schema_1_id: 'parent_schema_1_id_2', + }, + }, + }); + }); + + it('should fail foreignKey restriction validation when values (array) do not match in foreign schema (composite fk)', () => { + const parent_schema_2_data = [ + { + id1: ['id1_1', 'id1_2'], + id2: ['id2_1'], + }, + ]; + + const child_schema_composite_array_values_fk_data = [ + { + id: '1', + parent_schema_2_id1: ['id1_1'], + parent_schema_2_id12: ['id1_2', 'id2_1'], + }, + ]; + const schemaData = { + parent_schema_2: parent_schema_2_data, + child_schema_composite_array_values_fk: child_schema_composite_array_values_fk_data, + }; + + const result = schemaService.processSchemas(dictionary, schemaData); + const childSchemaErrors = result['child_schema_composite_array_values_fk'].validationErrors; + + chai.expect(childSchemaErrors.length).to.eq(1); + chai.expect(childSchemaErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_FOREIGN_KEY, + message: + 'Record violates foreign key restriction defined for field(s) parent_schema_2_id1, parent_schema_2_id12. Key parent_schema_2_id1: [id1_1], parent_schema_2_id12: [id1_2, id2_1] is not present in schema parent_schema_2.', + fieldName: 'parent_schema_2_id1, parent_schema_2_id12', + index: 0, + info: { + foreignSchema: 'parent_schema_2', + value: { + parent_schema_2_id1: ['id1_1'], + parent_schema_2_id12: ['id1_2', 'id2_1'], + }, + }, + }); + }); + + it('should pass uniqueKey restriction validation when only a record exists', () => { + const result = schemaService.processRecords(dictionary, 'unique_key_schema', [ + { + numeric_id_1: '1', + string_id_2: 'string_value', + array_string_id_3: ['array_element_1', 'array_element_2'], + }, + ]); + + chai.expect(result.validationErrors.length).to.eq(0); + }); + + it('should pass uniqueKey restriction validation when values are unique', () => { + const result = schemaService.processRecords(dictionary, 'unique_key_schema', [ + { + numeric_id_1: '1', + string_id_2: 'string_value', + array_string_id_3: ['array_element_1', 'array_element_2'], + }, + { + numeric_id_1: '1', + string_id_2: 'string_value', + array_string_id_3: ['array_element_1', 'array_element_x'], + }, + ]); + + chai.expect(result.validationErrors.length).to.eq(0); + }); + + it('should fail uniqueKey restriction validation when missing values are part of the key and they are not unique', () => { + const result = schemaService.processRecords(dictionary, 'unique_key_schema', [ + { + numeric_id_1: '', + string_id_2: '', + array_string_id_3: [], + }, + { + numeric_id_1: '', + string_id_2: '', + array_string_id_3: [], + }, + ]); + + chai.expect(result.validationErrors.length).to.eq(2); + chai.expect(result.validationErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, + message: + 'UniqueKey field values "numeric_id_1: null, string_id_2: null, array_string_id_3: null" must be unique.', + fieldName: 'numeric_id_1, string_id_2, array_string_id_3', + index: 0, + info: { + uniqueKeyFields: ['numeric_id_1', 'string_id_2', 'array_string_id_3'], + value: { + numeric_id_1: '', + string_id_2: '', + array_string_id_3: '', + }, + }, + }); + chai.expect(result.validationErrors[1]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, + message: + 'UniqueKey field values "numeric_id_1: null, string_id_2: null, array_string_id_3: null" must be unique.', + fieldName: 'numeric_id_1, string_id_2, array_string_id_3', + index: 1, + info: { + uniqueKeyFields: ['numeric_id_1', 'string_id_2', 'array_string_id_3'], + value: { + numeric_id_1: '', + string_id_2: '', + array_string_id_3: '', + }, + }, + }); + }); + + it('should fail uniqueKey restriction validation when values are not unique', () => { + const result = schemaService.processRecords(dictionary, 'unique_key_schema', [ + { + numeric_id_1: '1', + string_id_2: 'string_value', + array_string_id_3: ['array_element_1', 'array_element_2'], + }, + { + numeric_id_1: '1', + string_id_2: 'string_value', + array_string_id_3: ['array_element_1', 'array_element_2'], + }, + ]); + + chai.expect(result.validationErrors.length).to.eq(2); + chai.expect(result.validationErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, + message: + 'UniqueKey field values "numeric_id_1: 1, string_id_2: string_value, array_string_id_3: [array_element_1, array_element_2]" must be unique.', + fieldName: 'numeric_id_1, string_id_2, array_string_id_3', + index: 0, + info: { + uniqueKeyFields: ['numeric_id_1', 'string_id_2', 'array_string_id_3'], + value: { + numeric_id_1: 1, + string_id_2: 'string_value', + array_string_id_3: ['array_element_1', 'array_element_2'], + }, + }, + }); + chai.expect(result.validationErrors[1]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, + message: + 'UniqueKey field values "numeric_id_1: 1, string_id_2: string_value, array_string_id_3: [array_element_1, array_element_2]" must be unique.', + fieldName: 'numeric_id_1, string_id_2, array_string_id_3', + index: 1, + info: { + uniqueKeyFields: ['numeric_id_1', 'string_id_2', 'array_string_id_3'], + value: { + numeric_id_1: 1, + string_id_2: 'string_value', + array_string_id_3: ['array_element_1', 'array_element_2'], + }, + }, + }); + }); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 48ca307..279f9a5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -197,15 +197,15 @@ importers: deep-freeze: specifier: ^0.0.1 version: 0.0.1 + dictionary: + specifier: workspace:^ + version: link:../../libraries/dictionary lodash: specifier: ^4.17.21 version: 4.17.21 node-fetch: specifier: ^2.6.1 version: 2.7.0 - node-worker-threads-pool: - specifier: ^1.4.3 - version: 1.5.1 promise-tools: specifier: ^2.1.0 version: 2.1.0 @@ -3384,10 +3384,6 @@ packages: - supports-color dev: false - /node-worker-threads-pool@1.5.1: - resolution: {integrity: sha512-7TXAhpMm+jO4MfESxYLtMGSnJWv+itdNHMdaFmeZuPXxwFGU90mtEB42BciUULXOUAxYBfXILAuvrSG3rQZ7mw==} - dev: false - /nodemon@2.0.22: resolution: {integrity: sha512-B8YqaKMmyuCO7BowF1Z1/mkPqLk6cs/l63Ojtd6otKjMx47Dq1utxfRxcavH1I7VSaL8n5BUaoutadnsX3AAVQ==} engines: {node: '>=8.10.0'} From c783f64d530c6da1685eefb609fa89515e5ee1d1 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Sat, 15 Jun 2024 13:19:08 -0400 Subject: [PATCH 07/27] Move data validation functions into shared module (#212) * Create validation library * Move Data Record types to lectern-dictionary * Add regexp for boolean values for use in type validation * Move Validation functions into shared module * Convenience scripts for test/build for each sub-module * Adding references to validaiton library * Correct version regexp, add comments for string validations * Move validation pipelines alongside client processing code --- README.md | 25 +++++--- libraries/common/README.md | 2 +- .../dictionary/src/types/dataTypes.ts | 23 +++++-- .../dictionary/src/types/dictionaryTypes.ts | 28 ++++++++- libraries/dictionary/src/types/index.ts | 1 + libraries/dictionary/test/dataTypes.spec.ts | 63 +++++++++++++++++++ libraries/validation/.mocharc.json | 5 ++ libraries/validation/README.md | 9 +++ libraries/validation/package.json | 24 +++++++ .../fieldNamesValidation.ts | 0 .../valueTypeValidation.ts | 48 +++++++------- .../fieldRestrictions/codeListValidation.ts | 26 +++++--- .../src}/fieldRestrictions/rangeValidation.ts | 11 ++-- .../src}/fieldRestrictions/regexValidation.ts | 18 +++--- .../fieldRestrictions/requiredValidation.ts | 19 +++--- .../fieldRestrictions/scriptValidation.ts | 11 ++-- .../validation/src}/index.ts | 1 - .../foreignKeysValidation.ts | 2 +- .../schemaRestrictions/uniqueKeyValidation.ts | 0 .../schemaRestrictions/uniqueValidation.ts | 0 .../validation/src}/types/index.ts | 0 .../src}/types/validationErrorTypes.ts | 3 +- .../src}/types/validationFunctionTypes.ts | 3 +- .../validation/src}/utils/datasetUtils.ts | 2 +- .../validation/src/utils/isEmptyString.ts | 7 ++- .../validation/src}/utils/rangeToSymbol.ts | 0 .../validation/src/utils/typeUtils.ts | 39 +++++++----- libraries/validation/test/placeholder.spec.ts | 3 + libraries/validation/tsconfig.json | 20 ++++++ package.json | 10 +++ packages/client/README.md | 5 +- packages/client/package.json | 1 + packages/client/src/index.ts | 3 - .../src/processing/convertDataValueTypes.ts | 12 +++- packages/client/src/processing/index.ts | 19 +++--- .../src/processing/processingResultTypes.ts | 5 +- .../validationPipelines.ts | 7 +-- packages/client/test/processing.spec.ts | 2 +- packages/client/test/validation.spec.ts | 9 ++- pnpm-lock.yaml | 25 ++++++++ 40 files changed, 359 insertions(+), 132 deletions(-) rename packages/client/src/types/dataRecords.ts => libraries/dictionary/src/types/dataTypes.ts (77%) create mode 100644 libraries/dictionary/test/dataTypes.spec.ts create mode 100644 libraries/validation/.mocharc.json create mode 100644 libraries/validation/README.md create mode 100644 libraries/validation/package.json rename {packages/client/src/validation => libraries/validation/src}/dataRecordValidation/fieldNamesValidation.ts (100%) rename {packages/client/src/validation => libraries/validation/src}/dataRecordValidation/valueTypeValidation.ts (74%) rename {packages/client/src/validation => libraries/validation/src}/fieldRestrictions/codeListValidation.ts (77%) rename {packages/client/src/validation => libraries/validation/src}/fieldRestrictions/rangeValidation.ts (93%) rename {packages/client/src/validation => libraries/validation/src}/fieldRestrictions/regexValidation.ts (88%) rename {packages/client/src/validation => libraries/validation/src}/fieldRestrictions/requiredValidation.ts (82%) rename {packages/client/src/validation => libraries/validation/src}/fieldRestrictions/scriptValidation.ts (93%) rename {packages/client/src/validation => libraries/validation/src}/index.ts (97%) rename {packages/client/src/validation => libraries/validation/src}/schemaRestrictions/foreignKeysValidation.ts (99%) rename {packages/client/src/validation => libraries/validation/src}/schemaRestrictions/uniqueKeyValidation.ts (100%) rename {packages/client/src/validation => libraries/validation/src}/schemaRestrictions/uniqueValidation.ts (100%) rename {packages/client/src/validation => libraries/validation/src}/types/index.ts (100%) rename {packages/client/src/validation => libraries/validation/src}/types/validationErrorTypes.ts (97%) rename {packages/client/src/validation => libraries/validation/src}/types/validationFunctionTypes.ts (94%) rename {packages/client/src/validation => libraries/validation/src}/utils/datasetUtils.ts (98%) rename packages/client/src/types/index.ts => libraries/validation/src/utils/isEmptyString.ts (84%) rename {packages/client/src/validation => libraries/validation/src}/utils/rangeToSymbol.ts (100%) rename packages/client/src/validation/utils/fieldTypeUtils.ts => libraries/validation/src/utils/typeUtils.ts (57%) create mode 100644 libraries/validation/test/placeholder.spec.ts create mode 100644 libraries/validation/tsconfig.json rename packages/client/src/{validation => processing}/validationPipelines.ts (91%) diff --git a/README.md b/README.md index f03434b..0614fcc 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,8 @@ The repository is organized with the following directory structure: │ └── server ├── libraries/ │ ├── common -│ └── dictionary +│ ├── dictionary +│ └── validation └── packages/ └── client ``` @@ -39,9 +40,10 @@ The modules in the monorepo are organized into three categories: | Component | Type | Package Name | Path | Published Location | Description | | -------------------------------------------- | ----------- | ------------------------------ | --------------------- | ------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | [Lectern Server](apps/server/README.md) | Application | server | apps/server/ | [GHCR](https://github.com/overture-stack/lectern/pkgs/container/lectern) | Lectern Server web application. | -| [Lectern Client](packages/client/README.md) | Package | @overture-stack/lectern-client | packages/client | [NPM](https://www.npmjs.com/package/@overturebio-stack/lectern-client) | TypeScript Client to interact with Lectern Server and perform data validation. | +| [Lectern Client](packages/client/README.md) | Package | @overture-stack/lectern-client | packages/client | [NPM](https://www.npmjs.com/package/@overture-stack/lectern-client) | TypeScript Client to interact with Lectern Server and perform data validation. | | [common](libraries/common/README.md) | Library | common | libraries/common/ | N/A | Non-specific but commonly reusable utilities. Includes shared Error classes. | | [dictionary](libraries/dictionary/README.md) | Library | dictionary | libraries/dictionary/ | N/A | Dictionary meta-schema definition, includes TS types, and Zod schemas. This also exports all utilities for getting the diff of two dictionaries, and for validating data records with a Dictionary. | +| [validation](libraries/validation/README.md) | Library | @overture-stack/lectern-validation | libraries/validation/ | [NPM](https://www.npmjs.com/package/@overture-stack/lectern-validation) | Validate data using Lectern Dictionaries. | ## Developer Instructions @@ -55,13 +57,26 @@ Using `nx` will ensure all local dependencies are built, in the correct sequence This will ensure that all dependencies of `server` are built in correct order before the `server` build is run. -To work with any module in this repository, follow the instructions in the README provide in the module directory. +Note that the full name of from the package must be used for this to work, so for the client the command would be: + +`pnpm nx build @overture-stack/lectern-client` + +For convenience, scripts have been added to the root level [`package.json`](./package.json) to run `build` and `test` scripts for every service using short names. These follow the pattern `pnpm :`. For example, the same build command can be performed by: + +`pnpm build:client` + +To work with any module in this repository, follow the instructions in the README provide in that module's directory. Get started by running the [Lecter Server application](apps/server/README.md). ### Common Commands A few commonly reused scripts have been added to the root `package.json`. Run them from the root directory, or if you are in a sub directory then use `pnpm -w`. + +For example, when your current working directory is not in the project root, you can still conveniently test every module in the monorepo with the command: + +`pnpm -w test:all` + #### Build Everything `pnpm build:all` @@ -88,10 +103,6 @@ This can be used as a programing language agnostic schema for external applicati > > Don't manually update any files in the `./generated` path. This content is programatically generated from the source code. -### Sample Dictionaries - - - ## Support & Contributions We welcome community contributions! Please follow our [code of conduct](./code_of_conduct.md) diff --git a/libraries/common/README.md b/libraries/common/README.md index 3bf0e36..e9c1247 100644 --- a/libraries/common/README.md +++ b/libraries/common/README.md @@ -1,5 +1,5 @@ # Lectern Common Utils -Contains generic types and utilities used across all modules. +Contains generic types and utilities used across Lectern modules. Also exported are custom error classes. Please note: it is prefered to return Result objects with typed error details rather than throw unexpectedly. Use of thrown errors is being refactored out where possible. \ No newline at end of file diff --git a/packages/client/src/types/dataRecords.ts b/libraries/dictionary/src/types/dataTypes.ts similarity index 77% rename from packages/client/src/types/dataRecords.ts rename to libraries/dictionary/src/types/dataTypes.ts index 43adfda..8ce6254 100644 --- a/packages/client/src/types/dataRecords.ts +++ b/libraries/dictionary/src/types/dataTypes.ts @@ -21,9 +21,7 @@ * Represents a data record as taken from an input file. All values are the original strings and have not been validated into * numbers/bools or split into arrays. */ -export type UnprocessedDataRecord = { - [k: string]: string | string[]; -}; +export type UnprocessedDataRecord = Record; /** * The available data types for a field in a Lectern Schema. @@ -34,6 +32,19 @@ export type DataRecordValue = string | string[] | number | number[] | boolean | * Represents a data record after processing, with the data checked to be a valid type for a Lectern schema. * The type of data should match the expected type for the given field. */ -export type DataRecord = { - [key: string]: DataRecordValue; -}; +export type DataRecord = Record; + +/** + * RegExp to check if a string value represents a boolean value. + * This allows for case insensitive text (true, True, TRUE are all valid) and will ignore leading + * and trailing whitespace. + * + * Example Values: + * - `true` + * - `True` + * - `TRUE` + * - `false` + * - `False` + * - `FALSE` + */ +export const REGEXP_BOOLEAN_VALUE = /^[\s]*(true|false)[\s]*$/i; diff --git a/libraries/dictionary/src/types/dictionaryTypes.ts b/libraries/dictionary/src/types/dictionaryTypes.ts index e7bcd5e..6d0cfca 100644 --- a/libraries/dictionary/src/types/dictionaryTypes.ts +++ b/libraries/dictionary/src/types/dictionaryTypes.ts @@ -21,10 +21,19 @@ import { z as zod } from 'zod'; import allUnique from '../utils/allUnique'; import { ReferenceTag, References } from './referenceTypes'; +/** + * String rules for all name fields used in dictionary, including Dictionary, Schema, and Fields. + * This validates the format of the string since names are not allowed to have `.` characters. + * + * Example Values: + * - `donors` + * - `primary-site` + * - `maximumVelocity` + */ export const NameString = zod .string() .min(1, 'Name fields cannot be empty.') - .regex(RegExp('^[^.]+$'), 'Name fields cannot have `.` characters.'); + .regex(/^[^.]+$/, 'Name fields cannot have `.` characters.'); export type NameString = zod.infer; export const Integer = zod.number().int(); @@ -283,11 +292,24 @@ export type Schema = zod.infer; /* ********** * * Dictionary * * ********** */ - -export const VersionString = zod.string().regex(RegExp('^[0-9]+.[0-9]+$')); +/** + * Validation rules for dictionary version field. The dictionary version is a string with two numbers, + * a major and minor version. Minor dictionary versions are meant to be backwards compatible with all + * earlier dictionaries of the same Major version. This is done by making the changes additive for minor + * changes. Changes that break backwards compatibility of a dictionary should be given a new Major version. + * + * Example Values: + * - `0.0` + * - `1.0` + * - `1.1` + * - `23.45` + */ +export const VersionString = zod.string().regex(/^([0-9]+)\.[0-9]+$/); // TODO: Semantic Versioning version string, reference: https://gist.github.com/jhorsman/62eeea161a13b80e39f5249281e17c39 // update requires dictionary service changes, leaving like this for now // .regex(RegExp('^([0-9]+).([0-9]+).([0-9]+)(?:-([0-9A-Za-z-]+(?:.[0-9A-Za-z-]+)*))?(?:+[0-9A-Za-z-]+)?$')); +// This is a valid regexp for semantic versions, but we should instead use the npmjs semantic versioning library +// https://www.npmjs.com/package/semver // Dictionary Base is the dictionary object types only, no refinements enforcing the restriction rules. // This is needed to use for the base of the dbType DictionaryDocument diff --git a/libraries/dictionary/src/types/index.ts b/libraries/dictionary/src/types/index.ts index 055976f..a946640 100644 --- a/libraries/dictionary/src/types/index.ts +++ b/libraries/dictionary/src/types/index.ts @@ -1,3 +1,4 @@ +export * from './dataTypes'; export * from './dbTypes'; export * from './dictionaryTypes'; export * from './diffTypes'; diff --git a/libraries/dictionary/test/dataTypes.spec.ts b/libraries/dictionary/test/dataTypes.spec.ts new file mode 100644 index 0000000..9584c38 --- /dev/null +++ b/libraries/dictionary/test/dataTypes.spec.ts @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2023 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { expect } from 'chai'; +import { REGEXP_BOOLEAN_VALUE } from '../src'; + +describe('Data Types', () => { + describe('Boolean RegExp', () => { + it('Matches case insensitive values of `true` and `false`', () => { + expect('true'.match(REGEXP_BOOLEAN_VALUE)).to.not.be.null; + expect('True'.match(REGEXP_BOOLEAN_VALUE)).to.not.be.null; + expect('TRUE'.match(REGEXP_BOOLEAN_VALUE)).to.not.be.null; + expect('false'.match(REGEXP_BOOLEAN_VALUE)).to.not.be.null; + expect('False'.match(REGEXP_BOOLEAN_VALUE)).to.not.be.null; + expect('FALSE'.match(REGEXP_BOOLEAN_VALUE)).to.not.be.null; + }); + it('Matches strings with leading and trailing whitespace', () => { + // leading spaces + expect(' true'.match(REGEXP_BOOLEAN_VALUE)).to.not.be.null; + + // trailing tab + expect('True '.match(REGEXP_BOOLEAN_VALUE)).to.not.be.null; + + // trailing new line + expect( + 'TRUE\ + '.match(REGEXP_BOOLEAN_VALUE), + ).to.not.be.null; + + // leading and trailing spaces + expect(' false '.match(REGEXP_BOOLEAN_VALUE)).to.not.be.null; + }); + it('Rejects empty strings', () => { + expect(''.match(REGEXP_BOOLEAN_VALUE)).to.be.null; + }); + it('Rejects typos and invalid values', () => { + expect('ttrue'.match(REGEXP_BOOLEAN_VALUE)).to.be.null; + expect('Truee'.match(REGEXP_BOOLEAN_VALUE)).to.be.null; + expect('asdf'.match(REGEXP_BOOLEAN_VALUE)).to.be.null; + expect('tr ue'.match(REGEXP_BOOLEAN_VALUE)).to.be.null; + expect('true true'.match(REGEXP_BOOLEAN_VALUE)).to.be.null; + expect('true|false'.match(REGEXP_BOOLEAN_VALUE)).to.be.null; + expect('true,true'.match(REGEXP_BOOLEAN_VALUE)).to.be.null; + expect('text with true in the middle'.match(REGEXP_BOOLEAN_VALUE)).to.be.null; + }); + }); +}); diff --git a/libraries/validation/.mocharc.json b/libraries/validation/.mocharc.json new file mode 100644 index 0000000..6d0022d --- /dev/null +++ b/libraries/validation/.mocharc.json @@ -0,0 +1,5 @@ +{ + "extension": ["ts"], + "require": "ts-node/register", + "spec": "test/**/*.spec.ts" +} diff --git a/libraries/validation/README.md b/libraries/validation/README.md new file mode 100644 index 0000000..710b04d --- /dev/null +++ b/libraries/validation/README.md @@ -0,0 +1,9 @@ +# Lectern Validation + +[![NPM Version](https://img.shields.io/npm/v/@overture-stack/lectern-validation?color=%23cb3837&style=for-the-badge&logo=npm)](https://www.npmjs.com/package/@overture-stack/lectern-validation) + +Standalone library to validate data using a Lectern Dictionary. + +> **Note**: +> This may not be the module you want to import, it produced to be used as a shared dependency inside the Lectern monorepo. If you are building an application that will interact with a Lectern Server consider using the [Lectern Client](https://www.npmjs.com/package/@overture-stack/lectern-client) which includes this libary as a dependency while also providing REST Client functionality. + diff --git a/libraries/validation/package.json b/libraries/validation/package.json new file mode 100644 index 0000000..e36623c --- /dev/null +++ b/libraries/validation/package.json @@ -0,0 +1,24 @@ +{ + "name": "@overture-stack/lectern-validation", + "version": "0.1.0", + "description": "Logic for validating data using a Lectern dictionary", + "main": "dist/index.js", + "scripts": { + "build": "pnpm build:clean && tsc", + "build:clean": "rimraf dist/ && mkdir dist", + "test": "nyc mocha" + }, + "keywords": [], + "author": "Ontario Institute for Cancer Research", + "license": "AGPL-3.0", + "dependencies": { + "common": "workspace:^", + "dictionary": "workspace:^", + "lodash": "^4.17.21", + "zod": "^3.21.4" + }, + "devDependencies": { + "@types/lodash": "^4.14.195", + "rimraf": "^3.0.2" + } +} diff --git a/packages/client/src/validation/dataRecordValidation/fieldNamesValidation.ts b/libraries/validation/src/dataRecordValidation/fieldNamesValidation.ts similarity index 100% rename from packages/client/src/validation/dataRecordValidation/fieldNamesValidation.ts rename to libraries/validation/src/dataRecordValidation/fieldNamesValidation.ts diff --git a/packages/client/src/validation/dataRecordValidation/valueTypeValidation.ts b/libraries/validation/src/dataRecordValidation/valueTypeValidation.ts similarity index 74% rename from packages/client/src/validation/dataRecordValidation/valueTypeValidation.ts rename to libraries/validation/src/dataRecordValidation/valueTypeValidation.ts index b42304d..12d73e7 100644 --- a/packages/client/src/validation/dataRecordValidation/valueTypeValidation.ts +++ b/libraries/validation/src/dataRecordValidation/valueTypeValidation.ts @@ -17,16 +17,17 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { SchemaFieldValueType } from 'dictionary'; - -import { convertToArray, isEmpty, isEmptyString, isStringArray, notEmpty } from '../../utils'; +import { asArray } from 'common'; +import { REGEXP_BOOLEAN_VALUE, SchemaFieldValueType } from 'dictionary'; import { INVALID_VALUE_ERROR_MESSAGE, SchemaValidationErrorTypes, type BaseSchemaValidationError, type ValueTypeValidationError, } from '../types/validationErrorTypes'; -import type { UnprocessedRecordValidationFunction, ValidationFunction } from '../types/validationFunctionTypes'; +import type { UnprocessedRecordValidationFunction } from '../types/validationFunctionTypes'; +import { isEmptyString } from '../utils/isEmptyString'; +import { isDefined } from '../utils/typeUtils'; /** * Test the values provided to every field in a DataRecord to find any non-array fields that @@ -44,12 +45,12 @@ export const validateNonArrayFields: UnprocessedRecordValidationFunction = ( return schemaFields .map((field) => { const value = record[field.name]; - if (!field.isArray && isStringArray(value)) { + if (!field.isArray && Array.isArray(value)) { return buildFieldValueTypeError({ fieldName: field.name, index }, { value }); } return undefined; }) - .filter(notEmpty); + .filter(isDefined); }; /** @@ -67,12 +68,9 @@ export const validateValueTypes: UnprocessedRecordValidationFunction = ( ): ValueTypeValidationError[] => { return schemaFields .map((field) => { - if (isEmpty(record[field.name])) { - return undefined; - } - - const recordFieldValues = convertToArray(record[field.name]); // put all values into array - const invalidValues = recordFieldValues.filter((v) => v !== undefined && isInvalidFieldType(field.valueType, v)); + const invalidValues = asArray(record[field.name]).filter( + (value) => value !== undefined && !isValidFieldType(field.valueType, value), + ); const info = { value: invalidValues }; if (invalidValues.length !== 0) { @@ -80,7 +78,7 @@ export const validateValueTypes: UnprocessedRecordValidationFunction = ( } return undefined; }) - .filter(notEmpty); + .filter(isDefined); }; /** @@ -89,21 +87,27 @@ export const validateValueTypes: UnprocessedRecordValidationFunction = ( * @param value * @returns */ -const isInvalidFieldType = (valueType: SchemaFieldValueType, value: string) => { - // optional field if the value is absent at this point - if (isEmptyString(value)) return false; +const isValidFieldType = (valueType: SchemaFieldValueType, value: string): boolean => { + if (isEmptyString(value)) { + return true; + } switch (valueType) { - case SchemaFieldValueType.Values.string: - return false; + case SchemaFieldValueType.Values.boolean: + return isValidBoolean(value); case SchemaFieldValueType.Values.integer: - return !Number.isSafeInteger(Number(value)); + return isValidInteger(value); case SchemaFieldValueType.Values.number: - return isNaN(Number(value)); - case SchemaFieldValueType.Values.boolean: - return !(value.toLowerCase() === 'true' || value.toLowerCase() === 'false'); + return isValidNumber(value); + case SchemaFieldValueType.Values.string: + // input values all start as strings so this is valid in all cases + return true; } }; +const isValidInteger = (value: string): boolean => Number.isSafeInteger(Number(value)); +const isValidNumber = (value: string): boolean => isFinite(Number(value)); +const isValidBoolean = (value: string): boolean => value.trim().match(REGEXP_BOOLEAN_VALUE) !== null; + const buildFieldValueTypeError = ( errorData: BaseSchemaValidationError, info: ValueTypeValidationError['info'], diff --git a/packages/client/src/validation/fieldRestrictions/codeListValidation.ts b/libraries/validation/src/fieldRestrictions/codeListValidation.ts similarity index 77% rename from packages/client/src/validation/fieldRestrictions/codeListValidation.ts rename to libraries/validation/src/fieldRestrictions/codeListValidation.ts index 50d9a56..feec10e 100644 --- a/packages/client/src/validation/fieldRestrictions/codeListValidation.ts +++ b/libraries/validation/src/fieldRestrictions/codeListValidation.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { convertToArray, isAbsent, isEmptyString, notEmpty } from '../../utils'; +import { asArray } from 'common'; import { BaseSchemaValidationError, EnumValueValidationError, @@ -25,6 +25,8 @@ import { SchemaValidationErrorTypes, } from '../types/validationErrorTypes'; import { ValidationFunction } from '../types/validationFunctionTypes'; +import { isEmptyString } from '../utils/isEmptyString'; +import { isDefined } from '../utils/typeUtils'; /** * Check all values of a DataRecord pass codeList restrictions in their schema. @@ -45,8 +47,8 @@ export const validateCodeList: ValidationFunction = (rec, index, fields): EnumVa } // put all values into array to standardize validation for array and non array fields - const recordFieldValues = convertToArray(rec[field.name]); - const invalidValues = recordFieldValues.filter((val) => isInvalidEnumValue(codeList, val)); + const recordFieldValues = asArray(rec[field.name]); + const invalidValues = recordFieldValues.filter((val) => !isValidEnumValue(codeList, val)); if (invalidValues.length !== 0) { return buildCodeListError({ fieldName: field.name, index }, { value: invalidValues }); @@ -54,7 +56,7 @@ export const validateCodeList: ValidationFunction = (rec, index, fields): EnumVa } return undefined; }) - .filter(notEmpty); + .filter(isDefined); }; const buildCodeListError = ( @@ -71,11 +73,17 @@ const buildCodeListError = ( }; }; -const isInvalidEnumValue = (codeList: string[] | number[], value: string | boolean | number | undefined) => { - // only validate existing values - if (isAbsent(value) || (typeof value === 'string' && isEmptyString(value))) { - return false; +/** + * If value exists, confirm that it matches an option in the provided code list + * @param codeList + * @param value + * @returns + */ +const isValidEnumValue = (codeList: string[] | number[], value: string | boolean | number | undefined) => { + // do not run validation on empty values + if (value === undefined || (typeof value === 'string' && isEmptyString(value))) { + return true; } - return !codeList.some((allowedValue) => allowedValue === value); + return codeList.some((allowedValue) => allowedValue === value); }; diff --git a/packages/client/src/validation/fieldRestrictions/rangeValidation.ts b/libraries/validation/src/fieldRestrictions/rangeValidation.ts similarity index 93% rename from packages/client/src/validation/fieldRestrictions/rangeValidation.ts rename to libraries/validation/src/fieldRestrictions/rangeValidation.ts index 83d4408..9545a65 100644 --- a/packages/client/src/validation/fieldRestrictions/rangeValidation.ts +++ b/libraries/validation/src/fieldRestrictions/rangeValidation.ts @@ -17,15 +17,16 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +import { asArray } from 'common'; import { RestrictionRange } from 'dictionary'; -import { convertToArray, isEmpty, isNumberArray, notEmpty } from '../../utils'; -import { rangeToSymbol } from '../utils/rangeToSymbol'; import { BaseSchemaValidationError, RangeValidationError, SchemaValidationErrorTypes, } from '../types/validationErrorTypes'; import { ValidationFunction } from '../types/validationFunctionTypes'; +import { isDefined, isNumberArray } from '../utils/typeUtils'; +import { rangeToSymbol } from '../utils/rangeToSymbol'; /** * Check all values of a DataRecord pass range restrictions in their schema. @@ -37,13 +38,13 @@ import { ValidationFunction } from '../types/validationFunctionTypes'; export const validateRange: ValidationFunction = (record, index, schemaFields): RangeValidationError[] => { return schemaFields .map((field) => { - const recordFieldValues = convertToArray(record[field.name]); + const recordFieldValues = asArray(record[field.name]); if (!isNumberArray(recordFieldValues)) { return undefined; } const range = field.restrictions && 'range' in field.restrictions ? field.restrictions.range : undefined; - if (isEmpty(range)) { + if (range === undefined) { return undefined; } @@ -54,7 +55,7 @@ export const validateRange: ValidationFunction = (record, index, schemaFields): } return undefined; }) - .filter(notEmpty); + .filter(isDefined); }; const buildRangeError = ( diff --git a/packages/client/src/validation/fieldRestrictions/regexValidation.ts b/libraries/validation/src/fieldRestrictions/regexValidation.ts similarity index 88% rename from packages/client/src/validation/fieldRestrictions/regexValidation.ts rename to libraries/validation/src/fieldRestrictions/regexValidation.ts index a501bdd..430803e 100644 --- a/packages/client/src/validation/fieldRestrictions/regexValidation.ts +++ b/libraries/validation/src/fieldRestrictions/regexValidation.ts @@ -17,13 +17,15 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ import { SchemaFieldValueType } from 'dictionary'; -import { convertToArray, isEmpty, isEmptyString, isStringArray, notEmpty } from '../../utils'; import { BaseSchemaValidationError, RegexValidationError, SchemaValidationErrorTypes, } from '../types/validationErrorTypes'; import { ValidationFunction } from '../types/validationFunctionTypes'; +import { asArray } from 'common'; +import { isDefined, isStringArray } from '../utils/typeUtils'; +import { isEmptyString } from '../utils/isEmptyString'; /** * Check all values of a DataRecord pass regex restrictions in their schema. @@ -35,13 +37,9 @@ import { ValidationFunction } from '../types/validationFunctionTypes'; export const validateRegex: ValidationFunction = (record, index, fields): RegexValidationError[] => { return fields .map((field) => { - if ( - field.valueType === SchemaFieldValueType.Values.string && - field.restrictions && - !isEmpty(field.restrictions.regex) - ) { + if (field.valueType === SchemaFieldValueType.Values.string && field.restrictions && field.restrictions.regex) { const regex = field.restrictions.regex; - const recordFieldValues = convertToArray(record[field.name]); + const recordFieldValues = asArray(record[field.name]); if (!isStringArray(recordFieldValues)) { // This field value should be string or string array, we will skip validation if the type is wrong. return undefined; @@ -58,12 +56,14 @@ export const validateRegex: ValidationFunction = (record, index, fields): RegexV // Field does not have regex validation return undefined; }) - .filter(notEmpty); + .filter(isDefined); }; const isInvalidRegexValue = (regex: string, value: string) => { // optional field if the value is absent at this point - if (isEmptyString(value)) return false; + if (isEmptyString(value)) { + return false; + } const regexPattern = new RegExp(regex); return !regexPattern.test(value); }; diff --git a/packages/client/src/validation/fieldRestrictions/requiredValidation.ts b/libraries/validation/src/fieldRestrictions/requiredValidation.ts similarity index 82% rename from packages/client/src/validation/fieldRestrictions/requiredValidation.ts rename to libraries/validation/src/fieldRestrictions/requiredValidation.ts index 82bbc48..540e7d4 100644 --- a/packages/client/src/validation/fieldRestrictions/requiredValidation.ts +++ b/libraries/validation/src/fieldRestrictions/requiredValidation.ts @@ -17,15 +17,16 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { SchemaField } from 'dictionary'; -import { convertToArray, isEmpty, notEmpty } from '../../utils'; +import { DataRecord, SchemaField } from 'dictionary'; import { BaseSchemaValidationError, MissingRequiredFieldValidationError, SchemaValidationErrorTypes, } from '../types/validationErrorTypes'; import { ValidationFunction } from '../types/validationFunctionTypes'; -import { DataRecord } from '../../types/dataRecords'; +import { isDefined } from '../utils/typeUtils'; +import { asArray } from 'common'; +import { isEmptyString } from '../utils/isEmptyString'; /** * Check all values of a DataRecord pass required restrictions in their schema. @@ -46,7 +47,7 @@ export const validateRequiredFields: ValidationFunction = ( } return undefined; }) - .filter(notEmpty); + .filter(isDefined); }; const buildRequiredError = ( @@ -65,8 +66,12 @@ const buildRequiredError = ( const isRequiredMissing = (field: SchemaField, record: DataRecord) => { const isRequired = field.restrictions && field.restrictions.required; - if (!isRequired) return false; + if (!isRequired) { + return false; + } - const recordFieldValues = convertToArray(record[field.name]); - return recordFieldValues.every(isEmpty); + // a required field is missing if there are is no value provided for this field (or if an array, all array values are empty) + return asArray(record[field.name]).every( + (item) => item === undefined || (typeof item === 'string' && isEmptyString(item)), + ); }; diff --git a/packages/client/src/validation/fieldRestrictions/scriptValidation.ts b/libraries/validation/src/fieldRestrictions/scriptValidation.ts similarity index 93% rename from packages/client/src/validation/fieldRestrictions/scriptValidation.ts rename to libraries/validation/src/fieldRestrictions/scriptValidation.ts index 5f6ba0c..d002d84 100644 --- a/packages/client/src/validation/fieldRestrictions/scriptValidation.ts +++ b/libraries/validation/src/fieldRestrictions/scriptValidation.ts @@ -17,16 +17,16 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { SchemaField } from 'dictionary'; +import { DataRecord, SchemaField } from 'dictionary'; import vm from 'vm'; -import { DataRecord } from '../../types/dataRecords'; -import { notEmpty } from '../../utils'; import { BaseSchemaValidationError, SchemaValidationErrorTypes, ScriptValidationError, } from '../types/validationErrorTypes'; import { ValidationFunction } from '../types/validationFunctionTypes'; +import { isDefined } from '../utils/typeUtils'; +import { asArray } from 'common'; const ctx = vm.createContext(); @@ -59,7 +59,7 @@ export const validateScript: ValidationFunction = (record, index, fields) => { } return undefined; }) - .filter(notEmpty); + .filter(isDefined); }; const buildScriptError = ( @@ -101,8 +101,7 @@ const validateWithScript = ( // scripts should already be strings inside arrays, but ensure that they are to help transition between lectern versions // checking for this can be removed in future versions of lectern (feb 2020) - const scripts = - typeof field.restrictions.script === 'string' ? [field.restrictions.script] : field.restrictions.script; + const scripts = asArray(field.restrictions.script); let result: { valid: boolean; diff --git a/packages/client/src/validation/index.ts b/libraries/validation/src/index.ts similarity index 97% rename from packages/client/src/validation/index.ts rename to libraries/validation/src/index.ts index 28f5820..27d8aec 100644 --- a/packages/client/src/validation/index.ts +++ b/libraries/validation/src/index.ts @@ -28,4 +28,3 @@ export * from './schemaRestrictions/foreignKeysValidation'; export * from './schemaRestrictions/uniqueKeyValidation'; export * from './schemaRestrictions/uniqueValidation'; export * from './types'; -export * from './validationPipelines'; diff --git a/packages/client/src/validation/schemaRestrictions/foreignKeysValidation.ts b/libraries/validation/src/schemaRestrictions/foreignKeysValidation.ts similarity index 99% rename from packages/client/src/validation/schemaRestrictions/foreignKeysValidation.ts rename to libraries/validation/src/schemaRestrictions/foreignKeysValidation.ts index 9687e99..8d2a935 100644 --- a/packages/client/src/validation/schemaRestrictions/foreignKeysValidation.ts +++ b/libraries/validation/src/schemaRestrictions/foreignKeysValidation.ts @@ -18,7 +18,7 @@ */ import { differenceWith, isEqual } from 'lodash'; -import { DataRecord } from '../../types/dataRecords'; +import { DataRecord } from 'dictionary'; import { ForeignKeyValidationError, SchemaValidationErrorTypes } from '../types/validationErrorTypes'; import { CrossSchemaValidationFunction } from '../types/validationFunctionTypes'; import { selectFieldsFromDataset } from '../utils/datasetUtils'; diff --git a/packages/client/src/validation/schemaRestrictions/uniqueKeyValidation.ts b/libraries/validation/src/schemaRestrictions/uniqueKeyValidation.ts similarity index 100% rename from packages/client/src/validation/schemaRestrictions/uniqueKeyValidation.ts rename to libraries/validation/src/schemaRestrictions/uniqueKeyValidation.ts diff --git a/packages/client/src/validation/schemaRestrictions/uniqueValidation.ts b/libraries/validation/src/schemaRestrictions/uniqueValidation.ts similarity index 100% rename from packages/client/src/validation/schemaRestrictions/uniqueValidation.ts rename to libraries/validation/src/schemaRestrictions/uniqueValidation.ts diff --git a/packages/client/src/validation/types/index.ts b/libraries/validation/src/types/index.ts similarity index 100% rename from packages/client/src/validation/types/index.ts rename to libraries/validation/src/types/index.ts diff --git a/packages/client/src/validation/types/validationErrorTypes.ts b/libraries/validation/src/types/validationErrorTypes.ts similarity index 97% rename from packages/client/src/validation/types/validationErrorTypes.ts rename to libraries/validation/src/types/validationErrorTypes.ts index 33abed1..016ce2a 100644 --- a/packages/client/src/validation/types/validationErrorTypes.ts +++ b/libraries/validation/src/types/validationErrorTypes.ts @@ -18,8 +18,7 @@ */ import type { Values, Singular } from 'common'; -import type { DataRecord, DataRecordValue } from '../../types/dataRecords'; -import type { RestrictionRange } from 'dictionary'; +import type { DataRecord, DataRecordValue, RestrictionRange } from 'dictionary'; /** * Represents the common structure of a validation error without the custom content provided by specific error types. diff --git a/packages/client/src/validation/types/validationFunctionTypes.ts b/libraries/validation/src/types/validationFunctionTypes.ts similarity index 94% rename from packages/client/src/validation/types/validationFunctionTypes.ts rename to libraries/validation/src/types/validationFunctionTypes.ts index 2709c9c..bba1c30 100644 --- a/packages/client/src/validation/types/validationFunctionTypes.ts +++ b/libraries/validation/src/types/validationFunctionTypes.ts @@ -17,8 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { Schema, SchemaField } from 'dictionary'; -import { DataRecord, UnprocessedDataRecord } from '../../types/dataRecords'; +import { DataRecord, UnprocessedDataRecord, Schema } from 'dictionary'; import { SchemaValidationError } from './validationErrorTypes'; // these validation functions run AFTER the record has been converted to the correct types from raw strings diff --git a/packages/client/src/validation/utils/datasetUtils.ts b/libraries/validation/src/utils/datasetUtils.ts similarity index 98% rename from packages/client/src/validation/utils/datasetUtils.ts rename to libraries/validation/src/utils/datasetUtils.ts index e07f4c3..b757d45 100644 --- a/packages/client/src/validation/utils/datasetUtils.ts +++ b/libraries/validation/src/utils/datasetUtils.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { DataRecord } from '../../types/dataRecords'; +import { DataRecord } from 'dictionary'; /** * Returns a string representation of a record. The record is sorted by its properties so diff --git a/packages/client/src/types/index.ts b/libraries/validation/src/utils/isEmptyString.ts similarity index 84% rename from packages/client/src/types/index.ts rename to libraries/validation/src/utils/isEmptyString.ts index f35338d..316bdb2 100644 --- a/packages/client/src/types/index.ts +++ b/libraries/validation/src/utils/isEmptyString.ts @@ -17,4 +17,9 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -export * from './dataRecords'; +/** + * Check if a string is empty (exactly matches `''`) or contains only whitespace (ex. `' '` ). + * @param input + * @returns + */ +export const isEmptyString = (input: string): boolean => input.trim() === ''; diff --git a/packages/client/src/validation/utils/rangeToSymbol.ts b/libraries/validation/src/utils/rangeToSymbol.ts similarity index 100% rename from packages/client/src/validation/utils/rangeToSymbol.ts rename to libraries/validation/src/utils/rangeToSymbol.ts diff --git a/packages/client/src/validation/utils/fieldTypeUtils.ts b/libraries/validation/src/utils/typeUtils.ts similarity index 57% rename from packages/client/src/validation/utils/fieldTypeUtils.ts rename to libraries/validation/src/utils/typeUtils.ts index 405d3d8..a209e68 100644 --- a/packages/client/src/validation/utils/fieldTypeUtils.ts +++ b/libraries/validation/src/utils/typeUtils.ts @@ -17,27 +17,32 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { SchemaFieldValueType } from 'dictionary'; +/** + * Checks that the input does not equal undefined (and lets the type checker know). + * + * Useful for filtering undefined values out of lists. + * + * (input) => input !== undefined + * + * @example + * const combinedArray: Array = ['hello', undefined, 'world']; + * const stringArray = combinedArray.filter(isDefined); // type is: Array + */ +export const isDefined = (input: T | undefined): input is T => input !== undefined; -import { isEmptyString } from '../../utils'; +/** + * Determines if a variable is of type number[] + * @param value + * @returns + */ +export const isNumberArray = (value: unknown): value is number[] => + Array.isArray(value) && value.every((item) => typeof item === 'number'); /** - * Check a value is valid for a given schema value type. - * @param valueType + * Determines if a variable is of type string[] * @param value * @returns */ -export const isInvalidFieldType = (valueType: SchemaFieldValueType, value: string) => { - // optional field if the value is absent at this point - if (isEmptyString(value)) return false; - switch (valueType) { - case SchemaFieldValueType.Values.string: - return false; - case SchemaFieldValueType.Values.integer: - return !Number.isSafeInteger(Number(value)); - case SchemaFieldValueType.Values.number: - return isNaN(Number(value)); - case SchemaFieldValueType.Values.boolean: - return !(value.toLowerCase() === 'true' || value.toLowerCase() === 'false'); - } +export const isStringArray = (value: unknown): value is string[] => { + return Array.isArray(value) && value.every((item) => typeof item === 'string'); }; diff --git a/libraries/validation/test/placeholder.spec.ts b/libraries/validation/test/placeholder.spec.ts new file mode 100644 index 0000000..99c1673 --- /dev/null +++ b/libraries/validation/test/placeholder.spec.ts @@ -0,0 +1,3 @@ +describe('Placeholder', () => { + it('Dummy test', () => {}); +}); diff --git a/libraries/validation/tsconfig.json b/libraries/validation/tsconfig.json new file mode 100644 index 0000000..b474986 --- /dev/null +++ b/libraries/validation/tsconfig.json @@ -0,0 +1,20 @@ +{ + "compilerOptions": { + "target": "ESNext", + "lib": ["ESNext"], + "module": "CommonJS", + "moduleResolution": "node", + "resolveJsonModule": true, + "strict": true, + "noImplicitAny": true, + "noUnusedParameters": true, + "esModuleInterop": true, + "declaration": true, + "sourceMap": true, + "inlineSources": true, + "outDir": "dist/", + "downlevelIteration": true, + "skipLibCheck": true + }, + "include": ["./src/**/*.ts"] +} diff --git a/package.json b/package.json index e4117d7..fdc9c6a 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,17 @@ "main": "index.js", "scripts": { "build:all": "pnpm nx run-many --all --target=build", + "build:common": "pnpm nx build common", + "build:dictionary": "pnpm nx build dictionary", + "build:validation": "pnpm nx build @overture-stack/lectern-validation", + "build:server": "pnpm nx build server", + "build:client": "pnpm nx build @overture-stack/lectern-client", "test:all": "pnpm nx run-many --all --target=test", + "test:common": "pnpm nx test common", + "test:dictionary": "pnpm nx test dictionary", + "test:validation": "pnpm nx test @overture-stack/lectern-validation", + "test:server": "pnpm nx test server", + "test:client": "pnpm nx test @overture-stack/lectern-client", "generate": "pnpm -C scripts generate" }, "keywords": [], diff --git a/packages/client/README.md b/packages/client/README.md index e6ac7ed..1591e12 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -1,5 +1,7 @@ # Lectern TypeScript Client +[![NPM Version](https://img.shields.io/npm/v/@overture-stack/lectern-client?color=%23cb3837&style=for-the-badge&logo=npm)](https://www.npmjs.com/package/@overture-stack/lectern-client) + ## Features: - Runs different restrictions validations: regex, range, scripts, required fields, type checks, etc. - Transforms the data from string to their proper type. @@ -7,6 +9,3 @@ - Fetch dictionaries from the configured lectern service. - Provide typed definitions for the dictionary object. - Analyze dictionary versions diff. - -## Usage examples: -- icgc-argo/argo-clinical [https://github.com/icgc-argo/argo-clinical] \ No newline at end of file diff --git a/packages/client/package.json b/packages/client/package.json index f1306e6..9311314 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -39,6 +39,7 @@ "typescript": "^5.1.6" }, "dependencies": { + "@overture-stack/lectern-validation": "workspace:^", "cd": "^0.3.3", "common": "workspace:^", "deep-freeze": "^0.0.1", diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts index 4b8bd69..0013e31 100644 --- a/packages/client/src/index.ts +++ b/packages/client/src/index.ts @@ -17,9 +17,6 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -export * as DictionaryTypes from 'dictionary'; export * as analyzer from './changeAnalysis'; export * as functions from './processing'; export { restClient } from './rest'; - -export type { DataRecord, DataRecordValue, UnprocessedDataRecord } from './types'; diff --git a/packages/client/src/processing/convertDataValueTypes.ts b/packages/client/src/processing/convertDataValueTypes.ts index 986b728..e3c4a2a 100644 --- a/packages/client/src/processing/convertDataValueTypes.ts +++ b/packages/client/src/processing/convertDataValueTypes.ts @@ -20,11 +20,17 @@ import _ from 'lodash'; import { Singular } from 'common'; -import { Schema, SchemaField, SchemaFieldValueType } from 'dictionary'; +import { + DataRecord, + DataRecordValue, + UnprocessedDataRecord, + Schema, + SchemaField, + SchemaFieldValueType, +} from 'dictionary'; -import { DataRecord, DataRecordValue, UnprocessedDataRecord } from '../types'; import { convertToArray, isEmpty } from '../utils'; -import { SchemaValidationError, SchemaValidationErrorTypes } from '../validation'; +import { SchemaValidationError, SchemaValidationErrorTypes } from '@overture-stack/lectern-validation'; /** * Warning: diff --git a/packages/client/src/processing/index.ts b/packages/client/src/processing/index.ts index dfafb5c..0baea5d 100644 --- a/packages/client/src/processing/index.ts +++ b/packages/client/src/processing/index.ts @@ -17,17 +17,16 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +import type { SchemaValidationError } from '@overture-stack/lectern-validation'; +import * as validation from '@overture-stack/lectern-validation'; import { NotFoundError } from 'common'; -import { Dictionary, Schema } from 'dictionary'; +import { DataRecord, Dictionary, Schema, UnprocessedDataRecord } from 'dictionary'; import _ from 'lodash'; - import { loggerFor } from '../logger'; -import { DataRecord, UnprocessedDataRecord } from '../types/dataRecords'; import { convertToArray, isEmpty, isNotAbsent, isString, isStringArray, notEmpty } from '../utils'; -import type { SchemaValidationError } from '../validation'; -import * as validation from '../validation'; import { convertFromRawStrings } from './convertDataValueTypes'; import { BatchProcessingResult, FieldNamesByPriorityMap, SchemaProcessingResult } from './processingResultTypes'; +import * as pipelines from './validationPipelines'; const L = loggerFor(__filename); @@ -43,7 +42,7 @@ export const processSchemas = ( // Run cross-schema validations const schemaDef = getNotNullSchemaDefinitionFromDictionary(dictionary, schemaName); - const crossSchemaLevelValidationResults = validation + const crossSchemaLevelValidationResults = pipelines .runCrossSchemaValidationPipeline(schemaDef, schemasData, [validation.validateForeignKeys]) .filter(notEmpty); @@ -115,7 +114,7 @@ export const process = ( L.debug(`converted row #${index} from raw strings`); const postTypeConversionValidationResult = validateAfterTypeConversion( schemaDef, - _.cloneDeep(convertedRecord) as DataRecord, + _.cloneDeep(convertedRecord), index, ); @@ -202,7 +201,7 @@ const validateUnprocessedRecord = ( record: UnprocessedDataRecord, index: number, ): ReadonlyArray => { - const majorErrors = validation + const majorErrors = pipelines .runUnprocessedRecordValidationPipeline(record, index, schemaDef.fields, [ validation.validateFieldNames, validation.validateNonArrayFields, @@ -218,7 +217,7 @@ const validateAfterTypeConversion = ( record: DataRecord, index: number, ): ReadonlyArray => { - const validationErrors = validation + const validationErrors = pipelines .runRecordValidationPipeline(record, index, schemaDef.fields, [ validation.validateRegex, validation.validateRange, @@ -231,7 +230,7 @@ const validateAfterTypeConversion = ( }; function validateRecordsSet(schemaDef: Schema, processedRecords: DataRecord[]) { - const validationErrors = validation + const validationErrors = pipelines .runDatasetValidationPipeline(processedRecords, schemaDef, [ validation.validateUnique, validation.validateUniqueKey, diff --git a/packages/client/src/processing/processingResultTypes.ts b/packages/client/src/processing/processingResultTypes.ts index 0a66a4b..8360998 100644 --- a/packages/client/src/processing/processingResultTypes.ts +++ b/packages/client/src/processing/processingResultTypes.ts @@ -17,9 +17,8 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { Schema } from 'dictionary'; -import { DataRecord } from '../types/dataRecords'; -import { SchemaValidationError } from '../validation/types/validationErrorTypes'; +import { SchemaValidationError } from '@overture-stack/lectern-validation'; +import { DataRecord, Schema } from 'dictionary'; export type ProcessingFunction = (schema: Schema, rec: Readonly, index: number) => any; diff --git a/packages/client/src/validation/validationPipelines.ts b/packages/client/src/processing/validationPipelines.ts similarity index 91% rename from packages/client/src/validation/validationPipelines.ts rename to packages/client/src/processing/validationPipelines.ts index f11a5fc..47d335d 100644 --- a/packages/client/src/validation/validationPipelines.ts +++ b/packages/client/src/processing/validationPipelines.ts @@ -17,16 +17,15 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { Schema, SchemaField } from 'dictionary'; +import { DataRecord, UnprocessedDataRecord, Schema, SchemaField } from 'dictionary'; -import { DataRecord, UnprocessedDataRecord } from '../types/dataRecords'; -import { SchemaValidationError } from './types/validationErrorTypes'; +import { SchemaValidationError } from '../../../../libraries/validation/src/types/validationErrorTypes'; import { CrossSchemaValidationFunction, DatasetValidationFunction, UnprocessedRecordValidationFunction, ValidationFunction, -} from './types/validationFunctionTypes'; +} from '../../../../libraries/validation/src/types/validationFunctionTypes'; export const runUnprocessedRecordValidationPipeline = ( record: UnprocessedDataRecord, diff --git a/packages/client/test/processing.spec.ts b/packages/client/test/processing.spec.ts index 503239b..ccd84d1 100644 --- a/packages/client/test/processing.spec.ts +++ b/packages/client/test/processing.spec.ts @@ -17,10 +17,10 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +import { SchemaValidationErrorTypes } from '@overture-stack/lectern-validation'; import chai from 'chai'; import { functions as schemaService } from '../src'; import { loggerFor } from '../src/logger'; -import { SchemaValidationErrorTypes } from '../src/validation'; import dictionary from './fixtures/registrationSchema'; const L = loggerFor(__filename); diff --git a/packages/client/test/validation.spec.ts b/packages/client/test/validation.spec.ts index 74e6b29..36f72d7 100644 --- a/packages/client/test/validation.spec.ts +++ b/packages/client/test/validation.spec.ts @@ -17,13 +17,11 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +import { SchemaValidationErrorTypes } from '@overture-stack/lectern-validation'; import chai from 'chai'; import { functions as schemaService } from '../src'; import { loggerFor } from '../src/logger'; -import { SchemaValidationErrorTypes } from '../src/validation'; -import { rangeToSymbol } from '../src/validation/utils/rangeToSymbol'; import dictionary from './fixtures/registrationSchema'; -const L = loggerFor(__filename); chai.should(); @@ -136,12 +134,13 @@ describe('validation', () => { min: 0, value: [-1], }; + chai.expect(result.validationErrors).to.deep.include({ errorType: SchemaValidationErrorTypes.INVALID_BY_RANGE, fieldName: 'unit_number', index: 0, info: info1, - message: `Value is out of permissible range, it must be ${rangeToSymbol(info1)}.`, + message: `Value is out of permissible range, it must be >= 0 and < 999.`, }); const info2 = { @@ -154,7 +153,7 @@ describe('validation', () => { fieldName: 'unit_number', index: 2, info: info2, - message: `Value is out of permissible range, it must be ${rangeToSymbol(info2)}.`, + message: `Value is out of permissible range, it must be >= 0 and < 999.`, }); }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 279f9a5..70c817f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -186,8 +186,33 @@ importers: specifier: ^3.0.2 version: 3.0.2 + libraries/validation: + dependencies: + common: + specifier: workspace:^ + version: link:../common + dictionary: + specifier: workspace:^ + version: link:../dictionary + lodash: + specifier: ^4.17.21 + version: 4.17.21 + zod: + specifier: ^3.21.4 + version: 3.21.4 + devDependencies: + '@types/lodash': + specifier: ^4.14.195 + version: 4.14.195 + rimraf: + specifier: ^3.0.2 + version: 3.0.2 + packages/client: dependencies: + '@overture-stack/lectern-validation': + specifier: workspace:^ + version: link:../../libraries/validation cd: specifier: ^0.3.3 version: 0.3.3 From d3f55aaf98a30c658d1feeda47340c41b55c8006 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Sat, 15 Jun 2024 13:22:23 -0400 Subject: [PATCH 08/27] Update generated dictionary JSON Schema - Version String regexp now correctly checks for . character instead of any character --- generated/DictionaryMetaSchema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generated/DictionaryMetaSchema.json b/generated/DictionaryMetaSchema.json index a1d6d61..a43a52d 100644 --- a/generated/DictionaryMetaSchema.json +++ b/generated/DictionaryMetaSchema.json @@ -477,7 +477,7 @@ }, "version": { "type": "string", - "pattern": "^[0-9]+.[0-9]+$" + "pattern": "^([0-9]+)\\.[0-9]+$" } }, "required": [ From 49930fdccedf06b1f6c4473c2dcf5b5c292bfc80 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Sat, 15 Jun 2024 13:23:17 -0400 Subject: [PATCH 09/27] Always build dictionary before generating json schema --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index fdc9c6a..310d8d5 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "test:validation": "pnpm nx test @overture-stack/lectern-validation", "test:server": "pnpm nx test server", "test:client": "pnpm nx test @overture-stack/lectern-client", - "generate": "pnpm -C scripts generate" + "generate": "pnpm build:dictionary && pnpm -C scripts generate" }, "keywords": [], "author": "Ontario Institute for Cancer Research", From bd4aac2fd59ae20e27bc9036095fb0c82a853db7 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Sat, 15 Jun 2024 13:24:25 -0400 Subject: [PATCH 10/27] Auto-formatting correcting whitespace --- nx.json | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/nx.json b/nx.json index 54f253e..03619a0 100644 --- a/nx.json +++ b/nx.json @@ -1,24 +1,24 @@ { - "tasksRunnerOptions": { - "default": { - "runner": "nx/tasks-runners/default", - "options": { - "cacheableOperations": ["build", "lint", "test"] - } - } - }, - "targetDefaults": { - "build": { - "dependsOn": ["^build"] - }, - "dev": { - "dependsOn": ["^build"] - }, - "publish": { - "dependsOn": ["^build"] - }, + "tasksRunnerOptions": { + "default": { + "runner": "nx/tasks-runners/default", + "options": { + "cacheableOperations": ["build", "lint", "test"] + } + } + }, + "targetDefaults": { + "build": { + "dependsOn": ["^build"] + }, + "dev": { + "dependsOn": ["^build"] + }, + "publish": { + "dependsOn": ["^build"] + }, "test": { "dependsOn": ["^build"] } - } + } } From 8228b5f8865c25949d8ebea83c5388dfb27686e1 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Sat, 15 Jun 2024 13:24:59 -0400 Subject: [PATCH 11/27] Tests for version string (major.minor format) --- .../dictionary/test/dictionaryTypes.spec.ts | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/libraries/dictionary/test/dictionaryTypes.spec.ts b/libraries/dictionary/test/dictionaryTypes.spec.ts index 571e803..560cada 100644 --- a/libraries/dictionary/test/dictionaryTypes.spec.ts +++ b/libraries/dictionary/test/dictionaryTypes.spec.ts @@ -31,11 +31,12 @@ import { Schema, SchemaField, StringFieldRestrictions, + VersionString, } from '../src'; describe('Dictionary Types', () => { describe('NameString', () => { - it("Can't be empty string", () => { + it('Rejects empty string', () => { expect(NameString.safeParse('').success).false; }); it('Can be string', () => { @@ -307,6 +308,30 @@ describe('Dictionary Types', () => { }); }); }); + describe('Version String', () => { + it('Rejects empty string', () => { + expect(VersionString.safeParse('').success).false; + }); + it('Accepts valid version strings', () => { + expect(VersionString.safeParse('0.0').success).true; + expect(VersionString.safeParse('1.0').success).true; + expect(VersionString.safeParse('1.1').success).true; + expect(VersionString.safeParse('10.10').success).true; + expect(VersionString.safeParse('234.567').success).true; + }); + it('Rejects single number strings', () => { + expect(VersionString.safeParse('0').success).false; + expect(VersionString.safeParse('1').success).false; + expect(VersionString.safeParse('10').success).false; + expect(VersionString.safeParse('123').success).false; + }); + it('Rejects other invalid strings', () => { + expect(VersionString.safeParse('1-2').success).false; + expect(VersionString.safeParse('1.2.3').success).false; + expect(VersionString.safeParse('1.2.').success).false; + expect(VersionString.safeParse('one dot two').success).false; + }); + }); describe('Dictionary', () => { it("Can't have repeated schema names", () => { const sharedName = 'asdf'; From f1753c2b4596f2d1af0a52ed52bf576a6f43fb1d Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Sat, 15 Jun 2024 13:25:32 -0400 Subject: [PATCH 12/27] Add missing copyright text --- libraries/common/src/types/generics.ts | 19 +++++++++++++++++++ libraries/common/src/types/index.ts | 19 +++++++++++++++++++ libraries/common/src/utils/index.ts | 19 +++++++++++++++++++ libraries/common/src/utils/typeUtils.ts | 19 +++++++++++++++++++ 4 files changed, 76 insertions(+) diff --git a/libraries/common/src/types/generics.ts b/libraries/common/src/types/generics.ts index cbe6a92..fd35c14 100644 --- a/libraries/common/src/types/generics.ts +++ b/libraries/common/src/types/generics.ts @@ -1,3 +1,22 @@ +/* + * Copyright (c) 2023 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + /** * Strip aliases out from the top level of the TS reported type. * This will display type as an object with {key: value} pairs instead as an alias name. diff --git a/libraries/common/src/types/index.ts b/libraries/common/src/types/index.ts index 68cee3d..fcd0305 100644 --- a/libraries/common/src/types/index.ts +++ b/libraries/common/src/types/index.ts @@ -1,3 +1,22 @@ +/* + * Copyright (c) 2023 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + export * from './generics'; export * from './result'; export * from './singular'; diff --git a/libraries/common/src/utils/index.ts b/libraries/common/src/utils/index.ts index 27ea58d..8bcd0f1 100644 --- a/libraries/common/src/utils/index.ts +++ b/libraries/common/src/utils/index.ts @@ -1,2 +1,21 @@ +/* + * Copyright (c) 2023 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + export * from './stringUtils'; export * from './typeUtils'; diff --git a/libraries/common/src/utils/typeUtils.ts b/libraries/common/src/utils/typeUtils.ts index f3b75b1..00bd24e 100644 --- a/libraries/common/src/utils/typeUtils.ts +++ b/libraries/common/src/utils/typeUtils.ts @@ -1,3 +1,22 @@ +/* + * Copyright (c) 2023 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + /** * Ensure a value is wrapped in an array. * From 2a7162312809bbfbd322354b2420bbc09e29c3f4 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Sat, 15 Jun 2024 13:28:30 -0400 Subject: [PATCH 13/27] Remove unused parameter --- packages/client/src/changeAnalysis/changeAnalyzer.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/client/src/changeAnalysis/changeAnalyzer.ts b/packages/client/src/changeAnalysis/changeAnalyzer.ts index be430a3..77adcdf 100644 --- a/packages/client/src/changeAnalysis/changeAnalyzer.ts +++ b/packages/client/src/changeAnalysis/changeAnalyzer.ts @@ -95,7 +95,7 @@ export const analyzeChanges = (schemasDiff: DictionaryDiff): ChangeAnalysis => { } if (fieldDiff.valueType) { - categorizerValueTypeChange(analysis, fieldName, fieldDiff.valueType); + categorizerValueTypeChange(analysis, fieldName); } } } @@ -111,7 +111,7 @@ const categorizeFieldArrayDesignationChange = (analysis: ChangeAnalysis, field: } }; -const categorizerValueTypeChange = (analysis: ChangeAnalysis, field: string, changes: FieldChanges) => { +const categorizerValueTypeChange = (analysis: ChangeAnalysis, field: string) => { analysis.valueTypeChanges.push(field); }; From 83815c0cfa332ae0536e3f872e6b4abd9d1b771d Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Mon, 17 Jun 2024 20:46:28 -0400 Subject: [PATCH 14/27] Move all libraries into packages There was no need to maintain two distinct paths for shared packages, they are now all collected as either packages or apps. All references to libraries have been removed in documentation and configs. --- .dockerignore | 1 - README.md | 28 +++-- .../src/processing/validationPipelines.ts | 4 +- {libraries => packages}/common/.mocharc.json | 0 {libraries => packages}/common/README.md | 0 {libraries => packages}/common/package.json | 0 {libraries => packages}/common/src/errors.ts | 0 {libraries => packages}/common/src/index.ts | 0 .../common/src/types/generics.ts | 0 .../common/src/types/index.ts | 0 .../common/src/types/result.ts | 0 .../common/src/types/singular.ts | 0 .../common/src/utils/index.ts | 0 packages/common/src/utils/objectUtils.ts | 93 +++++++++++++++ .../common/src/utils/stringUtils.ts | 0 .../common/src/utils/typeUtils.ts | 0 {libraries => packages}/common/tsconfig.json | 0 .../dictionary/.mocharc.json | 0 {libraries => packages}/dictionary/README.md | 0 .../dictionary/package.json | 0 .../dictionary/src/diff.ts | 0 .../dictionary/src/index.ts | 0 .../dictionary/src/references.ts | 0 .../dictionary/src/types/dataTypes.ts | 0 .../dictionary/src/types/dbTypes.ts | 0 .../dictionary/src/types/dictionaryTypes.ts | 0 .../dictionary/src/types/diffTypes.ts | 0 .../dictionary/src/types/index.ts | 0 .../dictionary/src/types/referenceTypes.ts | 0 .../dictionary/src/utils/allUnique.ts | 0 .../src/utils/resolveRestrictions.ts | 51 ++++++++ .../dictionary/src/utils/schemaUtils.ts | 0 .../dictionary/src/utils/version.ts | 0 .../dictionary/test/dataTypes.spec.ts | 0 .../dictionary/test/dictionaryTypes.spec.ts | 0 .../dictionary/test/diff.spec.ts | 0 .../dictionary/test/fixtures/diff/initial.ts | 0 .../dictionary/test/fixtures/diff/updated.ts | 0 .../references/codeList_references/input.ts | 0 .../references/codeList_references/output.ts | 0 .../references/cyclic_references/input.ts | 0 .../references/cyclic_references/output.ts | 0 .../empty_references_section/input.ts | 0 .../empty_references_section/output.ts | 0 .../references/no_references_section/input.ts | 0 .../no_references_section/output.ts | 0 .../non_existing_references/input.ts | 0 .../non_existing_references/output.ts | 0 .../references_within_references/input.ts | 0 .../references_within_references/output.ts | 0 .../references/regex_reference/input.ts | 0 .../regex_reference/input_with_array.ts | 0 .../references/regex_reference/output.ts | 0 .../references/script_references/input.ts | 0 .../references/script_references/output.ts | 0 .../references/self_references/input.ts | 0 .../references/simple_references/input.ts | 0 .../references/simple_references/output.ts | 0 .../dictionary/test/references.spec.ts | 0 .../dictionary/test/versionUtils.spec.ts | 0 .../dictionary/tsconfig.json | 0 .../validation/.mocharc.json | 0 {libraries => packages}/validation/README.md | 0 .../validation/package.json | 0 .../fieldNamesValidation.ts | 0 .../valueTypeValidation.ts | 0 .../fieldRestrictions/codeListValidation.ts | 0 .../src/fieldRestrictions/rangeValidation.ts | 0 .../src/fieldRestrictions/regexValidation.ts | 0 .../fieldRestrictions/requiredValidation.ts | 0 .../src/fieldRestrictions/scriptValidation.ts | 0 .../validation/src/index.ts | 0 .../foreignKeysValidation.ts | 0 .../schemaRestrictions/uniqueKeyValidation.ts | 0 .../schemaRestrictions/uniqueValidation.ts | 0 .../validation/src/types/index.ts | 0 .../src/types/validationErrorTypes.ts | 0 .../src/types/validationFunctionTypes.ts | 0 .../validation/src/utils/datasetUtils.ts | 0 .../validation/src/utils/isEmptyString.ts | 0 .../validation/src/utils/rangeToSymbol.ts | 0 .../validation/src/utils/typeUtils.ts | 0 .../validation/test/placeholder.spec.ts | 0 .../validation/tsconfig.json | 0 pnpm-lock.yaml | 110 +++++++++--------- pnpm-workspace.yaml | 1 - 86 files changed, 214 insertions(+), 74 deletions(-) rename {libraries => packages}/common/.mocharc.json (100%) rename {libraries => packages}/common/README.md (100%) rename {libraries => packages}/common/package.json (100%) rename {libraries => packages}/common/src/errors.ts (100%) rename {libraries => packages}/common/src/index.ts (100%) rename {libraries => packages}/common/src/types/generics.ts (100%) rename {libraries => packages}/common/src/types/index.ts (100%) rename {libraries => packages}/common/src/types/result.ts (100%) rename {libraries => packages}/common/src/types/singular.ts (100%) rename {libraries => packages}/common/src/utils/index.ts (100%) create mode 100644 packages/common/src/utils/objectUtils.ts rename {libraries => packages}/common/src/utils/stringUtils.ts (100%) rename {libraries => packages}/common/src/utils/typeUtils.ts (100%) rename {libraries => packages}/common/tsconfig.json (100%) rename {libraries => packages}/dictionary/.mocharc.json (100%) rename {libraries => packages}/dictionary/README.md (100%) rename {libraries => packages}/dictionary/package.json (100%) rename {libraries => packages}/dictionary/src/diff.ts (100%) rename {libraries => packages}/dictionary/src/index.ts (100%) rename {libraries => packages}/dictionary/src/references.ts (100%) rename {libraries => packages}/dictionary/src/types/dataTypes.ts (100%) rename {libraries => packages}/dictionary/src/types/dbTypes.ts (100%) rename {libraries => packages}/dictionary/src/types/dictionaryTypes.ts (100%) rename {libraries => packages}/dictionary/src/types/diffTypes.ts (100%) rename {libraries => packages}/dictionary/src/types/index.ts (100%) rename {libraries => packages}/dictionary/src/types/referenceTypes.ts (100%) rename {libraries => packages}/dictionary/src/utils/allUnique.ts (100%) create mode 100644 packages/dictionary/src/utils/resolveRestrictions.ts rename {libraries => packages}/dictionary/src/utils/schemaUtils.ts (100%) rename {libraries => packages}/dictionary/src/utils/version.ts (100%) rename {libraries => packages}/dictionary/test/dataTypes.spec.ts (100%) rename {libraries => packages}/dictionary/test/dictionaryTypes.spec.ts (100%) rename {libraries => packages}/dictionary/test/diff.spec.ts (100%) rename {libraries => packages}/dictionary/test/fixtures/diff/initial.ts (100%) rename {libraries => packages}/dictionary/test/fixtures/diff/updated.ts (100%) rename {libraries => packages}/dictionary/test/fixtures/references/codeList_references/input.ts (100%) rename {libraries => packages}/dictionary/test/fixtures/references/codeList_references/output.ts (100%) rename {libraries => packages}/dictionary/test/fixtures/references/cyclic_references/input.ts (100%) rename {libraries => packages}/dictionary/test/fixtures/references/cyclic_references/output.ts (100%) rename {libraries => packages}/dictionary/test/fixtures/references/empty_references_section/input.ts (100%) rename {libraries => packages}/dictionary/test/fixtures/references/empty_references_section/output.ts (100%) rename {libraries => packages}/dictionary/test/fixtures/references/no_references_section/input.ts (100%) rename {libraries => packages}/dictionary/test/fixtures/references/no_references_section/output.ts (100%) rename {libraries => packages}/dictionary/test/fixtures/references/non_existing_references/input.ts (100%) rename {libraries => packages}/dictionary/test/fixtures/references/non_existing_references/output.ts (100%) rename {libraries => packages}/dictionary/test/fixtures/references/references_within_references/input.ts (100%) rename {libraries => packages}/dictionary/test/fixtures/references/references_within_references/output.ts (100%) rename {libraries => packages}/dictionary/test/fixtures/references/regex_reference/input.ts (100%) rename {libraries => packages}/dictionary/test/fixtures/references/regex_reference/input_with_array.ts (100%) rename {libraries => packages}/dictionary/test/fixtures/references/regex_reference/output.ts (100%) rename {libraries => packages}/dictionary/test/fixtures/references/script_references/input.ts (100%) rename {libraries => packages}/dictionary/test/fixtures/references/script_references/output.ts (100%) rename {libraries => packages}/dictionary/test/fixtures/references/self_references/input.ts (100%) rename {libraries => packages}/dictionary/test/fixtures/references/simple_references/input.ts (100%) rename {libraries => packages}/dictionary/test/fixtures/references/simple_references/output.ts (100%) rename {libraries => packages}/dictionary/test/references.spec.ts (100%) rename {libraries => packages}/dictionary/test/versionUtils.spec.ts (100%) rename {libraries => packages}/dictionary/tsconfig.json (100%) rename {libraries => packages}/validation/.mocharc.json (100%) rename {libraries => packages}/validation/README.md (100%) rename {libraries => packages}/validation/package.json (100%) rename {libraries => packages}/validation/src/dataRecordValidation/fieldNamesValidation.ts (100%) rename {libraries => packages}/validation/src/dataRecordValidation/valueTypeValidation.ts (100%) rename {libraries => packages}/validation/src/fieldRestrictions/codeListValidation.ts (100%) rename {libraries => packages}/validation/src/fieldRestrictions/rangeValidation.ts (100%) rename {libraries => packages}/validation/src/fieldRestrictions/regexValidation.ts (100%) rename {libraries => packages}/validation/src/fieldRestrictions/requiredValidation.ts (100%) rename {libraries => packages}/validation/src/fieldRestrictions/scriptValidation.ts (100%) rename {libraries => packages}/validation/src/index.ts (100%) rename {libraries => packages}/validation/src/schemaRestrictions/foreignKeysValidation.ts (100%) rename {libraries => packages}/validation/src/schemaRestrictions/uniqueKeyValidation.ts (100%) rename {libraries => packages}/validation/src/schemaRestrictions/uniqueValidation.ts (100%) rename {libraries => packages}/validation/src/types/index.ts (100%) rename {libraries => packages}/validation/src/types/validationErrorTypes.ts (100%) rename {libraries => packages}/validation/src/types/validationFunctionTypes.ts (100%) rename {libraries => packages}/validation/src/utils/datasetUtils.ts (100%) rename {libraries => packages}/validation/src/utils/isEmptyString.ts (100%) rename {libraries => packages}/validation/src/utils/rangeToSymbol.ts (100%) rename {libraries => packages}/validation/src/utils/typeUtils.ts (100%) rename {libraries => packages}/validation/test/placeholder.spec.ts (100%) rename {libraries => packages}/validation/tsconfig.json (100%) diff --git a/.dockerignore b/.dockerignore index 25fe2ea..e20d6c9 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,7 +3,6 @@ # Allow select paths !apps/* -!libraries/* !packages/* !package.json !pnpm-lock.yaml diff --git a/README.md b/README.md index 0614fcc..9a70d86 100644 --- a/README.md +++ b/README.md @@ -23,27 +23,25 @@ The repository is organized with the following directory structure: . ├── apps/ │ └── server -├── libraries/ -│ ├── common -│ ├── dictionary -│ └── validation └── packages/ - └── client + ├── client + ├── common + ├── dictionary + └── validation ``` The modules in the monorepo are organized into three categories: * __apps/__ - Standalone processes meant to be run. These are published to [ghcr.io](https://ghcr.io) as container images. - * __libraries/__ - Interal modules shared between other apps, libraries, and packages. - * __packages/__ - Packages published to [NPM](https://npmjs.com) meant to be imported into other TypeScript applications. - -| Component | Type | Package Name | Path | Published Location | Description | -| -------------------------------------------- | ----------- | ------------------------------ | --------------------- | ------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [Lectern Server](apps/server/README.md) | Application | server | apps/server/ | [GHCR](https://github.com/overture-stack/lectern/pkgs/container/lectern) | Lectern Server web application. | -| [Lectern Client](packages/client/README.md) | Package | @overture-stack/lectern-client | packages/client | [NPM](https://www.npmjs.com/package/@overture-stack/lectern-client) | TypeScript Client to interact with Lectern Server and perform data validation. | -| [common](libraries/common/README.md) | Library | common | libraries/common/ | N/A | Non-specific but commonly reusable utilities. Includes shared Error classes. | -| [dictionary](libraries/dictionary/README.md) | Library | dictionary | libraries/dictionary/ | N/A | Dictionary meta-schema definition, includes TS types, and Zod schemas. This also exports all utilities for getting the diff of two dictionaries, and for validating data records with a Dictionary. | -| [validation](libraries/validation/README.md) | Library | @overture-stack/lectern-validation | libraries/validation/ | [NPM](https://www.npmjs.com/package/@overture-stack/lectern-validation) | Validate data using Lectern Dictionaries. | + * __packages/__ - Reusable packages shared between applications and other packages. Packages are published to [NPM](https://npmjs.com) . + +| Component | Type | Package Name | Path | Published Location | Description | +| ------------------------------------------- | ----------- | ---------------------------------- | -------------------- | ------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [Lectern Server](apps/server/README.md) | Application | server | apps/server/ | [GHCR](https://github.com/overture-stack/lectern/pkgs/container/lectern) | Lectern Server web application. | +| [Lectern Client](packages/client/README.md) | Package | @overture-stack/lectern-client | packages/client | [NPM](https://www.npmjs.com/package/@overture-stack/lectern-client) | TypeScript Client to interact with Lectern Server and perform data validation. | +| [common](packages/common/README.md) | Package | common | packages/common/ | N/A | Non-specific but commonly reusable utilities. Includes shared Error classes. | +| [dictionary](packages/dictionary/README.md) | Package | dictionary | packages/dictionary/ | N/A | Dictionary meta-schema definition, includes TS types, and Zod schemas. This also exports all utilities for getting the diff of two dictionaries, and for validating data records with a Dictionary. | +| [validation](packages/validation/README.md) | Package | @overture-stack/lectern-validation | packages/validation/ | [NPM](https://www.npmjs.com/package/@overture-stack/lectern-validation) | Validate data using Lectern Dictionaries. | ## Developer Instructions diff --git a/packages/client/src/processing/validationPipelines.ts b/packages/client/src/processing/validationPipelines.ts index 47d335d..a5fe2f5 100644 --- a/packages/client/src/processing/validationPipelines.ts +++ b/packages/client/src/processing/validationPipelines.ts @@ -19,13 +19,13 @@ import { DataRecord, UnprocessedDataRecord, Schema, SchemaField } from 'dictionary'; -import { SchemaValidationError } from '../../../../libraries/validation/src/types/validationErrorTypes'; +import { SchemaValidationError } from '@overture-stack/lectern-validation'; import { CrossSchemaValidationFunction, DatasetValidationFunction, UnprocessedRecordValidationFunction, ValidationFunction, -} from '../../../../libraries/validation/src/types/validationFunctionTypes'; +} from '@overture-stack/lectern-validation'; export const runUnprocessedRecordValidationPipeline = ( record: UnprocessedDataRecord, diff --git a/libraries/common/.mocharc.json b/packages/common/.mocharc.json similarity index 100% rename from libraries/common/.mocharc.json rename to packages/common/.mocharc.json diff --git a/libraries/common/README.md b/packages/common/README.md similarity index 100% rename from libraries/common/README.md rename to packages/common/README.md diff --git a/libraries/common/package.json b/packages/common/package.json similarity index 100% rename from libraries/common/package.json rename to packages/common/package.json diff --git a/libraries/common/src/errors.ts b/packages/common/src/errors.ts similarity index 100% rename from libraries/common/src/errors.ts rename to packages/common/src/errors.ts diff --git a/libraries/common/src/index.ts b/packages/common/src/index.ts similarity index 100% rename from libraries/common/src/index.ts rename to packages/common/src/index.ts diff --git a/libraries/common/src/types/generics.ts b/packages/common/src/types/generics.ts similarity index 100% rename from libraries/common/src/types/generics.ts rename to packages/common/src/types/generics.ts diff --git a/libraries/common/src/types/index.ts b/packages/common/src/types/index.ts similarity index 100% rename from libraries/common/src/types/index.ts rename to packages/common/src/types/index.ts diff --git a/libraries/common/src/types/result.ts b/packages/common/src/types/result.ts similarity index 100% rename from libraries/common/src/types/result.ts rename to packages/common/src/types/result.ts diff --git a/libraries/common/src/types/singular.ts b/packages/common/src/types/singular.ts similarity index 100% rename from libraries/common/src/types/singular.ts rename to packages/common/src/types/singular.ts diff --git a/libraries/common/src/utils/index.ts b/packages/common/src/utils/index.ts similarity index 100% rename from libraries/common/src/utils/index.ts rename to packages/common/src/utils/index.ts diff --git a/packages/common/src/utils/objectUtils.ts b/packages/common/src/utils/objectUtils.ts new file mode 100644 index 0000000..48dcba8 --- /dev/null +++ b/packages/common/src/utils/objectUtils.ts @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2023 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +type CompareFunction = (a: T, b: T) => number; + +const defaultSort = (a: T, b: T) => { + return a > b ? 1 : -1; +}; + +/** + * Wrapper for a search function that can define a priority order for specific values. This will return a + * search function that will check if values are in the priorityArray, and if so will sort them in the + * order found in the array. All values not found in the array will be sorted according to the provided + * sortingFunction. + * @param priorityArray Array of values in a specified priority sort order + * @param sortingFunction Optional sorting function for case where neither value is found in the priorityArray. + * Default sorting function is the defaultSort which does a JavaScript greater than operation to compare values. + * @returns A new CompareFunction that will prioritize items found in the priorityArray. + * + * @example + * ``` + * const array = ["c", "z", "x", "b", "a", "y"]; + * const customSortFunction = sortWithPriority(["z", "y", "x"]); + * + * const sortedArray = array.sort(customSortFunction); + * // Result: ["z", "y", "x", "a", "b", "c"]; + * ``` + */ +export const sortWithPriority = + (priorityArray: T[], sortingFunction: CompareFunction = defaultSort): CompareFunction => + (a, b) => { + const indexA = priorityArray.indexOf(a); + const indexB = priorityArray.indexOf(b); + if (indexA >= 0) { + if (indexB >= 0) { + // Has A and B + return indexA > indexB ? 1 : -1; + } else { + // Has A not B + return -1; + } + } else if (indexB >= 0) { + // Has B not A + return 1; + } else { + // Has neither, use sortingFunction for remainder + return sortingFunction(a, b); + } + }; + +/** + * Create a deep clone of an object with all properties sorted. This is helpful for preparing a data structure + * for presentation in a predictable way. For example, a user submitted JSON object will have + * properties sorted randomly, while an API response that includes this user data should be always returned with + * object properties in a consistent order. + * + * The sort order will by default be alphabetical. A custom sortingFunction can be provided to + * sort the object properties with custom order or logic. + * + * Note: This only operates on own properties, not inherited properties. + * @param input An object to be cloned with sorted properties + * @param sortingFunction Optional property, a sorting function to determine the order of properties + * in the output object + * @returns Deep clone of the input object + */ +export const sortProperties = (input: T, sortingFunction = defaultSort): T => { + const sortedEntries = Object.keys(input).sort(sortingFunction); + return sortedEntries.reduce((acc, key) => { + const value = (input as any)[key]; + if (typeof value === 'object' && value !== null) { + acc[key] = sortProperties(value, sortingFunction); + } else { + acc[key] = value; + } + return acc; + }, {}); +}; diff --git a/libraries/common/src/utils/stringUtils.ts b/packages/common/src/utils/stringUtils.ts similarity index 100% rename from libraries/common/src/utils/stringUtils.ts rename to packages/common/src/utils/stringUtils.ts diff --git a/libraries/common/src/utils/typeUtils.ts b/packages/common/src/utils/typeUtils.ts similarity index 100% rename from libraries/common/src/utils/typeUtils.ts rename to packages/common/src/utils/typeUtils.ts diff --git a/libraries/common/tsconfig.json b/packages/common/tsconfig.json similarity index 100% rename from libraries/common/tsconfig.json rename to packages/common/tsconfig.json diff --git a/libraries/dictionary/.mocharc.json b/packages/dictionary/.mocharc.json similarity index 100% rename from libraries/dictionary/.mocharc.json rename to packages/dictionary/.mocharc.json diff --git a/libraries/dictionary/README.md b/packages/dictionary/README.md similarity index 100% rename from libraries/dictionary/README.md rename to packages/dictionary/README.md diff --git a/libraries/dictionary/package.json b/packages/dictionary/package.json similarity index 100% rename from libraries/dictionary/package.json rename to packages/dictionary/package.json diff --git a/libraries/dictionary/src/diff.ts b/packages/dictionary/src/diff.ts similarity index 100% rename from libraries/dictionary/src/diff.ts rename to packages/dictionary/src/diff.ts diff --git a/libraries/dictionary/src/index.ts b/packages/dictionary/src/index.ts similarity index 100% rename from libraries/dictionary/src/index.ts rename to packages/dictionary/src/index.ts diff --git a/libraries/dictionary/src/references.ts b/packages/dictionary/src/references.ts similarity index 100% rename from libraries/dictionary/src/references.ts rename to packages/dictionary/src/references.ts diff --git a/libraries/dictionary/src/types/dataTypes.ts b/packages/dictionary/src/types/dataTypes.ts similarity index 100% rename from libraries/dictionary/src/types/dataTypes.ts rename to packages/dictionary/src/types/dataTypes.ts diff --git a/libraries/dictionary/src/types/dbTypes.ts b/packages/dictionary/src/types/dbTypes.ts similarity index 100% rename from libraries/dictionary/src/types/dbTypes.ts rename to packages/dictionary/src/types/dbTypes.ts diff --git a/libraries/dictionary/src/types/dictionaryTypes.ts b/packages/dictionary/src/types/dictionaryTypes.ts similarity index 100% rename from libraries/dictionary/src/types/dictionaryTypes.ts rename to packages/dictionary/src/types/dictionaryTypes.ts diff --git a/libraries/dictionary/src/types/diffTypes.ts b/packages/dictionary/src/types/diffTypes.ts similarity index 100% rename from libraries/dictionary/src/types/diffTypes.ts rename to packages/dictionary/src/types/diffTypes.ts diff --git a/libraries/dictionary/src/types/index.ts b/packages/dictionary/src/types/index.ts similarity index 100% rename from libraries/dictionary/src/types/index.ts rename to packages/dictionary/src/types/index.ts diff --git a/libraries/dictionary/src/types/referenceTypes.ts b/packages/dictionary/src/types/referenceTypes.ts similarity index 100% rename from libraries/dictionary/src/types/referenceTypes.ts rename to packages/dictionary/src/types/referenceTypes.ts diff --git a/libraries/dictionary/src/utils/allUnique.ts b/packages/dictionary/src/utils/allUnique.ts similarity index 100% rename from libraries/dictionary/src/utils/allUnique.ts rename to packages/dictionary/src/utils/allUnique.ts diff --git a/packages/dictionary/src/utils/resolveRestrictions.ts b/packages/dictionary/src/utils/resolveRestrictions.ts new file mode 100644 index 0000000..74460fd --- /dev/null +++ b/packages/dictionary/src/utils/resolveRestrictions.ts @@ -0,0 +1,51 @@ +// import { +// BooleanFieldRestrictions, +// IntegerFieldRestrictions, +// NumberFieldRestrictions, +// SchemaBooleanField, +// SchemaField, +// SchemaIntegerField, +// SchemaNumberField, +// SchemaStringField, +// StringFieldRestrictions, +// } from 'types'; + +// type SingleElement = T extends readonly (infer Element)[] ? Element : T; + +// /** +// * Given a schema field, get the type of the restriction object +// */ +// export type RestrictionObject = T extends { restrictions?: infer RestrictionType } +// ? SingleElement +// : never; + +// type BR = RestrictionObject; +// const x: BR = {}; + +// export const resolveRestrictions = ( +// field: T, +// ): RestrictionObject => { +// const y: RestrictionObject = { required: true }; + +// if (!field.restrictions) { +// return {}; +// } +// if (field.valueType === 'integer') { +// const restrictions = field.restrictions; +// return {}; +// } +// return {}; +// // if(!field.restrictions) { +// // return {}; +// // } +// // if(!Array.isArray(field.)) +// }; + +// type A = { type: 'a'; thing: number }; +// type B = { type: 'b'; thing: string }; +// type DUnion = A | B; + +// type ThingType = T extends { thing: infer U } ? U : never; + +// type AThing = ThingType; +// type BThing = ThingType; diff --git a/libraries/dictionary/src/utils/schemaUtils.ts b/packages/dictionary/src/utils/schemaUtils.ts similarity index 100% rename from libraries/dictionary/src/utils/schemaUtils.ts rename to packages/dictionary/src/utils/schemaUtils.ts diff --git a/libraries/dictionary/src/utils/version.ts b/packages/dictionary/src/utils/version.ts similarity index 100% rename from libraries/dictionary/src/utils/version.ts rename to packages/dictionary/src/utils/version.ts diff --git a/libraries/dictionary/test/dataTypes.spec.ts b/packages/dictionary/test/dataTypes.spec.ts similarity index 100% rename from libraries/dictionary/test/dataTypes.spec.ts rename to packages/dictionary/test/dataTypes.spec.ts diff --git a/libraries/dictionary/test/dictionaryTypes.spec.ts b/packages/dictionary/test/dictionaryTypes.spec.ts similarity index 100% rename from libraries/dictionary/test/dictionaryTypes.spec.ts rename to packages/dictionary/test/dictionaryTypes.spec.ts diff --git a/libraries/dictionary/test/diff.spec.ts b/packages/dictionary/test/diff.spec.ts similarity index 100% rename from libraries/dictionary/test/diff.spec.ts rename to packages/dictionary/test/diff.spec.ts diff --git a/libraries/dictionary/test/fixtures/diff/initial.ts b/packages/dictionary/test/fixtures/diff/initial.ts similarity index 100% rename from libraries/dictionary/test/fixtures/diff/initial.ts rename to packages/dictionary/test/fixtures/diff/initial.ts diff --git a/libraries/dictionary/test/fixtures/diff/updated.ts b/packages/dictionary/test/fixtures/diff/updated.ts similarity index 100% rename from libraries/dictionary/test/fixtures/diff/updated.ts rename to packages/dictionary/test/fixtures/diff/updated.ts diff --git a/libraries/dictionary/test/fixtures/references/codeList_references/input.ts b/packages/dictionary/test/fixtures/references/codeList_references/input.ts similarity index 100% rename from libraries/dictionary/test/fixtures/references/codeList_references/input.ts rename to packages/dictionary/test/fixtures/references/codeList_references/input.ts diff --git a/libraries/dictionary/test/fixtures/references/codeList_references/output.ts b/packages/dictionary/test/fixtures/references/codeList_references/output.ts similarity index 100% rename from libraries/dictionary/test/fixtures/references/codeList_references/output.ts rename to packages/dictionary/test/fixtures/references/codeList_references/output.ts diff --git a/libraries/dictionary/test/fixtures/references/cyclic_references/input.ts b/packages/dictionary/test/fixtures/references/cyclic_references/input.ts similarity index 100% rename from libraries/dictionary/test/fixtures/references/cyclic_references/input.ts rename to packages/dictionary/test/fixtures/references/cyclic_references/input.ts diff --git a/libraries/dictionary/test/fixtures/references/cyclic_references/output.ts b/packages/dictionary/test/fixtures/references/cyclic_references/output.ts similarity index 100% rename from libraries/dictionary/test/fixtures/references/cyclic_references/output.ts rename to packages/dictionary/test/fixtures/references/cyclic_references/output.ts diff --git a/libraries/dictionary/test/fixtures/references/empty_references_section/input.ts b/packages/dictionary/test/fixtures/references/empty_references_section/input.ts similarity index 100% rename from libraries/dictionary/test/fixtures/references/empty_references_section/input.ts rename to packages/dictionary/test/fixtures/references/empty_references_section/input.ts diff --git a/libraries/dictionary/test/fixtures/references/empty_references_section/output.ts b/packages/dictionary/test/fixtures/references/empty_references_section/output.ts similarity index 100% rename from libraries/dictionary/test/fixtures/references/empty_references_section/output.ts rename to packages/dictionary/test/fixtures/references/empty_references_section/output.ts diff --git a/libraries/dictionary/test/fixtures/references/no_references_section/input.ts b/packages/dictionary/test/fixtures/references/no_references_section/input.ts similarity index 100% rename from libraries/dictionary/test/fixtures/references/no_references_section/input.ts rename to packages/dictionary/test/fixtures/references/no_references_section/input.ts diff --git a/libraries/dictionary/test/fixtures/references/no_references_section/output.ts b/packages/dictionary/test/fixtures/references/no_references_section/output.ts similarity index 100% rename from libraries/dictionary/test/fixtures/references/no_references_section/output.ts rename to packages/dictionary/test/fixtures/references/no_references_section/output.ts diff --git a/libraries/dictionary/test/fixtures/references/non_existing_references/input.ts b/packages/dictionary/test/fixtures/references/non_existing_references/input.ts similarity index 100% rename from libraries/dictionary/test/fixtures/references/non_existing_references/input.ts rename to packages/dictionary/test/fixtures/references/non_existing_references/input.ts diff --git a/libraries/dictionary/test/fixtures/references/non_existing_references/output.ts b/packages/dictionary/test/fixtures/references/non_existing_references/output.ts similarity index 100% rename from libraries/dictionary/test/fixtures/references/non_existing_references/output.ts rename to packages/dictionary/test/fixtures/references/non_existing_references/output.ts diff --git a/libraries/dictionary/test/fixtures/references/references_within_references/input.ts b/packages/dictionary/test/fixtures/references/references_within_references/input.ts similarity index 100% rename from libraries/dictionary/test/fixtures/references/references_within_references/input.ts rename to packages/dictionary/test/fixtures/references/references_within_references/input.ts diff --git a/libraries/dictionary/test/fixtures/references/references_within_references/output.ts b/packages/dictionary/test/fixtures/references/references_within_references/output.ts similarity index 100% rename from libraries/dictionary/test/fixtures/references/references_within_references/output.ts rename to packages/dictionary/test/fixtures/references/references_within_references/output.ts diff --git a/libraries/dictionary/test/fixtures/references/regex_reference/input.ts b/packages/dictionary/test/fixtures/references/regex_reference/input.ts similarity index 100% rename from libraries/dictionary/test/fixtures/references/regex_reference/input.ts rename to packages/dictionary/test/fixtures/references/regex_reference/input.ts diff --git a/libraries/dictionary/test/fixtures/references/regex_reference/input_with_array.ts b/packages/dictionary/test/fixtures/references/regex_reference/input_with_array.ts similarity index 100% rename from libraries/dictionary/test/fixtures/references/regex_reference/input_with_array.ts rename to packages/dictionary/test/fixtures/references/regex_reference/input_with_array.ts diff --git a/libraries/dictionary/test/fixtures/references/regex_reference/output.ts b/packages/dictionary/test/fixtures/references/regex_reference/output.ts similarity index 100% rename from libraries/dictionary/test/fixtures/references/regex_reference/output.ts rename to packages/dictionary/test/fixtures/references/regex_reference/output.ts diff --git a/libraries/dictionary/test/fixtures/references/script_references/input.ts b/packages/dictionary/test/fixtures/references/script_references/input.ts similarity index 100% rename from libraries/dictionary/test/fixtures/references/script_references/input.ts rename to packages/dictionary/test/fixtures/references/script_references/input.ts diff --git a/libraries/dictionary/test/fixtures/references/script_references/output.ts b/packages/dictionary/test/fixtures/references/script_references/output.ts similarity index 100% rename from libraries/dictionary/test/fixtures/references/script_references/output.ts rename to packages/dictionary/test/fixtures/references/script_references/output.ts diff --git a/libraries/dictionary/test/fixtures/references/self_references/input.ts b/packages/dictionary/test/fixtures/references/self_references/input.ts similarity index 100% rename from libraries/dictionary/test/fixtures/references/self_references/input.ts rename to packages/dictionary/test/fixtures/references/self_references/input.ts diff --git a/libraries/dictionary/test/fixtures/references/simple_references/input.ts b/packages/dictionary/test/fixtures/references/simple_references/input.ts similarity index 100% rename from libraries/dictionary/test/fixtures/references/simple_references/input.ts rename to packages/dictionary/test/fixtures/references/simple_references/input.ts diff --git a/libraries/dictionary/test/fixtures/references/simple_references/output.ts b/packages/dictionary/test/fixtures/references/simple_references/output.ts similarity index 100% rename from libraries/dictionary/test/fixtures/references/simple_references/output.ts rename to packages/dictionary/test/fixtures/references/simple_references/output.ts diff --git a/libraries/dictionary/test/references.spec.ts b/packages/dictionary/test/references.spec.ts similarity index 100% rename from libraries/dictionary/test/references.spec.ts rename to packages/dictionary/test/references.spec.ts diff --git a/libraries/dictionary/test/versionUtils.spec.ts b/packages/dictionary/test/versionUtils.spec.ts similarity index 100% rename from libraries/dictionary/test/versionUtils.spec.ts rename to packages/dictionary/test/versionUtils.spec.ts diff --git a/libraries/dictionary/tsconfig.json b/packages/dictionary/tsconfig.json similarity index 100% rename from libraries/dictionary/tsconfig.json rename to packages/dictionary/tsconfig.json diff --git a/libraries/validation/.mocharc.json b/packages/validation/.mocharc.json similarity index 100% rename from libraries/validation/.mocharc.json rename to packages/validation/.mocharc.json diff --git a/libraries/validation/README.md b/packages/validation/README.md similarity index 100% rename from libraries/validation/README.md rename to packages/validation/README.md diff --git a/libraries/validation/package.json b/packages/validation/package.json similarity index 100% rename from libraries/validation/package.json rename to packages/validation/package.json diff --git a/libraries/validation/src/dataRecordValidation/fieldNamesValidation.ts b/packages/validation/src/dataRecordValidation/fieldNamesValidation.ts similarity index 100% rename from libraries/validation/src/dataRecordValidation/fieldNamesValidation.ts rename to packages/validation/src/dataRecordValidation/fieldNamesValidation.ts diff --git a/libraries/validation/src/dataRecordValidation/valueTypeValidation.ts b/packages/validation/src/dataRecordValidation/valueTypeValidation.ts similarity index 100% rename from libraries/validation/src/dataRecordValidation/valueTypeValidation.ts rename to packages/validation/src/dataRecordValidation/valueTypeValidation.ts diff --git a/libraries/validation/src/fieldRestrictions/codeListValidation.ts b/packages/validation/src/fieldRestrictions/codeListValidation.ts similarity index 100% rename from libraries/validation/src/fieldRestrictions/codeListValidation.ts rename to packages/validation/src/fieldRestrictions/codeListValidation.ts diff --git a/libraries/validation/src/fieldRestrictions/rangeValidation.ts b/packages/validation/src/fieldRestrictions/rangeValidation.ts similarity index 100% rename from libraries/validation/src/fieldRestrictions/rangeValidation.ts rename to packages/validation/src/fieldRestrictions/rangeValidation.ts diff --git a/libraries/validation/src/fieldRestrictions/regexValidation.ts b/packages/validation/src/fieldRestrictions/regexValidation.ts similarity index 100% rename from libraries/validation/src/fieldRestrictions/regexValidation.ts rename to packages/validation/src/fieldRestrictions/regexValidation.ts diff --git a/libraries/validation/src/fieldRestrictions/requiredValidation.ts b/packages/validation/src/fieldRestrictions/requiredValidation.ts similarity index 100% rename from libraries/validation/src/fieldRestrictions/requiredValidation.ts rename to packages/validation/src/fieldRestrictions/requiredValidation.ts diff --git a/libraries/validation/src/fieldRestrictions/scriptValidation.ts b/packages/validation/src/fieldRestrictions/scriptValidation.ts similarity index 100% rename from libraries/validation/src/fieldRestrictions/scriptValidation.ts rename to packages/validation/src/fieldRestrictions/scriptValidation.ts diff --git a/libraries/validation/src/index.ts b/packages/validation/src/index.ts similarity index 100% rename from libraries/validation/src/index.ts rename to packages/validation/src/index.ts diff --git a/libraries/validation/src/schemaRestrictions/foreignKeysValidation.ts b/packages/validation/src/schemaRestrictions/foreignKeysValidation.ts similarity index 100% rename from libraries/validation/src/schemaRestrictions/foreignKeysValidation.ts rename to packages/validation/src/schemaRestrictions/foreignKeysValidation.ts diff --git a/libraries/validation/src/schemaRestrictions/uniqueKeyValidation.ts b/packages/validation/src/schemaRestrictions/uniqueKeyValidation.ts similarity index 100% rename from libraries/validation/src/schemaRestrictions/uniqueKeyValidation.ts rename to packages/validation/src/schemaRestrictions/uniqueKeyValidation.ts diff --git a/libraries/validation/src/schemaRestrictions/uniqueValidation.ts b/packages/validation/src/schemaRestrictions/uniqueValidation.ts similarity index 100% rename from libraries/validation/src/schemaRestrictions/uniqueValidation.ts rename to packages/validation/src/schemaRestrictions/uniqueValidation.ts diff --git a/libraries/validation/src/types/index.ts b/packages/validation/src/types/index.ts similarity index 100% rename from libraries/validation/src/types/index.ts rename to packages/validation/src/types/index.ts diff --git a/libraries/validation/src/types/validationErrorTypes.ts b/packages/validation/src/types/validationErrorTypes.ts similarity index 100% rename from libraries/validation/src/types/validationErrorTypes.ts rename to packages/validation/src/types/validationErrorTypes.ts diff --git a/libraries/validation/src/types/validationFunctionTypes.ts b/packages/validation/src/types/validationFunctionTypes.ts similarity index 100% rename from libraries/validation/src/types/validationFunctionTypes.ts rename to packages/validation/src/types/validationFunctionTypes.ts diff --git a/libraries/validation/src/utils/datasetUtils.ts b/packages/validation/src/utils/datasetUtils.ts similarity index 100% rename from libraries/validation/src/utils/datasetUtils.ts rename to packages/validation/src/utils/datasetUtils.ts diff --git a/libraries/validation/src/utils/isEmptyString.ts b/packages/validation/src/utils/isEmptyString.ts similarity index 100% rename from libraries/validation/src/utils/isEmptyString.ts rename to packages/validation/src/utils/isEmptyString.ts diff --git a/libraries/validation/src/utils/rangeToSymbol.ts b/packages/validation/src/utils/rangeToSymbol.ts similarity index 100% rename from libraries/validation/src/utils/rangeToSymbol.ts rename to packages/validation/src/utils/rangeToSymbol.ts diff --git a/libraries/validation/src/utils/typeUtils.ts b/packages/validation/src/utils/typeUtils.ts similarity index 100% rename from libraries/validation/src/utils/typeUtils.ts rename to packages/validation/src/utils/typeUtils.ts diff --git a/libraries/validation/test/placeholder.spec.ts b/packages/validation/test/placeholder.spec.ts similarity index 100% rename from libraries/validation/test/placeholder.spec.ts rename to packages/validation/test/placeholder.spec.ts diff --git a/libraries/validation/tsconfig.json b/packages/validation/tsconfig.json similarity index 100% rename from libraries/validation/tsconfig.json rename to packages/validation/tsconfig.json diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 70c817f..78328d4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -68,10 +68,10 @@ importers: version: 1.20.2 common: specifier: workspace:^ - version: link:../../libraries/common + version: link:../../packages/common dictionary: specifier: workspace:^ - version: link:../../libraries/dictionary + version: link:../../packages/dictionary dotenv: specifier: ^16.3.1 version: 16.3.1 @@ -158,73 +158,23 @@ importers: specifier: ^3.21.3 version: 3.21.3(zod@3.21.4) - libraries/common: - devDependencies: - rimraf: - specifier: ^3.0.2 - version: 3.0.2 - - libraries/dictionary: - dependencies: - common: - specifier: workspace:^ - version: link:../common - immer: - specifier: ^10.0.2 - version: 10.0.2 - lodash: - specifier: ^4.17.21 - version: 4.17.21 - zod: - specifier: ^3.21.4 - version: 3.21.4 - devDependencies: - '@types/lodash': - specifier: ^4.14.195 - version: 4.14.195 - rimraf: - specifier: ^3.0.2 - version: 3.0.2 - - libraries/validation: - dependencies: - common: - specifier: workspace:^ - version: link:../common - dictionary: - specifier: workspace:^ - version: link:../dictionary - lodash: - specifier: ^4.17.21 - version: 4.17.21 - zod: - specifier: ^3.21.4 - version: 3.21.4 - devDependencies: - '@types/lodash': - specifier: ^4.14.195 - version: 4.14.195 - rimraf: - specifier: ^3.0.2 - version: 3.0.2 - packages/client: dependencies: '@overture-stack/lectern-validation': specifier: workspace:^ - version: link:../../libraries/validation + version: link:../validation cd: specifier: ^0.3.3 version: 0.3.3 common: specifier: workspace:^ - version: link:../../libraries/common + version: link:../common deep-freeze: specifier: ^0.0.1 version: 0.0.1 dictionary: specifier: workspace:^ - version: link:../../libraries/dictionary + version: link:../dictionary lodash: specifier: ^4.17.21 version: 4.17.21 @@ -287,6 +237,56 @@ importers: specifier: ^5.1.6 version: 5.1.6 + packages/common: + devDependencies: + rimraf: + specifier: ^3.0.2 + version: 3.0.2 + + packages/dictionary: + dependencies: + common: + specifier: workspace:^ + version: link:../common + immer: + specifier: ^10.0.2 + version: 10.0.2 + lodash: + specifier: ^4.17.21 + version: 4.17.21 + zod: + specifier: ^3.21.4 + version: 3.21.4 + devDependencies: + '@types/lodash': + specifier: ^4.14.195 + version: 4.14.195 + rimraf: + specifier: ^3.0.2 + version: 3.0.2 + + packages/validation: + dependencies: + common: + specifier: workspace:^ + version: link:../common + dictionary: + specifier: workspace:^ + version: link:../dictionary + lodash: + specifier: ^4.17.21 + version: 4.17.21 + zod: + specifier: ^3.21.4 + version: 3.21.4 + devDependencies: + '@types/lodash': + specifier: ^4.14.195 + version: 4.14.195 + rimraf: + specifier: ^3.0.2 + version: 3.0.2 + packages: /@ampproject/remapping@2.2.1: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 8d26293..c53e539 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,4 +1,3 @@ packages: - 'apps/*' - - 'libraries/*' - 'packages/*' \ No newline at end of file From 6dc405bd43f4594ad5e50fee843e5d7ea5119fe5 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Mon, 17 Jun 2024 20:49:59 -0400 Subject: [PATCH 15/27] Corrected a word: two categories --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9a70d86..462faa2 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ The repository is organized with the following directory structure: └── validation ``` -The modules in the monorepo are organized into three categories: +The modules in the monorepo are organized into two categories: * __apps/__ - Standalone processes meant to be run. These are published to [ghcr.io](https://ghcr.io) as container images. * __packages/__ - Reusable packages shared between applications and other packages. Packages are published to [NPM](https://npmjs.com) . From 6784f6c74812ac3f7627ac53d06679088e57f0bf Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Tue, 18 Jun 2024 01:36:56 -0400 Subject: [PATCH 16/27] Add description and code use examples --- packages/client/README.md | 52 +++++++++++++++++++++++++++++++++------ 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/packages/client/README.md b/packages/client/README.md index 1591e12..f517a4f 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -1,11 +1,47 @@ -# Lectern TypeScript Client +# Lectern Client [![NPM Version](https://img.shields.io/npm/v/@overture-stack/lectern-client?color=%23cb3837&style=for-the-badge&logo=npm)](https://www.npmjs.com/package/@overture-stack/lectern-client) -## Features: -- Runs different restrictions validations: regex, range, scripts, required fields, type checks, etc. -- Transforms the data from string to their proper type. -- Report validation errors. -- Fetch dictionaries from the configured lectern service. -- Provide typed definitions for the dictionary object. -- Analyze dictionary versions diff. +The Lectern client provides developers the mechanism to interact with Lectern servers and to use Lectern Dictionaries. The client provides all the validation logic to check that submitted data is valid based on the structure and restrictions of a Lectern dictionary. It also provides a REST client to fetch Lectern Dictionary data from a Lectern server. + +## Features +- Process data using a Lectern Dictionary: + - Convert raw string inputs into properly typed values. + - Check the structure of input data is valid. + - Apply all restrictions, both across schemas and on individual fields, to validate input data. + - Report all validation errors found in the input data. +- Interact with lectern servers: + - Fetch dictionary by name and version + - Fetch difference summaries between dictionary versions + +## Data Processing Usage + +### Process Data for a Single Schema + +The following example shows how to process data using the Lectern Client. The input `donorData` is presented as hardcoded, but in a typical scenario this would be submitted to the application through an uploaded TSV, form entry, or similar user submission system. + +To process data records which all belong to the same schema we use the `processRecords` function: + +```ts +import * as lectern from '@overture-stack/lectern-client'; +import type { Dictionary, UnprocessedDataRecord } from '@overture-stack/lectern-client'; + + +const dictionary: Dictionary = await getLecternDictionary(); + +const donorData: UnprocessedDataRecord = [{submitter_donor_id: "abc123", gender: "Male", age: "28"}, {submitter_donor_id: "def456", gender: "Female", age: "37"}] + +const { processedRecords, validationErrors } = lectern.process.schema(dictionary, "donors", donorData); +``` + +## Data Fetching Example + +```ts +import * as lectern from '@overture-stack/lectern-client'; + +const currentVersion = "2.3"; +const previousVersion = "2.1"; + +const dictionary = client.restClient.fetchSchema('http://lectern.example.com', 'dictionary-name', currentVersion); +const versionUpdates = client.restClient.fetchDiff('http://lectern.example.com', 'dictionary-name', currentVersion, previousVersion); +``` From b56d9341bac5653906ce787ae2764794c9b97fe6 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Tue, 18 Jun 2024 11:42:52 -0400 Subject: [PATCH 17/27] Use badges for published repository locations --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 462faa2..d51a298 100644 --- a/README.md +++ b/README.md @@ -37,11 +37,11 @@ The modules in the monorepo are organized into two categories: | Component | Type | Package Name | Path | Published Location | Description | | ------------------------------------------- | ----------- | ---------------------------------- | -------------------- | ------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [Lectern Server](apps/server/README.md) | Application | server | apps/server/ | [GHCR](https://github.com/overture-stack/lectern/pkgs/container/lectern) | Lectern Server web application. | -| [Lectern Client](packages/client/README.md) | Package | @overture-stack/lectern-client | packages/client | [NPM](https://www.npmjs.com/package/@overture-stack/lectern-client) | TypeScript Client to interact with Lectern Server and perform data validation. | +| [Lectern Server](apps/server/README.md) | Application | server | apps/server/ | [![Lectern GHCR Packages](https://img.shields.io/badge/GHCR-lectern-brightgreen?style=for-the-badge&logo=github)](https://github.com/overture-stack/lectern/pkgs/container/lectern) | Lectern Server web application. | +| [Lectern Client](packages/client/README.md) | Package | @overture-stack/lectern-client | packages/client | [![Lectern Client NPM Package](https://img.shields.io/npm/v/@overture-stack/lectern-client?color=%23cb3837&style=for-the-badge&logo=npm)](https://www.npmjs.com/package/@overture-stack/lectern-client) | TypeScript Client to interact with Lectern Server and perform data validation. | | [common](packages/common/README.md) | Package | common | packages/common/ | N/A | Non-specific but commonly reusable utilities. Includes shared Error classes. | | [dictionary](packages/dictionary/README.md) | Package | dictionary | packages/dictionary/ | N/A | Dictionary meta-schema definition, includes TS types, and Zod schemas. This also exports all utilities for getting the diff of two dictionaries, and for validating data records with a Dictionary. | -| [validation](packages/validation/README.md) | Package | @overture-stack/lectern-validation | packages/validation/ | [NPM](https://www.npmjs.com/package/@overture-stack/lectern-validation) | Validate data using Lectern Dictionaries. | +| [validation](packages/validation/README.md) | Package | @overture-stack/lectern-validation | packages/validation/ | [![Lectern Validation NPM Package](https://img.shields.io/npm/v/@overture-stack/lectern-client?color=%23cb3837&style=for-the-badge&logo=npm)](https://www.npmjs.com/package/@overture-stack/lectern-client) | Validate data using Lectern Dictionaries. | ## Developer Instructions From d2f0a941a6f00e67ee6b55aad2f4d096ec702169 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Wed, 19 Jun 2024 01:33:47 -0400 Subject: [PATCH 18/27] Remove unused deep-freeze dependency --- packages/client/package.json | 2 -- packages/client/src/utils.ts | 3 --- pnpm-lock.yaml | 14 -------------- 3 files changed, 19 deletions(-) diff --git a/packages/client/package.json b/packages/client/package.json index 9311314..5805d30 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -22,7 +22,6 @@ "license": "AGPL-3.0", "devDependencies": { "@types/chai": "^4.2.16", - "@types/deep-freeze": "^0.1.2", "@types/lodash": "^4.14.195", "@types/mocha": "^8.2.2", "@types/node": "^12.0.10", @@ -42,7 +41,6 @@ "@overture-stack/lectern-validation": "workspace:^", "cd": "^0.3.3", "common": "workspace:^", - "deep-freeze": "^0.0.1", "dictionary": "workspace:^", "lodash": "^4.17.21", "node-fetch": "^2.6.1", diff --git a/packages/client/src/utils.ts b/packages/client/src/utils.ts index bb4d2aa..1f294d8 100644 --- a/packages/client/src/utils.ts +++ b/packages/client/src/utils.ts @@ -17,7 +17,6 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import deepFreeze from 'deep-freeze'; import _ from 'lodash'; // type gaurd to filter out undefined and null @@ -115,5 +114,3 @@ export function isValueNotEqual(value: any, other: any) { export function convertToTrimmedString(val: unknown | undefined | string | number | boolean | null) { return val === undefined || val === null ? '' : String(val).trim(); } - -export const F = deepFreeze; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 78328d4..5037267 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -169,9 +169,6 @@ importers: common: specifier: workspace:^ version: link:../common - deep-freeze: - specifier: ^0.0.1 - version: 0.0.1 dictionary: specifier: workspace:^ version: link:../dictionary @@ -191,9 +188,6 @@ importers: '@types/chai': specifier: ^4.2.16 version: 4.3.5 - '@types/deep-freeze': - specifier: ^0.1.2 - version: 0.1.5 '@types/lodash': specifier: ^4.14.195 version: 4.14.195 @@ -789,10 +783,6 @@ packages: resolution: {integrity: sha512-t73xJJrvdTjXrn4jLS9VSGRbz0nUY3cl2DMGDU48lKl+HR9dbbjW2A9r3g40VA++mQpy6uuHg33gy7du2BKpog==} dev: true - /@types/deep-freeze@0.1.5: - resolution: {integrity: sha512-KZtR+jtmgkCpgE0f+We/QEI2Fi0towBV/tTkvHVhMzx+qhUVGXMx7pWvAtDp6vEWIjdKLTKpqbI/sORRCo8TKg==} - dev: true - /@types/errorhandler@0.0.32: resolution: {integrity: sha512-wC9CfwPMIzklPd5lEYC8HnQdlMC1PswlohWmEDMWlw+E/rMYuz5eSqKBc72Earb29KptKJrRl77qVRJzrZndww==} dependencies: @@ -1766,10 +1756,6 @@ packages: type-detect: 4.0.8 dev: true - /deep-freeze@0.0.1: - resolution: {integrity: sha512-Z+z8HiAvsGwmjqlphnHW5oz6yWlOwu6EQfFTjmeTWlDeda3FS2yv3jhq35TX/ewmsnqB+RX2IdsIOyjJCQN5tg==} - dev: false - /default-require-extensions@3.0.1: resolution: {integrity: sha512-eXTJmRbm2TIt9MgWTsOH1wEuhew6XGZcMeGKCtLedIg/NCsg1iBePXkceTdK4Fii7pzmN9tGsZhKzZ4h7O/fxw==} engines: {node: '>=8'} From 3b7cb3c4aea6aea5e396f5822e44948763a48960 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Fri, 21 Jun 2024 10:07:32 -0400 Subject: [PATCH 19/27] TS minmum up to 5.5, upgrade deprecated rimraf --- package.json | 7 ++- packages/client/package.json | 2 +- packages/common/package.json | 2 +- packages/dictionary/package.json | 2 +- packages/validation/package.json | 2 +- pnpm-lock.yaml | 80 ++++++++++++++++++++++++++------ 6 files changed, 74 insertions(+), 21 deletions(-) diff --git a/package.json b/package.json index 310d8d5..b18cb69 100644 --- a/package.json +++ b/package.json @@ -35,9 +35,8 @@ "nyc": "^15.1.0", "prettier": "^3", "sinon": "^15.2.0", - "typescript": "^5.1.6" - }, - "dependencies": { + "typescript": "^5.5.0", "ts-node": "^10.9.1" - } + }, + "dependencies": {} } diff --git a/packages/client/package.json b/packages/client/package.json index 5805d30..6e5660d 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -31,7 +31,7 @@ "mocha": "^8.3.2", "prettier": "^2.2.1", "pretty-quick": "^3.1.0", - "rimraf": "^3.0.2", + "rimraf": "^5.0.0", "ts-node": "^9.1.1", "tslint": "^6.1.3", "typedoc": "^0.17.7", diff --git a/packages/common/package.json b/packages/common/package.json index 322db78..80bb858 100644 --- a/packages/common/package.json +++ b/packages/common/package.json @@ -11,6 +11,6 @@ "author": "Ontario Institute for Cancer Research", "license": "AGPL-3.0", "devDependencies": { - "rimraf": "^3.0.2" + "rimraf": "^5.0.0" } } diff --git a/packages/dictionary/package.json b/packages/dictionary/package.json index 0b74639..fc24a33 100644 --- a/packages/dictionary/package.json +++ b/packages/dictionary/package.json @@ -19,6 +19,6 @@ }, "devDependencies": { "@types/lodash": "^4.14.195", - "rimraf": "^3.0.2" + "rimraf": "^5.0.0" } } diff --git a/packages/validation/package.json b/packages/validation/package.json index e36623c..42a12de 100644 --- a/packages/validation/package.json +++ b/packages/validation/package.json @@ -19,6 +19,6 @@ }, "devDependencies": { "@types/lodash": "^4.14.195", - "rimraf": "^3.0.2" + "rimraf": "^5.0.0" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5037267..aebb949 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -10,7 +10,7 @@ importers: dependencies: ts-node: specifier: ^10.9.1 - version: 10.9.1(@types/node@20.4.1)(typescript@5.1.6) + version: 10.9.1(@types/node@20.4.1)(typescript@5.5.2) devDependencies: '@types/chai': specifier: ^4.3.5 @@ -52,8 +52,8 @@ importers: specifier: ^15.2.0 version: 15.2.0 typescript: - specifier: ^5.1.6 - version: 5.1.6 + specifier: ^5.5.0 + version: 5.5.2 apps/server: dependencies: @@ -216,8 +216,8 @@ importers: specifier: ^3.1.0 version: 3.3.1(prettier@2.8.8) rimraf: - specifier: ^3.0.2 - version: 3.0.2 + specifier: ^4.0.0 + version: 4.4.1 ts-node: specifier: ^9.1.1 version: 9.1.1(typescript@5.1.6) @@ -234,8 +234,8 @@ importers: packages/common: devDependencies: rimraf: - specifier: ^3.0.2 - version: 3.0.2 + specifier: ^4.0.0 + version: 4.4.1 packages/dictionary: dependencies: @@ -256,8 +256,8 @@ importers: specifier: ^4.14.195 version: 4.14.195 rimraf: - specifier: ^3.0.2 - version: 3.0.2 + specifier: ^4.0.0 + version: 4.4.1 packages/validation: dependencies: @@ -278,8 +278,8 @@ importers: specifier: ^4.14.195 version: 4.14.195 rimraf: - specifier: ^3.0.2 - version: 3.0.2 + specifier: ^4.0.0 + version: 4.4.1 packages: @@ -2361,6 +2361,16 @@ packages: path-is-absolute: 1.0.1 dev: true + /glob@9.3.5: + resolution: {integrity: sha512-e1LleDykUz2Iu+MTYdkSsuWX8lvAjAcs0Xef0lNIu0S2wOAzuTxCJtcd9S3cijlwYF18EsU3rzb8jPVobxDh9Q==} + engines: {node: '>=16 || 14 >=14.17'} + dependencies: + fs.realpath: 1.0.0 + minimatch: 8.0.4 + minipass: 4.2.8 + path-scurry: 1.11.1 + dev: true + /globals@11.12.0: resolution: {integrity: sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==} engines: {node: '>=4'} @@ -3010,6 +3020,11 @@ packages: get-func-name: 2.0.0 dev: true + /lru-cache@10.2.2: + resolution: {integrity: sha512-9hp3Vp2/hFQUiIwKo8XCeFVnrg8Pk3TYNPIR7tJADKi5YfcF7vEaK7avFHTlSy3kOKYaJQaalfEo6YuXdceBOQ==} + engines: {node: 14 || >=16.14} + dev: true + /lru-cache@5.1.1: resolution: {integrity: sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==} dependencies: @@ -3150,10 +3165,27 @@ packages: brace-expansion: 2.0.1 dev: true + /minimatch@8.0.4: + resolution: {integrity: sha512-W0Wvr9HyFXZRGIDgCicunpQ299OKXs9RgZfaukz4qAW/pJhcpUfupc9c+OObPOFueNy8VSrZgEmDtk6Kh4WzDA==} + engines: {node: '>=16 || 14 >=14.17'} + dependencies: + brace-expansion: 2.0.1 + dev: true + /minimist@1.2.8: resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} dev: true + /minipass@4.2.8: + resolution: {integrity: sha512-fNzuVyifolSLFL4NzpF+wEF4qrgqaaKX0haXPQEdQ7NKAN+WecoKMHV09YcuL/DHxrUsYQOK3MiuDf7Ip2OXfQ==} + engines: {node: '>=8'} + dev: true + + /minipass@7.1.2: + resolution: {integrity: sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==} + engines: {node: '>=16 || 14 >=14.17'} + dev: true + /mkdirp-classic@0.5.3: resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==} dev: true @@ -3706,6 +3738,14 @@ packages: resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==} dev: true + /path-scurry@1.11.1: + resolution: {integrity: sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==} + engines: {node: '>=16 || 14 >=14.18'} + dependencies: + lru-cache: 10.2.2 + minipass: 7.1.2 + dev: true + /path-to-regexp@0.1.7: resolution: {integrity: sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==} dev: false @@ -4085,6 +4125,14 @@ packages: glob: 7.2.0 dev: true + /rimraf@4.4.1: + resolution: {integrity: sha512-Gk8NlF062+T9CqNGn6h4tls3k6T1+/nXdOcSZVikNVtlRdYpA7wRJJMoXmuvOnLW844rPjdQ7JgXCYM6PPC/og==} + engines: {node: '>=14'} + hasBin: true + dependencies: + glob: 9.3.5 + dev: true + /run-node@1.0.0: resolution: {integrity: sha512-kc120TBlQ3mih1LSzdAJXo4xn/GWS2ec0l3S+syHDXP9uRr0JAT8Qd3mdMuyjqCzeZktgP3try92cEgf9Nks8A==} engines: {node: '>=4'} @@ -4721,7 +4769,7 @@ packages: resolution: {integrity: sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw==} dev: false - /ts-node@10.9.1(@types/node@20.4.1)(typescript@5.1.6): + /ts-node@10.9.1(@types/node@20.4.1)(typescript@5.5.2): resolution: {integrity: sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==} hasBin: true peerDependencies: @@ -4747,7 +4795,7 @@ packages: create-require: 1.1.1 diff: 4.0.2 make-error: 1.3.6 - typescript: 5.1.6 + typescript: 5.5.2 v8-compile-cache-lib: 3.0.1 yn: 3.1.1 dev: false @@ -4909,6 +4957,12 @@ packages: resolution: {integrity: sha512-zaWCozRZ6DLEWAWFrVDz1H6FVXzUSfTy5FUMWsQlU8Ym5JP9eO4xkTIROFCQvhQf61z6O/G6ugw3SgAnvvm+HA==} engines: {node: '>=14.17'} hasBin: true + dev: true + + /typescript@5.5.2: + resolution: {integrity: sha512-NcRtPEOsPFFWjobJEtfihkLCZCXZt/os3zf8nTxjVH3RvTSxjrCamJpbExGvYOF+tFHc3pA65qpdwPbzjohhew==} + engines: {node: '>=14.17'} + hasBin: true /uglify-js@3.17.4: resolution: {integrity: sha512-T9q82TJI9e/C1TAxYvfb16xO120tMVFZrGA3f9/P4424DNu6ypK103y0GPFVa17yotwSyZW5iYXgjYHkGrJW/g==} From 2c06589a64ef6aeffb72ed78e3e111a47a42f818 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Sat, 22 Jun 2024 00:47:33 -0400 Subject: [PATCH 20/27] Dependency version updates --- package.json | 27 +- packages/client/package.json | 28 +- pnpm-lock.yaml | 1430 +++++++++++++++++++++------------- 3 files changed, 911 insertions(+), 574 deletions(-) diff --git a/package.json b/package.json index b18cb69..2cf4881 100644 --- a/package.json +++ b/package.json @@ -22,21 +22,20 @@ "author": "Ontario Institute for Cancer Research", "license": "AGPL-3.0", "devDependencies": { - "@types/chai": "^4.3.5", - "@types/chai-as-promised": "^7.1.5", - "@types/mocha": "^10", - "@types/node": "^20.4.1", - "@types/sinon": "^10.0.15", - "chai": "^4.3.7", - "chai-as-promised": "^7.1.1", + "@types/chai": "^4.3.16", + "@types/chai-as-promised": "^7.1.8", + "@types/mocha": "^10.0.6", + "@types/node": "^20.14.7", + "@types/sinon": "^10.0.20", + "chai": "^4.4.1", + "chai-as-promised": "^7.1.2", "chai-http": "^4.4.0", - "mocha": "^10", - "nx": "^16.5.0", + "mocha": "^10.4.0", + "nx": "^16.10.0", "nyc": "^15.1.0", - "prettier": "^3", + "prettier": "^3.3.2", "sinon": "^15.2.0", - "typescript": "^5.5.0", - "ts-node": "^10.9.1" - }, - "dependencies": {} + "ts-node": "^10.9.2", + "typescript": "^5.5.2" + } } diff --git a/packages/client/package.json b/packages/client/package.json index 6e5660d..1bb4c2d 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -21,21 +21,21 @@ }, "license": "AGPL-3.0", "devDependencies": { - "@types/chai": "^4.2.16", - "@types/lodash": "^4.14.195", - "@types/mocha": "^8.2.2", - "@types/node": "^12.0.10", - "@types/node-fetch": "^2.5.10", - "chai": "^4.3.4", + "@types/chai": "^4.3.16", + "@types/lodash": "^4.17.5", + "@types/mocha": "^8.2.3", + "@types/node": "^12.20.55", + "@types/node-fetch": "^2.6.11", + "chai": "^4.4.1", "husky": "^6.0.0", - "mocha": "^8.3.2", - "prettier": "^2.2.1", - "pretty-quick": "^3.1.0", - "rimraf": "^5.0.0", + "mocha": "^8.4.0", + "prettier": "^2.8.8", + "pretty-quick": "^3.3.1", + "rimraf": "^5.0.7", "ts-node": "^9.1.1", "tslint": "^6.1.3", - "typedoc": "^0.17.7", - "typescript": "^5.1.6" + "typedoc": "^0.17.8", + "typescript": "^5.5.2" }, "dependencies": { "@overture-stack/lectern-validation": "workspace:^", @@ -43,9 +43,9 @@ "common": "workspace:^", "dictionary": "workspace:^", "lodash": "^4.17.21", - "node-fetch": "^2.6.1", + "node-fetch": "^2.7.0", "promise-tools": "^2.1.0", - "winston": "^3.3.3" + "winston": "^3.13.0" }, "author": "Ontario Institute for Cancer Research" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index aebb949..9b21509 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -7,52 +7,51 @@ settings: importers: .: - dependencies: - ts-node: - specifier: ^10.9.1 - version: 10.9.1(@types/node@20.4.1)(typescript@5.5.2) devDependencies: '@types/chai': - specifier: ^4.3.5 - version: 4.3.5 + specifier: ^4.3.16 + version: 4.3.16 '@types/chai-as-promised': - specifier: ^7.1.5 - version: 7.1.5 + specifier: ^7.1.8 + version: 7.1.8 '@types/mocha': - specifier: ^10 - version: 10.0.0 + specifier: ^10.0.6 + version: 10.0.6 '@types/node': - specifier: ^20.4.1 - version: 20.4.1 + specifier: ^20.14.7 + version: 20.14.7 '@types/sinon': - specifier: ^10.0.15 - version: 10.0.15 + specifier: ^10.0.20 + version: 10.0.20 chai: - specifier: ^4.3.7 - version: 4.3.7 + specifier: ^4.4.1 + version: 4.4.1 chai-as-promised: - specifier: ^7.1.1 - version: 7.1.1(chai@4.3.7) + specifier: ^7.1.2 + version: 7.1.2(chai@4.4.1) chai-http: specifier: ^4.4.0 version: 4.4.0 mocha: - specifier: ^10 - version: 10.0.0 + specifier: ^10.4.0 + version: 10.4.0 nx: - specifier: ^16.5.0 - version: 16.5.0 + specifier: ^16.10.0 + version: 16.10.0 nyc: specifier: ^15.1.0 version: 15.1.0 prettier: - specifier: ^3 - version: 3.0.0 + specifier: ^3.3.2 + version: 3.3.2 sinon: specifier: ^15.2.0 version: 15.2.0 + ts-node: + specifier: ^10.9.2 + version: 10.9.2(@types/node@20.14.7)(typescript@5.5.2) typescript: - specifier: ^5.5.0 + specifier: ^5.5.2 version: 5.5.2 apps/server: @@ -176,66 +175,66 @@ importers: specifier: ^4.17.21 version: 4.17.21 node-fetch: - specifier: ^2.6.1 + specifier: ^2.7.0 version: 2.7.0 promise-tools: specifier: ^2.1.0 version: 2.1.0 winston: - specifier: ^3.3.3 - version: 3.9.0 + specifier: ^3.13.0 + version: 3.13.0 devDependencies: '@types/chai': - specifier: ^4.2.16 - version: 4.3.5 + specifier: ^4.3.16 + version: 4.3.16 '@types/lodash': - specifier: ^4.14.195 - version: 4.14.195 + specifier: ^4.17.5 + version: 4.17.5 '@types/mocha': - specifier: ^8.2.2 + specifier: ^8.2.3 version: 8.2.3 '@types/node': - specifier: ^12.0.10 + specifier: ^12.20.55 version: 12.20.55 '@types/node-fetch': - specifier: ^2.5.10 + specifier: ^2.6.11 version: 2.6.11 chai: - specifier: ^4.3.4 - version: 4.3.7 + specifier: ^4.4.1 + version: 4.4.1 husky: specifier: ^6.0.0 version: 6.0.0 mocha: - specifier: ^8.3.2 + specifier: ^8.4.0 version: 8.4.0 prettier: - specifier: ^2.2.1 + specifier: ^2.8.8 version: 2.8.8 pretty-quick: - specifier: ^3.1.0 + specifier: ^3.3.1 version: 3.3.1(prettier@2.8.8) rimraf: - specifier: ^4.0.0 - version: 4.4.1 + specifier: ^5.0.7 + version: 5.0.7 ts-node: specifier: ^9.1.1 - version: 9.1.1(typescript@5.1.6) + version: 9.1.1(typescript@5.5.2) tslint: specifier: ^6.1.3 - version: 6.1.3(typescript@5.1.6) + version: 6.1.3(typescript@5.5.2) typedoc: - specifier: ^0.17.7 - version: 0.17.8(typescript@5.1.6) + specifier: ^0.17.8 + version: 0.17.8(typescript@5.5.2) typescript: - specifier: ^5.1.6 - version: 5.1.6 + specifier: ^5.5.2 + version: 5.5.2 packages/common: devDependencies: rimraf: - specifier: ^4.0.0 - version: 4.4.1 + specifier: ^5.0.0 + version: 5.0.7 packages/dictionary: dependencies: @@ -256,8 +255,8 @@ importers: specifier: ^4.14.195 version: 4.14.195 rimraf: - specifier: ^4.0.0 - version: 4.4.1 + specifier: ^5.0.0 + version: 5.0.7 packages/validation: dependencies: @@ -278,17 +277,17 @@ importers: specifier: ^4.14.195 version: 4.14.195 rimraf: - specifier: ^4.0.0 - version: 4.4.1 + specifier: ^5.0.0 + version: 5.0.7 packages: - /@ampproject/remapping@2.2.1: - resolution: {integrity: sha512-lFMjJTrFL3j7L9yBxwYfCq2k6qqwHyzuUl/XBnif78PWTJYyL/dfowQHWE3sp6U6ZzqWiiIZnpTMO96zhkjwtg==} + /@ampproject/remapping@2.3.0: + resolution: {integrity: sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==} engines: {node: '>=6.0.0'} dependencies: - '@jridgewell/gen-mapping': 0.3.3 - '@jridgewell/trace-mapping': 0.3.18 + '@jridgewell/gen-mapping': 0.3.5 + '@jridgewell/trace-mapping': 0.3.25 dev: true /@babel/code-frame@7.22.5: @@ -298,117 +297,130 @@ packages: '@babel/highlight': 7.22.5 dev: true - /@babel/compat-data@7.22.6: - resolution: {integrity: sha512-29tfsWTq2Ftu7MXmimyC0C5FDZv5DYxOZkh3XD3+QW4V/BYuv/LyEsjj3c0hqedEaDt6DBfDvexMKU8YevdqFg==} + /@babel/code-frame@7.24.7: + resolution: {integrity: sha512-BcYH1CVJBO9tvyIZ2jVeXgSIMvGZ2FDRvDdOIVQyuklNKSsx+eppDEBq/g47Ayw+RqNFE+URvOShmf+f/qwAlA==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/highlight': 7.24.7 + picocolors: 1.0.1 + dev: true + + /@babel/compat-data@7.24.7: + resolution: {integrity: sha512-qJzAIcv03PyaWqxRgO4mSU3lihncDT296vnyuE2O8uA4w3UHWI4S3hgeZd1L8W1Bft40w9JxJ2b412iDUFFRhw==} engines: {node: '>=6.9.0'} dev: true - /@babel/core@7.22.8: - resolution: {integrity: sha512-75+KxFB4CZqYRXjx4NlR4J7yGvKumBuZTmV4NV6v09dVXXkuYVYLT68N6HCzLvfJ+fWCxQsntNzKwwIXL4bHnw==} + /@babel/core@7.24.7: + resolution: {integrity: sha512-nykK+LEK86ahTkX/3TgauT0ikKoNCfKHEaZYTUVupJdTLzGNvrblu4u6fa7DhZONAltdf8e662t/abY8idrd/g==} engines: {node: '>=6.9.0'} dependencies: - '@ampproject/remapping': 2.2.1 - '@babel/code-frame': 7.22.5 - '@babel/generator': 7.22.7 - '@babel/helper-compilation-targets': 7.22.6(@babel/core@7.22.8) - '@babel/helper-module-transforms': 7.22.5 - '@babel/helpers': 7.22.6 - '@babel/parser': 7.22.7 - '@babel/template': 7.22.5 - '@babel/traverse': 7.22.8 - '@babel/types': 7.22.5 - '@nicolo-ribaudo/semver-v6': 6.3.3 - convert-source-map: 1.9.0 - debug: 4.3.4(supports-color@8.1.1) + '@ampproject/remapping': 2.3.0 + '@babel/code-frame': 7.24.7 + '@babel/generator': 7.24.7 + '@babel/helper-compilation-targets': 7.24.7 + '@babel/helper-module-transforms': 7.24.7(@babel/core@7.24.7) + '@babel/helpers': 7.24.7 + '@babel/parser': 7.24.7 + '@babel/template': 7.24.7 + '@babel/traverse': 7.24.7 + '@babel/types': 7.24.7 + convert-source-map: 2.0.0 + debug: 4.3.5 gensync: 1.0.0-beta.2 json5: 2.2.3 + semver: 6.3.1 transitivePeerDependencies: - supports-color dev: true - /@babel/generator@7.22.7: - resolution: {integrity: sha512-p+jPjMG+SI8yvIaxGgeW24u7q9+5+TGpZh8/CuB7RhBKd7RCy8FayNEFNNKrNK/eUcY/4ExQqLmyrvBXKsIcwQ==} + /@babel/generator@7.24.7: + resolution: {integrity: sha512-oipXieGC3i45Y1A41t4tAqpnEZWgB/lC6Ehh6+rOviR5XWpTtMmLN+fGjz9vOiNRt0p6RtO6DtD0pdU3vpqdSA==} engines: {node: '>=6.9.0'} dependencies: - '@babel/types': 7.22.5 - '@jridgewell/gen-mapping': 0.3.3 - '@jridgewell/trace-mapping': 0.3.18 + '@babel/types': 7.24.7 + '@jridgewell/gen-mapping': 0.3.5 + '@jridgewell/trace-mapping': 0.3.25 jsesc: 2.5.2 dev: true - /@babel/helper-compilation-targets@7.22.6(@babel/core@7.22.8): - resolution: {integrity: sha512-534sYEqWD9VfUm3IPn2SLcH4Q3P86XL+QvqdC7ZsFrzyyPF3T4XGiVghF6PTYNdWg6pXuoqXxNQAhbYeEInTzA==} + /@babel/helper-compilation-targets@7.24.7: + resolution: {integrity: sha512-ctSdRHBi20qWOfy27RUb4Fhp07KSJ3sXcuSvTrXrc4aG8NSYDo1ici3Vhg9bg69y5bj0Mr1lh0aeEgTvc12rMg==} engines: {node: '>=6.9.0'} - peerDependencies: - '@babel/core': ^7.0.0 dependencies: - '@babel/compat-data': 7.22.6 - '@babel/core': 7.22.8 - '@babel/helper-validator-option': 7.22.5 - '@nicolo-ribaudo/semver-v6': 6.3.3 - browserslist: 4.21.9 + '@babel/compat-data': 7.24.7 + '@babel/helper-validator-option': 7.24.7 + browserslist: 4.23.1 lru-cache: 5.1.1 + semver: 6.3.1 dev: true - /@babel/helper-environment-visitor@7.22.5: - resolution: {integrity: sha512-XGmhECfVA/5sAt+H+xpSg0mfrHq6FzNr9Oxh7PSEBBRUb/mL7Kz3NICXb194rCqAEdxkhPT1a88teizAFyvk8Q==} + /@babel/helper-environment-visitor@7.24.7: + resolution: {integrity: sha512-DoiN84+4Gnd0ncbBOM9AZENV4a5ZiL39HYMyZJGZ/AZEykHYdJw0wW3kdcsh9/Kn+BRXHLkkklZ51ecPKmI1CQ==} engines: {node: '>=6.9.0'} + dependencies: + '@babel/types': 7.24.7 dev: true - /@babel/helper-function-name@7.22.5: - resolution: {integrity: sha512-wtHSq6jMRE3uF2otvfuD3DIvVhOsSNshQl0Qrd7qC9oQJzHvOL4qQXlQn2916+CXGywIjpGuIkoyZRRxHPiNQQ==} + /@babel/helper-function-name@7.24.7: + resolution: {integrity: sha512-FyoJTsj/PEUWu1/TYRiXTIHc8lbw+TDYkZuoE43opPS5TrI7MyONBE1oNvfguEXAD9yhQRrVBnXdXzSLQl9XnA==} engines: {node: '>=6.9.0'} dependencies: - '@babel/template': 7.22.5 - '@babel/types': 7.22.5 + '@babel/template': 7.24.7 + '@babel/types': 7.24.7 dev: true - /@babel/helper-hoist-variables@7.22.5: - resolution: {integrity: sha512-wGjk9QZVzvknA6yKIUURb8zY3grXCcOZt+/7Wcy8O2uctxhplmUPkOdlgoNhmdVee2c92JXbf1xpMtVNbfoxRw==} + /@babel/helper-hoist-variables@7.24.7: + resolution: {integrity: sha512-MJJwhkoGy5c4ehfoRyrJ/owKeMl19U54h27YYftT0o2teQ3FJ3nQUf/I3LlJsX4l3qlw7WRXUmiyajvHXoTubQ==} engines: {node: '>=6.9.0'} dependencies: - '@babel/types': 7.22.5 + '@babel/types': 7.24.7 dev: true - /@babel/helper-module-imports@7.22.5: - resolution: {integrity: sha512-8Dl6+HD/cKifutF5qGd/8ZJi84QeAKh+CEe1sBzz8UayBBGg1dAIJrdHOcOM5b2MpzWL2yuotJTtGjETq0qjXg==} + /@babel/helper-module-imports@7.24.7: + resolution: {integrity: sha512-8AyH3C+74cgCVVXow/myrynrAGv+nTVg5vKu2nZph9x7RcRwzmh0VFallJuFTZ9mx6u4eSdXZfcOzSqTUm0HCA==} engines: {node: '>=6.9.0'} dependencies: - '@babel/types': 7.22.5 + '@babel/traverse': 7.24.7 + '@babel/types': 7.24.7 + transitivePeerDependencies: + - supports-color dev: true - /@babel/helper-module-transforms@7.22.5: - resolution: {integrity: sha512-+hGKDt/Ze8GFExiVHno/2dvG5IdstpzCq0y4Qc9OJ25D4q3pKfiIP/4Vp3/JvhDkLKsDK2api3q3fpIgiIF5bw==} + /@babel/helper-module-transforms@7.24.7(@babel/core@7.24.7): + resolution: {integrity: sha512-1fuJEwIrp+97rM4RWdO+qrRsZlAeL1lQJoPqtCYWv0NL115XM93hIH4CSRln2w52SqvmY5hqdtauB6QFCDiZNQ==} engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 dependencies: - '@babel/helper-environment-visitor': 7.22.5 - '@babel/helper-module-imports': 7.22.5 - '@babel/helper-simple-access': 7.22.5 - '@babel/helper-split-export-declaration': 7.22.6 - '@babel/helper-validator-identifier': 7.22.5 - '@babel/template': 7.22.5 - '@babel/traverse': 7.22.8 - '@babel/types': 7.22.5 + '@babel/core': 7.24.7 + '@babel/helper-environment-visitor': 7.24.7 + '@babel/helper-module-imports': 7.24.7 + '@babel/helper-simple-access': 7.24.7 + '@babel/helper-split-export-declaration': 7.24.7 + '@babel/helper-validator-identifier': 7.24.7 transitivePeerDependencies: - supports-color dev: true - /@babel/helper-simple-access@7.22.5: - resolution: {integrity: sha512-n0H99E/K+Bika3++WNL17POvo4rKWZ7lZEp1Q+fStVbUi8nxPQEBOlTmCOxW/0JsS56SKKQ+ojAe2pHKJHN35w==} + /@babel/helper-simple-access@7.24.7: + resolution: {integrity: sha512-zBAIvbCMh5Ts+b86r/CjU+4XGYIs+R1j951gxI3KmmxBMhCg4oQMsv6ZXQ64XOm/cvzfU1FmoCyt6+owc5QMYg==} engines: {node: '>=6.9.0'} dependencies: - '@babel/types': 7.22.5 + '@babel/traverse': 7.24.7 + '@babel/types': 7.24.7 + transitivePeerDependencies: + - supports-color dev: true - /@babel/helper-split-export-declaration@7.22.6: - resolution: {integrity: sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g==} + /@babel/helper-split-export-declaration@7.24.7: + resolution: {integrity: sha512-oy5V7pD+UvfkEATUKvIjvIAH/xCzfsFVw7ygW2SI6NClZzquT+mwdTfgfdbUiceh6iQO0CHtCPsyze/MZ2YbAA==} engines: {node: '>=6.9.0'} dependencies: - '@babel/types': 7.22.5 + '@babel/types': 7.24.7 dev: true - /@babel/helper-string-parser@7.22.5: - resolution: {integrity: sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw==} + /@babel/helper-string-parser@7.24.7: + resolution: {integrity: sha512-7MbVt6xrwFQbunH2DNQsAP5sTGxfqQtErvBIvIMi6EQnbgUOuVYanvREcmFrOPhoXBrTtjhhP+lW+o5UfK+tDg==} engines: {node: '>=6.9.0'} dev: true @@ -417,20 +429,22 @@ packages: engines: {node: '>=6.9.0'} dev: true - /@babel/helper-validator-option@7.22.5: - resolution: {integrity: sha512-R3oB6xlIVKUnxNUxbmgq7pKjxpru24zlimpE8WK47fACIlM0II/Hm1RS8IaOI7NgCr6LNS+jl5l75m20npAziw==} + /@babel/helper-validator-identifier@7.24.7: + resolution: {integrity: sha512-rR+PBcQ1SMQDDyF6X0wxtG8QyLCgUB0eRAGguqRLfkCA87l7yAP7ehq8SNj96OOGTO8OBV70KhuFYcIkHXOg0w==} + engines: {node: '>=6.9.0'} + dev: true + + /@babel/helper-validator-option@7.24.7: + resolution: {integrity: sha512-yy1/KvjhV/ZCL+SM7hBrvnZJ3ZuT9OuZgIJAGpPEToANvc3iM6iDvBnRjtElWibHU6n8/LPR/EjX9EtIEYO3pw==} engines: {node: '>=6.9.0'} dev: true - /@babel/helpers@7.22.6: - resolution: {integrity: sha512-YjDs6y/fVOYFV8hAf1rxd1QvR9wJe1pDBZ2AREKq/SDayfPzgk0PBnVuTCE5X1acEpMMNOVUqoe+OwiZGJ+OaA==} + /@babel/helpers@7.24.7: + resolution: {integrity: sha512-NlmJJtvcw72yRJRcnCmGvSi+3jDEg8qFu3z0AFoymmzLx5ERVWyzd9kVXr7Th9/8yIJi2Zc6av4Tqz3wFs8QWg==} engines: {node: '>=6.9.0'} dependencies: - '@babel/template': 7.22.5 - '@babel/traverse': 7.22.8 - '@babel/types': 7.22.5 - transitivePeerDependencies: - - supports-color + '@babel/template': 7.24.7 + '@babel/types': 7.24.7 dev: true /@babel/highlight@7.22.5: @@ -442,12 +456,22 @@ packages: js-tokens: 4.0.0 dev: true - /@babel/parser@7.22.7: - resolution: {integrity: sha512-7NF8pOkHP5o2vpmGgNGcfAeCvOYhGLyA3Z4eBQkT1RJlWu47n63bCs93QfJ2hIAFCil7L5P2IWhs1oToVgrL0Q==} + /@babel/highlight@7.24.7: + resolution: {integrity: sha512-EStJpq4OuY8xYfhGVXngigBJRWxftKX9ksiGDnmlY3o7B/V7KIAc9X4oiK87uPJSc/vs5L869bem5fhZa8caZw==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/helper-validator-identifier': 7.24.7 + chalk: 2.4.2 + js-tokens: 4.0.0 + picocolors: 1.0.1 + dev: true + + /@babel/parser@7.24.7: + resolution: {integrity: sha512-9uUYRm6OqQrCqQdG1iCBwBPZgN8ciDBro2nIOFaiRz1/BCxaI7CNvQbDHvsArAC7Tw9Hda/B3U+6ui9u4HWXPw==} engines: {node: '>=6.0.0'} hasBin: true dependencies: - '@babel/types': 7.22.5 + '@babel/types': 7.24.7 dev: true /@babel/runtime@7.22.6: @@ -457,39 +481,39 @@ packages: regenerator-runtime: 0.13.11 dev: true - /@babel/template@7.22.5: - resolution: {integrity: sha512-X7yV7eiwAxdj9k94NEylvbVHLiVG1nvzCV2EAowhxLTwODV1jl9UzZ48leOC0sH7OnuHrIkllaBgneUykIcZaw==} + /@babel/template@7.24.7: + resolution: {integrity: sha512-jYqfPrU9JTF0PmPy1tLYHW4Mp4KlgxJD9l2nP9fD6yT/ICi554DmrWBAEYpIelzjHf1msDP3PxJIRt/nFNfBig==} engines: {node: '>=6.9.0'} dependencies: - '@babel/code-frame': 7.22.5 - '@babel/parser': 7.22.7 - '@babel/types': 7.22.5 + '@babel/code-frame': 7.24.7 + '@babel/parser': 7.24.7 + '@babel/types': 7.24.7 dev: true - /@babel/traverse@7.22.8: - resolution: {integrity: sha512-y6LPR+wpM2I3qJrsheCTwhIinzkETbplIgPBbwvqPKc+uljeA5gP+3nP8irdYt1mjQaDnlIcG+dw8OjAco4GXw==} + /@babel/traverse@7.24.7: + resolution: {integrity: sha512-yb65Ed5S/QAcewNPh0nZczy9JdYXkkAbIsEo+P7BE7yO3txAY30Y/oPa3QkQ5It3xVG2kpKMg9MsdxZaO31uKA==} engines: {node: '>=6.9.0'} dependencies: - '@babel/code-frame': 7.22.5 - '@babel/generator': 7.22.7 - '@babel/helper-environment-visitor': 7.22.5 - '@babel/helper-function-name': 7.22.5 - '@babel/helper-hoist-variables': 7.22.5 - '@babel/helper-split-export-declaration': 7.22.6 - '@babel/parser': 7.22.7 - '@babel/types': 7.22.5 - debug: 4.3.4(supports-color@8.1.1) + '@babel/code-frame': 7.24.7 + '@babel/generator': 7.24.7 + '@babel/helper-environment-visitor': 7.24.7 + '@babel/helper-function-name': 7.24.7 + '@babel/helper-hoist-variables': 7.24.7 + '@babel/helper-split-export-declaration': 7.24.7 + '@babel/parser': 7.24.7 + '@babel/types': 7.24.7 + debug: 4.3.5 globals: 11.12.0 transitivePeerDependencies: - supports-color dev: true - /@babel/types@7.22.5: - resolution: {integrity: sha512-zo3MIHGOkPOfoRXitsgHLjEXmlDaD/5KU1Uzuc9GNiZPhSqVxVRtxuPaSBZDsYZ9qV88AjtMtWW7ww98loJ9KA==} + /@babel/types@7.24.7: + resolution: {integrity: sha512-XEFXSlxiG5td2EJRe8vOmRbaXVgfcBlszKujvVmWIK/UpywWljQCfzAv3RQCGujWQ1RD4YYWEAqDXfuJiy8f5Q==} engines: {node: '>=6.9.0'} dependencies: - '@babel/helper-string-parser': 7.22.5 - '@babel/helper-validator-identifier': 7.22.5 + '@babel/helper-string-parser': 7.24.7 + '@babel/helper-validator-identifier': 7.24.7 to-fast-properties: 2.0.0 dev: true @@ -498,12 +522,17 @@ packages: engines: {node: '>=0.1.90'} dev: false + /@colors/colors@1.6.0: + resolution: {integrity: sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==} + engines: {node: '>=0.1.90'} + dev: false + /@cspotcode/source-map-support@0.8.1: resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} engines: {node: '>=12'} dependencies: '@jridgewell/trace-mapping': 0.3.9 - dev: false + dev: true /@dabh/diagnostics@2.0.3: resolution: {integrity: sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==} @@ -513,6 +542,18 @@ packages: kuler: 2.0.0 dev: false + /@isaacs/cliui@8.0.2: + resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} + engines: {node: '>=12'} + dependencies: + string-width: 5.1.2 + string-width-cjs: /string-width@4.2.3 + strip-ansi: 7.1.0 + strip-ansi-cjs: /strip-ansi@6.0.1 + wrap-ansi: 8.1.0 + wrap-ansi-cjs: /wrap-ansi@7.0.0 + dev: true + /@istanbuljs/load-nyc-config@1.1.0: resolution: {integrity: sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ==} engines: {node: '>=8'} @@ -529,90 +570,64 @@ packages: engines: {node: '>=8'} dev: true - /@jridgewell/gen-mapping@0.3.3: - resolution: {integrity: sha512-HLhSWOLRi875zjjMG/r+Nv0oCW8umGb0BgEhyX3dDX3egwZtB8PqLnjz3yedt8R5StBrzcg4aBpnh8UA9D1BoQ==} - engines: {node: '>=6.0.0'} + /@jest/schemas@29.6.3: + resolution: {integrity: sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} dependencies: - '@jridgewell/set-array': 1.1.2 - '@jridgewell/sourcemap-codec': 1.4.15 - '@jridgewell/trace-mapping': 0.3.18 + '@sinclair/typebox': 0.27.8 dev: true - /@jridgewell/resolve-uri@3.1.0: - resolution: {integrity: sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==} + /@jridgewell/gen-mapping@0.3.5: + resolution: {integrity: sha512-IzL8ZoEDIBRWEzlCcRhOaCupYyN5gdIK+Q6fbFdPDg6HqX6jpkItn7DFIpW9LQzXG6Df9sA7+OKnq0qlz/GaQg==} engines: {node: '>=6.0.0'} + dependencies: + '@jridgewell/set-array': 1.2.1 + '@jridgewell/sourcemap-codec': 1.4.15 + '@jridgewell/trace-mapping': 0.3.25 dev: true - /@jridgewell/resolve-uri@3.1.1: - resolution: {integrity: sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA==} - engines: {node: '>=6.0.0'} - dev: false - - /@jridgewell/set-array@1.1.2: - resolution: {integrity: sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==} + /@jridgewell/resolve-uri@3.1.2: + resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==} engines: {node: '>=6.0.0'} dev: true - /@jridgewell/sourcemap-codec@1.4.14: - resolution: {integrity: sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==} + /@jridgewell/set-array@1.2.1: + resolution: {integrity: sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A==} + engines: {node: '>=6.0.0'} dev: true /@jridgewell/sourcemap-codec@1.4.15: resolution: {integrity: sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==} + dev: true - /@jridgewell/trace-mapping@0.3.18: - resolution: {integrity: sha512-w+niJYzMHdd7USdiH2U6869nqhD2nbfZXND5Yp93qIbEmnDNk7PD48o+YchRVpzMU7M6jVCbenTR7PA1FLQ9pA==} + /@jridgewell/trace-mapping@0.3.25: + resolution: {integrity: sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==} dependencies: - '@jridgewell/resolve-uri': 3.1.0 - '@jridgewell/sourcemap-codec': 1.4.14 + '@jridgewell/resolve-uri': 3.1.2 + '@jridgewell/sourcemap-codec': 1.4.15 dev: true /@jridgewell/trace-mapping@0.3.9: resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==} dependencies: - '@jridgewell/resolve-uri': 3.1.1 + '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.4.15 - dev: false - - /@nicolo-ribaudo/semver-v6@6.3.3: - resolution: {integrity: sha512-3Yc1fUTs69MG/uZbJlLSI3JISMn2UV2rg+1D/vROUqZyh3l6iYHCs7GMp+M40ZD7yOdDbYjJcU1oTJhrc+dGKg==} - hasBin: true - dev: true - - /@nodelib/fs.scandir@2.1.5: - resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} - engines: {node: '>= 8'} - dependencies: - '@nodelib/fs.stat': 2.0.5 - run-parallel: 1.2.0 - dev: true - - /@nodelib/fs.stat@2.0.5: - resolution: {integrity: sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==} - engines: {node: '>= 8'} dev: true - /@nodelib/fs.walk@1.2.8: - resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==} - engines: {node: '>= 8'} - dependencies: - '@nodelib/fs.scandir': 2.1.5 - fastq: 1.15.0 - dev: true - - /@nrwl/tao@16.5.0: - resolution: {integrity: sha512-lY/XV2n7iulHY77Uakt3Epa9m/NG7oTSN196baLjFykxUvLJI47PMX5qytugHkS8JLdcAB5p0qGsrQSHoi6jvg==} + /@nrwl/tao@16.10.0: + resolution: {integrity: sha512-QNAanpINbr+Pod6e1xNgFbzK1x5wmZl+jMocgiEFXZ67KHvmbD6MAQQr0MMz+GPhIu7EE4QCTLTyCEMlAG+K5Q==} hasBin: true dependencies: - nx: 16.5.0 + nx: 16.10.0 + tslib: 2.6.3 transitivePeerDependencies: - '@swc-node/register' - '@swc/core' - debug dev: true - /@nx/nx-darwin-arm64@16.5.0: - resolution: {integrity: sha512-0+5FH3ot5o0lpL0OKD4fO2n0a6LqLxr0LwU2VYxaAR1GLzOeVE5W3jBWY9ztOE+ktm8mGaZsdIIOQ77Iz/xwsQ==} + /@nx/nx-darwin-arm64@16.10.0: + resolution: {integrity: sha512-YF+MIpeuwFkyvM5OwgY/rTNRpgVAI/YiR0yTYCZR+X3AAvP775IVlusNgQ3oedTBRUzyRnI4Tknj1WniENFsvQ==} engines: {node: '>= 10'} cpu: [arm64] os: [darwin] @@ -620,8 +635,8 @@ packages: dev: true optional: true - /@nx/nx-darwin-x64@16.5.0: - resolution: {integrity: sha512-yziX2oXUSyOPOcRLmFMRsNs0eBVla5IGjAKqpY4OXAPBuyrOfgsW5ztj0PQM34gvqipXtTlN04Xt/U0jzQLudA==} + /@nx/nx-darwin-x64@16.10.0: + resolution: {integrity: sha512-ypi6YxwXgb0kg2ixKXE3pwf5myVNUgWf1CsV5OzVccCM8NzheMO51KDXTDmEpXdzUsfT0AkO1sk5GZeCjhVONg==} engines: {node: '>= 10'} cpu: [x64] os: [darwin] @@ -629,8 +644,8 @@ packages: dev: true optional: true - /@nx/nx-freebsd-x64@16.5.0: - resolution: {integrity: sha512-hwIRRMyWrT2R4ozp6yXRNR1fwcclBlkkIQ51/1IzINPQxynMguuOvNZaJFD4OuZIDmI526++GmogPZc0aMzwkg==} + /@nx/nx-freebsd-x64@16.10.0: + resolution: {integrity: sha512-UeEYFDmdbbDkTQamqvtU8ibgu5jQLgFF1ruNb/U4Ywvwutw2d4ruOMl2e0u9hiNja9NFFAnDbvzrDcMo7jYqYw==} engines: {node: '>= 10'} cpu: [x64] os: [freebsd] @@ -638,8 +653,8 @@ packages: dev: true optional: true - /@nx/nx-linux-arm-gnueabihf@16.5.0: - resolution: {integrity: sha512-BEWLpBhJ2AcZNDsiExLDcM9kmQ4+E+0YUcOsrAeX1s5D4HXXVtHMdTmOucKs4NNFqMuJ2Cf3ZzqmAIkRug0beA==} + /@nx/nx-linux-arm-gnueabihf@16.10.0: + resolution: {integrity: sha512-WV3XUC2DB6/+bz1sx+d1Ai9q2Cdr+kTZRN50SOkfmZUQyEBaF6DRYpx/a4ahhxH3ktpNfyY8Maa9OEYxGCBkQA==} engines: {node: '>= 10'} cpu: [arm] os: [linux] @@ -647,8 +662,8 @@ packages: dev: true optional: true - /@nx/nx-linux-arm64-gnu@16.5.0: - resolution: {integrity: sha512-EWmTbDLbBIjM/OJ594hoFKsEka/b8jM6NehL37mlIXL6fixUEA8LlO0MfUQ+kIPg79nWIujzulkIEhYFDWM1WA==} + /@nx/nx-linux-arm64-gnu@16.10.0: + resolution: {integrity: sha512-aWIkOUw995V3ItfpAi5FuxQ+1e9EWLS1cjWM1jmeuo+5WtaKToJn5itgQOkvSlPz+HSLgM3VfXMvOFALNk125g==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] @@ -656,8 +671,8 @@ packages: dev: true optional: true - /@nx/nx-linux-arm64-musl@16.5.0: - resolution: {integrity: sha512-np/7+HEtEEvtu4zo3GBBPtTG8IP++vvH3o8VXpAB9eD4Jctz3rYzbfMc7GtLZkz8LCmCsjzqnrNtmcmoaRbomQ==} + /@nx/nx-linux-arm64-musl@16.10.0: + resolution: {integrity: sha512-uO6Gg+irqpVcCKMcEPIQcTFZ+tDI02AZkqkP7koQAjniLEappd8DnUBSQdcn53T086pHpdc264X/ZEpXFfrKWQ==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] @@ -665,8 +680,8 @@ packages: dev: true optional: true - /@nx/nx-linux-x64-gnu@16.5.0: - resolution: {integrity: sha512-iLOwgAaa1BHPLFhkBVi7GLAf6LfdYuv/R2rxlqq4d6fhv4Eq91Wo08LsqbFds+LpMN0CA+W/QMc3w9IIS/MPrA==} + /@nx/nx-linux-x64-gnu@16.10.0: + resolution: {integrity: sha512-134PW/u/arNFAQKpqMJniC7irbChMPz+W+qtyKPAUXE0XFKPa7c1GtlI/wK2dvP9qJDZ6bKf0KtA0U/m2HMUOA==} engines: {node: '>= 10'} cpu: [x64] os: [linux] @@ -674,8 +689,8 @@ packages: dev: true optional: true - /@nx/nx-linux-x64-musl@16.5.0: - resolution: {integrity: sha512-UE3tpgli7a08AsRaw/o1BUXnFOxICGzcYj1aglHBh6urVeUHK0aNt11djZcQ6ETgPgcjoGdwr7RqpANGnJQH9g==} + /@nx/nx-linux-x64-musl@16.10.0: + resolution: {integrity: sha512-q8sINYLdIJxK/iUx9vRk5jWAWb/2O0PAbOJFwv4qkxBv4rLoN7y+otgCZ5v0xfx/zztFgk/oNY4lg5xYjIso2Q==} engines: {node: '>= 10'} cpu: [x64] os: [linux] @@ -683,8 +698,8 @@ packages: dev: true optional: true - /@nx/nx-win32-arm64-msvc@16.5.0: - resolution: {integrity: sha512-u9cNKP8zrNIdeyaK5LHX+Zh+rkadE8tSE+vNulphCLhGuXJRpjaVY1juq9UQEo41NJQE6DuWWk2fnj4gALWugQ==} + /@nx/nx-win32-arm64-msvc@16.10.0: + resolution: {integrity: sha512-moJkL9kcqxUdJSRpG7dET3UeLIciwrfP08mzBQ12ewo8K8FzxU8ZUsTIVVdNrwt01CXOdXoweGfdQLjJ4qTURA==} engines: {node: '>= 10'} cpu: [arm64] os: [win32] @@ -692,8 +707,8 @@ packages: dev: true optional: true - /@nx/nx-win32-x64-msvc@16.5.0: - resolution: {integrity: sha512-E9109SAYNZXqCeWikZXyxNd7SZnCbdKGvqtQktS7dedHGwOmgIWfJ6bsvA7s2zHr09THQKJ4+7U1tDkWVNR9cg==} + /@nx/nx-win32-x64-msvc@16.10.0: + resolution: {integrity: sha512-5iV2NKZnzxJwZZ4DM5JVbRG/nkhAbzEskKaLBB82PmYGKzaDHuMHP1lcPoD/rtYMlowZgNA/RQndfKvPBPwmXA==} engines: {node: '>= 10'} cpu: [x64] os: [win32] @@ -707,7 +722,18 @@ packages: requiresBuild: true dependencies: node-addon-api: 3.2.1 - node-gyp-build: 4.6.0 + node-gyp-build: 4.8.1 + dev: true + + /@pkgjs/parseargs@0.11.0: + resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} + engines: {node: '>=14'} + requiresBuild: true + dev: true + optional: true + + /@sinclair/typebox@0.27.8: + resolution: {integrity: sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==} dev: true /@sinonjs/commons@2.0.0: @@ -716,8 +742,8 @@ packages: type-detect: 4.0.8 dev: true - /@sinonjs/commons@3.0.0: - resolution: {integrity: sha512-jXBtWAF4vmdNmZgD5FoKsVLv3rPgDnLgPbU84LIJ3otV44vJlDRokVng5v8NFJdCf/da9legHcKaRuZs4L7faA==} + /@sinonjs/commons@3.0.1: + resolution: {integrity: sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ==} dependencies: type-detect: 4.0.8 dev: true @@ -725,7 +751,13 @@ packages: /@sinonjs/fake-timers@10.3.0: resolution: {integrity: sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA==} dependencies: - '@sinonjs/commons': 3.0.0 + '@sinonjs/commons': 3.0.1 + dev: true + + /@sinonjs/fake-timers@11.2.2: + resolution: {integrity: sha512-G2piCSxQ7oWOxwGSAyFHfPIsyeJGXYtc6mFbnFA+kRXkiEnTl8c/8jul2S329iFBnDI9HGoeWWAZvuvOkZccgw==} + dependencies: + '@sinonjs/commons': 3.0.1 dev: true /@sinonjs/samsam@8.0.0: @@ -740,49 +772,53 @@ packages: resolution: {integrity: sha512-sXXKG+uL9IrKqViTtao2Ws6dy0znu9sOaP1di/jKGW1M6VssO8vlpXCQcpZ+jisQ1tTFAC5Jo/EOzFbggBagFQ==} dev: true - /@tsconfig/node10@1.0.9: - resolution: {integrity: sha512-jNsYVVxU8v5g43Erja32laIDHXeoNvFEpX33OK4d6hljo3jDhCBDhx5dhCCTMWUojscpAagGiRkBKxpdl9fxqA==} - dev: false + /@tsconfig/node10@1.0.11: + resolution: {integrity: sha512-DcRjDCujK/kCk/cUe8Xz8ZSpm8mS3mNNpta+jGCA6USEDfktlNvm1+IuZ9eTcDbNk41BHwpHHeW+N1lKCz4zOw==} + dev: true /@tsconfig/node12@1.0.11: resolution: {integrity: sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==} - dev: false + dev: true /@tsconfig/node14@1.0.3: resolution: {integrity: sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==} - dev: false + dev: true /@tsconfig/node16@1.0.4: resolution: {integrity: sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==} - dev: false + dev: true /@types/body-parser@1.19.2: resolution: {integrity: sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g==} dependencies: '@types/connect': 3.4.35 - '@types/node': 20.4.1 + '@types/node': 20.14.7 dev: true - /@types/chai-as-promised@7.1.5: - resolution: {integrity: sha512-jStwss93SITGBwt/niYrkf2C+/1KTeZCZl1LaeezTlqppAKeoQC7jxyqYuP72sxBGKCIbw7oHgbYssIRzT5FCQ==} + /@types/chai-as-promised@7.1.8: + resolution: {integrity: sha512-ThlRVIJhr69FLlh6IctTXFkmhtP3NpMZ2QGq69StYLyKZFp/HOp1VdKZj7RvfNWYYcJ1xlbLGLLWj1UvP5u/Gw==} dependencies: - '@types/chai': 4.3.5 + '@types/chai': 4.3.16 dev: true - /@types/chai@4.3.5: - resolution: {integrity: sha512-mEo1sAde+UCE6b2hxn332f1g1E8WfYRu6p5SvTKr2ZKC1f7gFJXk4h5PyGP9Dt6gCaG8y8XhwnXWC6Iy2cmBng==} + /@types/chai@4.3.16: + resolution: {integrity: sha512-PatH4iOdyh3MyWtmHVFXLWCCIhUbopaltqddG9BzB+gMIzee2MJrvd+jouii9Z3wzQJruGWAm7WOMjgfG8hQlQ==} dev: true /@types/connect@3.4.35: resolution: {integrity: sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==} dependencies: - '@types/node': 20.4.1 + '@types/node': 20.14.7 dev: true /@types/cookiejar@2.1.2: resolution: {integrity: sha512-t73xJJrvdTjXrn4jLS9VSGRbz0nUY3cl2DMGDU48lKl+HR9dbbjW2A9r3g40VA++mQpy6uuHg33gy7du2BKpog==} dev: true + /@types/cookiejar@2.1.5: + resolution: {integrity: sha512-he+DHOWReW0nghN24E1WUqM0efK4kI9oTqDm6XmK8ZPe2djZ90BSNdGnIyCLzCPw7/pogPlGbzI2wHGGmi4O/Q==} + dev: true + /@types/errorhandler@0.0.32: resolution: {integrity: sha512-wC9CfwPMIzklPd5lEYC8HnQdlMC1PswlohWmEDMWlw+E/rMYuz5eSqKBc72Earb29KptKJrRl77qVRJzrZndww==} dependencies: @@ -792,7 +828,7 @@ packages: /@types/express-serve-static-core@4.17.35: resolution: {integrity: sha512-wALWQwrgiB2AWTT91CB62b6Yt0sNHpznUXeZEcnPU3DRdlDIz74x8Qg1UUYKSVFi+va5vKOLYRBI1bRKiLLKIg==} dependencies: - '@types/node': 20.4.1 + '@types/node': 20.14.7 '@types/qs': 6.9.7 '@types/range-parser': 1.2.4 '@types/send': 0.17.1 @@ -814,13 +850,17 @@ packages: /@types/jsonwebtoken@8.5.9: resolution: {integrity: sha512-272FMnFGzAVMGtu9tkr29hRL6bZj4Zs1KZNeHLnKqAvp06tAIcarTMwOh8/8bz4FmKRcMxZhZNeUAQsNLoiPhg==} dependencies: - '@types/node': 20.4.1 + '@types/node': 20.14.7 dev: true /@types/lodash@4.14.195: resolution: {integrity: sha512-Hwx9EUgdwf2GLarOjQp5ZH8ZmblzcbTBC2wtQWNKARBSxM9ezRIAUpeDTgoQRAFB0+8CNWXVA9+MaSOzOF3nPg==} dev: true + /@types/lodash@4.17.5: + resolution: {integrity: sha512-MBIOHVZqVqgfro1euRDWX7OO0fBVUUMrN6Pwm8LQsz8cWhEpihlvR70ENj3f40j58TNxZaWv2ndSkInykNBBJw==} + dev: true + /@types/memoizee@0.4.8: resolution: {integrity: sha512-qDpXKGgwKywnQt/64fH1O0LiPA++QGIYeykEUiZ51HymKVRLnUSGcRuF60IfpPeeXiuRwiR/W4y7S5VzbrgLCA==} dev: true @@ -833,8 +873,8 @@ packages: resolution: {integrity: sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA==} dev: true - /@types/mocha@10.0.0: - resolution: {integrity: sha512-rADY+HtTOA52l9VZWtgQfn4p+UDVM2eDVkMZT1I6syp0YKxW2F9v+0pbRZLsvskhQv/vMb6ZfCay81GHbz5SHg==} + /@types/mocha@10.0.6: + resolution: {integrity: sha512-dJvrYWxP/UcXm36Qn36fxhUKu8A/xMRXVT2cliFF1Z7UA9liG5Psj3ezNSZw+5puH2czDXRLcXQxf8JbJt0ejg==} dev: true /@types/mocha@8.2.3: @@ -856,8 +896,10 @@ packages: resolution: {integrity: sha512-J8xLz7q2OFulZ2cyGTLE1TbbZcjpno7FaN6zdJNrgAdrJ+DZzh/uFR6YrTb4C+nXakvud8Q4+rbhoIWlYQbUFQ==} dev: true - /@types/node@20.4.1: - resolution: {integrity: sha512-JIzsAvJeA/5iY6Y/OxZbv1lUcc8dNSE77lb2gnBH+/PJ3lFR1Ccvgwl5JWnHAkNHcRsT0TbpVOsiMKZ1F/yyJg==} + /@types/node@20.14.7: + resolution: {integrity: sha512-uTr2m2IbJJucF3KUxgnGOZvYbN0QgkGyWxG6973HCpMYFy2KfcgYuIwkJQMQkt1VbBMlvWRbpshFTLxnxCZjKQ==} + dependencies: + undici-types: 5.26.5 /@types/normalize-package-data@2.4.1: resolution: {integrity: sha512-Gj7cI7z+98M282Tqmp2K5EIsoouUEzbBJhQQzDE3jSIRk6r9gsz0oUokqIUR4u1R3dMHo0pDHM7sNOHyhulypw==} @@ -875,7 +917,7 @@ packages: resolution: {integrity: sha512-Cwo8LE/0rnvX7kIIa3QHCkcuF21c05Ayb0ZfxPiv0W8VRiZiNW/WuRupHKpqqGVGf7SUA44QSOUKaEd9lIrd/Q==} dependencies: '@types/mime': 1.3.2 - '@types/node': 20.4.1 + '@types/node': 20.14.7 dev: true /@types/serve-static@1.15.2: @@ -883,31 +925,31 @@ packages: dependencies: '@types/http-errors': 2.0.1 '@types/mime': 3.0.1 - '@types/node': 20.4.1 + '@types/node': 20.14.7 dev: true - /@types/sinon@10.0.15: - resolution: {integrity: sha512-3lrFNQG0Kr2LDzvjyjB6AMJk4ge+8iYhQfdnSwIwlG88FUOV43kPcQqDZkDa/h3WSZy6i8Fr0BSjfQtB1B3xuQ==} + /@types/sinon@10.0.20: + resolution: {integrity: sha512-2APKKruFNCAZgx3daAyACGzWuJ028VVCUDk6o2rw/Z4PXT0ogwdV4KUegW0MwVs0Zu59auPXbbuBJHF12Sx1Eg==} dependencies: - '@types/sinonjs__fake-timers': 8.1.2 + '@types/sinonjs__fake-timers': 8.1.5 dev: true - /@types/sinonjs__fake-timers@8.1.2: - resolution: {integrity: sha512-9GcLXF0/v3t80caGs5p2rRfkB+a8VBGLJZVih6CNFkx8IZ994wiKKLSRs9nuFwk1HevWs/1mnUmkApGrSGsShA==} + /@types/sinonjs__fake-timers@8.1.5: + resolution: {integrity: sha512-mQkU2jY8jJEF7YHjHvsQO8+3ughTL1mcnn96igfhONmR+fUPSKIkefQYpSe8bsly2Ep7oQbn/6VG5/9/0qcArQ==} dev: true /@types/superagent@4.1.13: resolution: {integrity: sha512-YIGelp3ZyMiH0/A09PMAORO0EBGlF5xIKfDpK74wdYvWUs2o96b5CItJcWPdH409b7SAXIIG6p8NdU/4U2Maww==} dependencies: - '@types/cookiejar': 2.1.2 - '@types/node': 20.4.1 + '@types/cookiejar': 2.1.5 + '@types/node': 20.14.7 dev: true /@types/superagent@4.1.18: resolution: {integrity: sha512-LOWgpacIV8GHhrsQU+QMZuomfqXiqzz3ILLkCtKx3Us6AmomFViuzKT9D693QTKgyut2oCytMG8/efOop+DB+w==} dependencies: '@types/cookiejar': 2.1.2 - '@types/node': 20.4.1 + '@types/node': 20.14.7 dev: true /@types/swagger-ui-express@3.0.1: @@ -921,6 +963,10 @@ packages: resolution: {integrity: sha512-txGIh+0eDFzKGC25zORnswy+br1Ha7hj5cMVwKIU7+s0U2AxxJru/jZSMU6OC9MJWP6+pc/hc6ZjyZShpsyY2g==} dev: false + /@types/triple-beam@1.3.5: + resolution: {integrity: sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==} + dev: false + /@types/webidl-conversions@7.0.0: resolution: {integrity: sha512-xTE1E+YF4aWPJJeUzaZI5DRntlkY3+BCVJi0axFptnjGmAoWxkyREIh/XMrfxVLejwQxMCfDXdICo0VLxThrog==} dev: false @@ -928,7 +974,7 @@ packages: /@types/whatwg-url@8.2.2: resolution: {integrity: sha512-FtQu10RWgn3D9U4aazdwIE2yzphmTJREDqNdODHrbrZmmMqI0vMheC/6NE/J1Yveaj8H+ela+YwWTjq5PGmuhA==} dependencies: - '@types/node': 20.4.1 + '@types/node': 20.14.7 '@types/webidl-conversions': 7.0.0 dev: false @@ -945,7 +991,7 @@ packages: engines: {node: '>=14.15.0'} dependencies: js-yaml: 3.14.1 - tslib: 2.6.0 + tslib: 2.6.3 dev: true /@zkochan/js-yaml@0.0.6: @@ -975,16 +1021,18 @@ packages: negotiator: 0.6.3 dev: false - /acorn-walk@8.2.0: - resolution: {integrity: sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA==} + /acorn-walk@8.3.3: + resolution: {integrity: sha512-MxXdReSRhGO7VlFe1bRG/oI7/mdLV9B9JJT0N8vZOhF7gFRR5l3M8W9G8JxmKV+JC5mGqJ0QvqfSOLsCPa4nUw==} engines: {node: '>=0.4.0'} - dev: false + dependencies: + acorn: 8.12.0 + dev: true - /acorn@8.10.0: - resolution: {integrity: sha512-F0SAmZ8iUtS//m8DmCTA0jlh6TDKkHQyK6xc6V4KDTyZKA9dnvX9/3sRTVQrWm79glUAZbnmmNcdYwUIHWVybw==} + /acorn@8.12.0: + resolution: {integrity: sha512-RTvkC4w+KNXrM39/lWCUaG0IbRkWdCv7W/IOW9oU6SawyxulvkQy5HQPVTKxEjczcUvapcrw3cFx/60VN/NRNw==} engines: {node: '>=0.4.0'} hasBin: true - dev: false + dev: true /aggregate-error@3.1.0: resolution: {integrity: sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==} @@ -1037,6 +1085,11 @@ packages: engines: {node: '>=8'} dev: true + /ansi-regex@6.0.1: + resolution: {integrity: sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==} + engines: {node: '>=12'} + dev: true + /ansi-styles@3.2.1: resolution: {integrity: sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==} engines: {node: '>=4'} @@ -1051,6 +1104,16 @@ packages: color-convert: 2.0.1 dev: true + /ansi-styles@5.2.0: + resolution: {integrity: sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==} + engines: {node: '>=10'} + dev: true + + /ansi-styles@6.2.1: + resolution: {integrity: sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==} + engines: {node: '>=12'} + dev: true + /any-promise@1.3.0: resolution: {integrity: sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==} dev: true @@ -1076,6 +1139,7 @@ packages: /arg@4.1.3: resolution: {integrity: sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==} + dev: true /argparse@1.0.10: resolution: {integrity: sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==} @@ -1114,6 +1178,10 @@ packages: resolution: {integrity: sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==} dev: false + /async@3.2.5: + resolution: {integrity: sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg==} + dev: false + /asynckit@0.4.0: resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} @@ -1133,6 +1201,17 @@ packages: proxy-from-env: 1.1.0 transitivePeerDependencies: - debug + dev: false + + /axios@1.7.2: + resolution: {integrity: sha512-2A8QhOMrbomlDuiLeK9XibIBzuHeRcqqNOHp0Cyp5EoJ1IFDh+XZH3A6BkXtv0K4gFGCI0Y4BM7B1wOEi0Rmgw==} + dependencies: + follow-redirects: 1.15.6 + form-data: 4.0.0 + proxy-from-env: 1.1.0 + transitivePeerDependencies: + - debug + dev: true /balanced-match@1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} @@ -1148,8 +1227,8 @@ packages: tweetnacl: 0.14.5 dev: false - /binary-extensions@2.2.0: - resolution: {integrity: sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==} + /binary-extensions@2.3.0: + resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==} engines: {node: '>=8'} dev: true @@ -1221,26 +1300,26 @@ packages: balanced-match: 1.0.2 dev: true - /braces@3.0.2: - resolution: {integrity: sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==} + /braces@3.0.3: + resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} engines: {node: '>=8'} dependencies: - fill-range: 7.0.1 + fill-range: 7.1.1 dev: true /browser-stdout@1.3.1: resolution: {integrity: sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==} dev: true - /browserslist@4.21.9: - resolution: {integrity: sha512-M0MFoZzbUrRU4KNfCrDLnvyE7gub+peetoTid3TBIqtunaDJyXlwhakT+/VkvSXcfIzFfK/nkCs4nmyTmxdNSg==} + /browserslist@4.23.1: + resolution: {integrity: sha512-TUfofFo/KsK/bWZ9TWQ5O26tsWW4Uhmt8IYklbnUa70udB6P2wA7w7o4PY4muaEPBQaAX+CEnmmIA41NVHtPVw==} engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} hasBin: true dependencies: - caniuse-lite: 1.0.30001513 - electron-to-chromium: 1.4.454 - node-releases: 2.0.13 - update-browserslist-db: 1.0.11(browserslist@4.21.9) + caniuse-lite: 1.0.30001636 + electron-to-chromium: 1.4.808 + node-releases: 2.0.14 + update-browserslist-db: 1.0.16(browserslist@4.23.1) dev: true /bson@5.4.0: @@ -1308,6 +1387,18 @@ packages: dependencies: function-bind: 1.1.1 get-intrinsic: 1.2.1 + dev: false + + /call-bind@1.0.7: + resolution: {integrity: sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==} + engines: {node: '>= 0.4'} + dependencies: + es-define-property: 1.0.0 + es-errors: 1.3.0 + function-bind: 1.1.2 + get-intrinsic: 1.2.4 + set-function-length: 1.2.2 + dev: true /caller-callsite@2.0.0: resolution: {integrity: sha512-JuG3qI4QOftFsZyOn1qq87fq5grLIyk1JYd5lJmdA+fG7aQ9pA/i3JIJGcO3q0MrRcHlOt1U+ZeHW8Dq9axALQ==} @@ -1338,8 +1429,8 @@ packages: engines: {node: '>=10'} dev: true - /caniuse-lite@1.0.30001513: - resolution: {integrity: sha512-pnjGJo7SOOjAGytZZ203Em95MRM8Cr6jhCXNF/FAXTpCTRTECnqQWLpiTRqrFtdYcth8hf4WECUpkezuYsMVww==} + /caniuse-lite@1.0.30001636: + resolution: {integrity: sha512-bMg2vmr8XBsbL6Lr0UHXy/21m84FTxDLWn2FSqMd5PrlbMxwJlQnC2YWYxVgp66PZE+BBNF2jYQUBKCo1FDeZg==} dev: true /caseless@0.12.0: @@ -1350,40 +1441,40 @@ packages: resolution: {integrity: sha512-X2y0Ssu48ucdkrNgCdg6k3EZWjWVy/dsEywUUTeZEIW31f3bQfq65Svm+TzU1Hz+qqhdmyCdjGhUvRsSKHl/mw==} dev: false - /chai-as-promised@7.1.1(chai@4.3.7): - resolution: {integrity: sha512-azL6xMoi+uxu6z4rhWQ1jbdUhOMhis2PvscD/xjLqNMkv3BPPp2JyyuTHOrf9BOosGpNQ11v6BKv/g57RXbiaA==} + /chai-as-promised@7.1.2(chai@4.4.1): + resolution: {integrity: sha512-aBDHZxRzYnUYuIAIPBH2s511DjlKPzXNlXSGFC8CwmroWQLfrW0LtE1nK3MAwwNhJPa9raEjNCmRoFpG0Hurdw==} peerDependencies: - chai: '>= 2.1.2 < 5' + chai: '>= 2.1.2 < 6' dependencies: - chai: 4.3.7 - check-error: 1.0.2 + chai: 4.4.1 + check-error: 1.0.3 dev: true /chai-http@4.4.0: resolution: {integrity: sha512-uswN3rZpawlRaa5NiDUHcDZ3v2dw5QgLyAwnQ2tnVNuP7CwIsOFuYJ0xR1WiR7ymD4roBnJIzOUep7w9jQMFJA==} engines: {node: '>=10'} dependencies: - '@types/chai': 4.3.5 + '@types/chai': 4.3.16 '@types/superagent': 4.1.13 charset: 1.0.1 cookiejar: 2.1.4 is-ip: 2.0.0 methods: 1.1.2 - qs: 6.11.2 - superagent: 8.0.9 + qs: 6.12.1 + superagent: 8.1.2 transitivePeerDependencies: - supports-color dev: true - /chai@4.3.7: - resolution: {integrity: sha512-HLnAzZ2iupm25PlN0xFreAlBA5zaBSv3og0DdeGA4Ar6h6rJ3A0rolRUKJhSF2V10GZKDgWF/VmAEsNWjCRB+A==} + /chai@4.4.1: + resolution: {integrity: sha512-13sOfMv2+DWduEU+/xbun3LScLoqN17nBeTLUsmDfKdoiC1fr0n9PU4guu4AhRcOVFk/sW8LyZWHuhWtQZiF+g==} engines: {node: '>=4'} dependencies: assertion-error: 1.1.0 - check-error: 1.0.2 - deep-eql: 4.1.3 - get-func-name: 2.0.0 - loupe: 2.3.6 + check-error: 1.0.3 + deep-eql: 4.1.4 + get-func-name: 2.0.2 + loupe: 2.3.7 pathval: 1.1.1 type-detect: 4.0.8 dev: true @@ -1410,8 +1501,10 @@ packages: engines: {node: '>=4.0.0'} dev: true - /check-error@1.0.2: - resolution: {integrity: sha512-BrgHpW9NURQgzoNyjfq0Wu6VFO6D7IZEmJNdtgNqpzGG8RuNFHt2jQxWlAs4HMe119chBnv+34syEZtc6IhLtA==} + /check-error@1.0.3: + resolution: {integrity: sha512-iKEoDYaRmd1mxM90a2OEfWhjsjPpYPuQ+lMYsoxB126+t8fw7ySEO48nmDg5COTjxDI65/Y2OWpeEHk3ZOe8zg==} + dependencies: + get-func-name: 2.0.2 dev: true /chokidar@3.5.1: @@ -1419,14 +1512,14 @@ packages: engines: {node: '>= 8.10.0'} dependencies: anymatch: 3.1.3 - braces: 3.0.2 + braces: 3.0.3 glob-parent: 5.1.2 is-binary-path: 2.1.0 is-glob: 4.0.3 normalize-path: 3.0.0 readdirp: 3.5.0 optionalDependencies: - fsevents: 2.3.2 + fsevents: 2.3.3 dev: true /chokidar@3.5.3: @@ -1434,14 +1527,14 @@ packages: engines: {node: '>= 8.10.0'} dependencies: anymatch: 3.1.3 - braces: 3.0.2 + braces: 3.0.3 glob-parent: 5.1.2 is-binary-path: 2.1.0 is-glob: 4.0.3 normalize-path: 3.0.0 readdirp: 3.6.0 optionalDependencies: - fsevents: 2.3.2 + fsevents: 2.3.3 dev: true /chownr@1.1.4: @@ -1555,8 +1648,8 @@ packages: resolution: {integrity: sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==} dev: true - /component-emitter@1.3.0: - resolution: {integrity: sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg==} + /component-emitter@1.3.1: + resolution: {integrity: sha512-T0+barUSQRTUQASh8bx02dl+DhF54GtIDY13Y3m9oWTklKbb3Wv974meRpeZ3lp1JpLVECWWNHC4vaG2XHXouQ==} dev: true /concat-map@0.0.1: @@ -1605,6 +1698,10 @@ packages: resolution: {integrity: sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A==} dev: true + /convert-source-map@2.0.0: + resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} + dev: true + /cookie-signature@1.0.6: resolution: {integrity: sha1-4wOogrNCzD7oylE6eZmXNNqzriw=} dev: false @@ -1638,6 +1735,7 @@ packages: /create-require@1.1.1: resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==} + dev: true /cross-spawn@6.0.5: resolution: {integrity: sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==} @@ -1739,6 +1837,18 @@ packages: ms: 2.1.2 supports-color: 8.1.1 + /debug@4.3.5: + resolution: {integrity: sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + dependencies: + ms: 2.1.2 + dev: true + /decamelize@1.2.0: resolution: {integrity: sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==} engines: {node: '>=0.10.0'} @@ -1749,8 +1859,8 @@ packages: engines: {node: '>=10'} dev: true - /deep-eql@4.1.3: - resolution: {integrity: sha512-WaEtAOpRA1MQ0eohqZjpGD8zdI0Ovsm8mmFhaDN8dvDZzyoUMcYDnf5Y6iu7HTXxf8JDS23qWa4a+hKCDyOPzw==} + /deep-eql@4.1.4: + resolution: {integrity: sha512-SUwdGfqdKOwxCPeVYjwSyRpJ7Z+fhpwIAtmCUdZIWZ/YP5R9WAsyuSgpLVDi9bjWoN2LXHNss/dk3urXtdQxGg==} engines: {node: '>=6'} dependencies: type-detect: 4.0.8 @@ -1763,6 +1873,15 @@ packages: strip-bom: 4.0.0 dev: true + /define-data-property@1.1.4: + resolution: {integrity: sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==} + engines: {node: '>= 0.4'} + dependencies: + es-define-property: 1.0.0 + es-errors: 1.3.0 + gopd: 1.0.1 + dev: true + /define-lazy-prop@2.0.0: resolution: {integrity: sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og==} engines: {node: '>=8'} @@ -1789,17 +1908,23 @@ packages: wrappy: 1.0.2 dev: true + /diff-sequences@29.6.3: + resolution: {integrity: sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + dev: true + /diff@4.0.2: resolution: {integrity: sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==} engines: {node: '>=0.3.1'} + dev: true /diff@5.0.0: resolution: {integrity: sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w==} engines: {node: '>=0.3.1'} dev: true - /diff@5.1.0: - resolution: {integrity: sha512-D+mk+qE8VC/PAUrlAU34N+VfXev0ghe5ywmpqrawphmVZc1bEfn56uo9qpyGp1p4xpzOHkSW4ztBd6L7Xx4ACw==} + /diff@5.2.0: + resolution: {integrity: sha512-uIFDxqpRZGZ6ThOk84hEfqWoHx2devRFvpTZcTHur85vImfaxUbTW9Ryh4CpCuDnToOP1CEtXKIgytHBPVff5A==} engines: {node: '>=0.3.1'} dev: true @@ -1826,9 +1951,9 @@ packages: - supports-color dev: true - /dotenv@10.0.0: - resolution: {integrity: sha512-rlBi9d8jpv9Sf1klPjNfFAuWDjKLwTIJJ/VxtoTwIR6hnZxcEOQCZg2oIL3MWBYw5GpUDKOEnND7LXTbIpQ03Q==} - engines: {node: '>=10'} + /dotenv-expand@10.0.0: + resolution: {integrity: sha512-GopVGCpVS1UKH75VKHGuQFqS1Gusej0z4FyQkPdwjil2gNIv+LNsqBlboOzpJFZKVT95GkCyWJbBSdFEFUWI2A==} + engines: {node: '>=12'} dev: true /dotenv@16.3.1: @@ -1836,10 +1961,19 @@ packages: engines: {node: '>=12'} dev: false + /dotenv@16.3.2: + resolution: {integrity: sha512-HTlk5nmhkm8F6JcdXvHIzaorzCoziNQT9mGxLPVXW8wJF1TiGSL60ZGB4gHWabHOaMmWmhvk2/lPHfnBiT78AQ==} + engines: {node: '>=12'} + dev: true + /duplexer@0.1.2: resolution: {integrity: sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==} dev: true + /eastasianwidth@0.2.0: + resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} + dev: true + /ecc-jsbn@0.1.2: resolution: {integrity: sha512-eh9O+hwRHNbG4BLTjEl3nw044CkGm5X6LoaCf7LPp7UU8Qrt47JYNi6nPX8xjW97TKGKm1ouctg0QSpZe9qrnw==} dependencies: @@ -1857,8 +1991,8 @@ packages: resolution: {integrity: sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=} dev: false - /electron-to-chromium@1.4.454: - resolution: {integrity: sha512-pmf1rbAStw8UEQ0sr2cdJtWl48ZMuPD9Sto8HVQOq9vx9j2WgDEN6lYoaqFvqEHYOmGA9oRGn7LqWI9ta0YugQ==} + /electron-to-chromium@1.4.808: + resolution: {integrity: sha512-0ItWyhPYnww2VOuCGF4s1LTfbrdAV2ajy/TN+ZTuhR23AHI6rWHCrBXJ/uxoXOvRRqw8qjYVrG81HFI7x/2wdQ==} dev: true /emoji-regex@7.0.3: @@ -1869,6 +2003,10 @@ packages: resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} dev: true + /emoji-regex@9.2.2: + resolution: {integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==} + dev: true + /enabled@2.0.0: resolution: {integrity: sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==} dev: false @@ -1905,6 +2043,18 @@ packages: escape-html: 1.0.3 dev: false + /es-define-property@1.0.0: + resolution: {integrity: sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==} + engines: {node: '>= 0.4'} + dependencies: + get-intrinsic: 1.2.4 + dev: true + + /es-errors@1.3.0: + resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==} + engines: {node: '>= 0.4'} + dev: true + /es5-ext@0.10.62: resolution: {integrity: sha512-BHLqn0klhEpnOKSrzn/Xsz2UIW8j+cGmo9JLzr8BiUapV8hPL9+FliFqjwr9ngW7jWdnxv6eO+/LqyhJVqgrjA==} engines: {node: '>=0.10'} @@ -1943,8 +2093,8 @@ packages: es6-symbol: 3.1.3 dev: false - /escalade@3.1.1: - resolution: {integrity: sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==} + /escalade@3.1.2: + resolution: {integrity: sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==} engines: {node: '>=6'} dev: true @@ -2066,17 +2216,6 @@ packages: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} dev: false - /fast-glob@3.2.7: - resolution: {integrity: sha512-rYGMRwip6lUMvYD3BTScMwT1HtAs2d71SMv66Vrxs0IekGZEjhM0pcMfjQPnknBt2zeCwQMEupiN02ZP4DiT1Q==} - engines: {node: '>=8'} - dependencies: - '@nodelib/fs.stat': 2.0.5 - '@nodelib/fs.walk': 1.2.8 - glob-parent: 5.1.2 - merge2: 1.4.1 - micromatch: 4.0.5 - dev: true - /fast-json-stable-stringify@2.1.0: resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==} dev: false @@ -2085,12 +2224,6 @@ packages: resolution: {integrity: sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==} dev: true - /fastq@1.15.0: - resolution: {integrity: sha512-wBrocU2LCXXa+lWBt8RoIRD89Fi8OdABODa/kEnyeyjS5aZO5/GNvI5sEINADqP/h8M29UHTHUb53sUu5Ihqdw==} - dependencies: - reusify: 1.0.4 - dev: true - /fecha@4.2.3: resolution: {integrity: sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==} dev: false @@ -2102,8 +2235,8 @@ packages: escape-string-regexp: 1.0.5 dev: true - /fill-range@7.0.1: - resolution: {integrity: sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==} + /fill-range@7.1.1: + resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} engines: {node: '>=8'} dependencies: to-regex-range: 5.0.1 @@ -2173,6 +2306,17 @@ packages: peerDependenciesMeta: debug: optional: true + dev: false + + /follow-redirects@1.15.6: + resolution: {integrity: sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==} + engines: {node: '>=4.0'} + peerDependencies: + debug: '*' + peerDependenciesMeta: + debug: + optional: true + dev: true /foreground-child@2.0.0: resolution: {integrity: sha512-dCIq9FpEcyQyXKCkyzmlPTFNgrCzPudOe+mhvJU5zAtlBnGVy2yKxtfsxK2tQBThwq225jcvBjpw1Gr40uzZCA==} @@ -2182,6 +2326,14 @@ packages: signal-exit: 3.0.7 dev: true + /foreground-child@3.2.1: + resolution: {integrity: sha512-PXUUyLqrR2XCWICfv6ukppP96sdFwWbNEnfEMt7jNsISjMsvaLNinAHNDYyvkyU+SZG2BTSbT5NjG+vZslfGTA==} + engines: {node: '>=14'} + dependencies: + cross-spawn: 7.0.3 + signal-exit: 4.1.0 + dev: true + /forever-agent@0.6.1: resolution: {integrity: sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw==} dev: false @@ -2209,7 +2361,7 @@ packages: dezalgo: 1.0.4 hexoid: 1.0.0 once: 1.4.0 - qs: 6.11.2 + qs: 6.12.1 dev: true /forwarded@0.2.0: @@ -2230,13 +2382,13 @@ packages: resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==} dev: true - /fs-extra@11.1.1: - resolution: {integrity: sha512-MGIE4HOvQCeUCzmlHs0vXpih4ysz4wg9qiSAu6cd42lVwPbTM1TjV7RusoyQqMmk/95gdQZX72u+YW+c3eEpFQ==} + /fs-extra@11.2.0: + resolution: {integrity: sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw==} engines: {node: '>=14.14'} dependencies: graceful-fs: 4.2.11 jsonfile: 6.1.0 - universalify: 2.0.0 + universalify: 2.0.1 dev: true /fs-extra@8.1.0: @@ -2252,8 +2404,8 @@ packages: resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==} dev: true - /fsevents@2.3.2: - resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==} + /fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} os: [darwin] requiresBuild: true @@ -2263,6 +2415,10 @@ packages: /function-bind@1.1.1: resolution: {integrity: sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==} + /function-bind@1.1.2: + resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + dev: true + /gensync@1.0.0-beta.2: resolution: {integrity: sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==} engines: {node: '>=6.9.0'} @@ -2273,8 +2429,8 @@ packages: engines: {node: 6.* || 8.* || >= 10.*} dev: true - /get-func-name@2.0.0: - resolution: {integrity: sha512-Hm0ixYtaSZ/V7C8FJrtZIuBBI+iSgL+1Aq82zSu8VQNB4S3Gk8e7Qs3VwBDJAhmRZcFqkl3tQu36g/Foh5I5ig==} + /get-func-name@2.0.2: + resolution: {integrity: sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==} dev: true /get-intrinsic@1.2.1: @@ -2284,6 +2440,18 @@ packages: has: 1.0.3 has-proto: 1.0.1 has-symbols: 1.0.3 + dev: false + + /get-intrinsic@1.2.4: + resolution: {integrity: sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==} + engines: {node: '>= 0.4'} + dependencies: + es-errors: 1.3.0 + function-bind: 1.1.2 + has-proto: 1.0.3 + has-symbols: 1.0.3 + hasown: 2.0.2 + dev: true /get-package-type@0.1.0: resolution: {integrity: sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==} @@ -2327,30 +2495,45 @@ packages: is-glob: 4.0.3 dev: true + /glob@10.4.2: + resolution: {integrity: sha512-GwMlUF6PkPo3Gk21UxkCohOv0PLcIXVtKyLlpEI28R/cO/4eNOdmLk3CMW1wROV/WR/EsZOWAfBbBOqYvs88/w==} + engines: {node: '>=16 || 14 >=14.18'} + hasBin: true + dependencies: + foreground-child: 3.2.1 + jackspeak: 3.4.0 + minimatch: 9.0.4 + minipass: 7.1.2 + package-json-from-dist: 1.0.0 + path-scurry: 1.11.1 + dev: true + /glob@7.1.4: resolution: {integrity: sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A==} + deprecated: Glob versions prior to v9 are no longer supported dependencies: fs.realpath: 1.0.0 inflight: 1.0.6 inherits: 2.0.4 - minimatch: 3.1.2 + minimatch: 3.0.5 once: 1.4.0 path-is-absolute: 1.0.1 dev: true /glob@7.1.6: resolution: {integrity: sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==} + deprecated: Glob versions prior to v9 are no longer supported dependencies: fs.realpath: 1.0.0 inflight: 1.0.6 inherits: 2.0.4 - minimatch: 3.1.2 + minimatch: 3.0.4 once: 1.4.0 path-is-absolute: 1.0.1 dev: true - /glob@7.2.0: - resolution: {integrity: sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==} + /glob@7.2.3: + resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==} deprecated: Glob versions prior to v9 are no longer supported dependencies: fs.realpath: 1.0.0 @@ -2361,14 +2544,16 @@ packages: path-is-absolute: 1.0.1 dev: true - /glob@9.3.5: - resolution: {integrity: sha512-e1LleDykUz2Iu+MTYdkSsuWX8lvAjAcs0Xef0lNIu0S2wOAzuTxCJtcd9S3cijlwYF18EsU3rzb8jPVobxDh9Q==} - engines: {node: '>=16 || 14 >=14.17'} + /glob@8.1.0: + resolution: {integrity: sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==} + engines: {node: '>=12'} + deprecated: Glob versions prior to v9 are no longer supported dependencies: fs.realpath: 1.0.0 - minimatch: 8.0.4 - minipass: 4.2.8 - path-scurry: 1.11.1 + inflight: 1.0.6 + inherits: 2.0.4 + minimatch: 5.0.1 + once: 1.4.0 dev: true /globals@11.12.0: @@ -2376,6 +2561,12 @@ packages: engines: {node: '>=4'} dev: true + /gopd@1.0.1: + resolution: {integrity: sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==} + dependencies: + get-intrinsic: 1.2.4 + dev: true + /graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} dev: true @@ -2395,7 +2586,7 @@ packages: source-map: 0.6.1 wordwrap: 1.0.0 optionalDependencies: - uglify-js: 3.17.4 + uglify-js: 3.18.0 dev: true /har-schema@2.0.0: @@ -2421,9 +2612,21 @@ packages: resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} engines: {node: '>=8'} + /has-property-descriptors@1.0.2: + resolution: {integrity: sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==} + dependencies: + es-define-property: 1.0.0 + dev: true + /has-proto@1.0.1: resolution: {integrity: sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==} engines: {node: '>= 0.4'} + dev: false + + /has-proto@1.0.3: + resolution: {integrity: sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==} + engines: {node: '>= 0.4'} + dev: true /has-symbols@1.0.3: resolution: {integrity: sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==} @@ -2443,6 +2646,13 @@ packages: type-fest: 0.8.1 dev: true + /hasown@2.0.2: + resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} + engines: {node: '>= 0.4'} + dependencies: + function-bind: 1.1.2 + dev: true + /he@1.2.0: resolution: {integrity: sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==} hasBin: true @@ -2529,11 +2739,6 @@ packages: resolution: {integrity: sha512-Ius2VYcGNk7T90CppJqcIkS5ooHUZyIQK+ClZfMfMNFEF9VSE73Fq+906u/CWu92x4gzZMWOwfFYckPObzdEbA==} dev: true - /ignore@5.2.4: - resolution: {integrity: sha512-MAb38BcSbH0eHNBxn7ql2NH/kX33OkB3lZ1BNdh7ENeRChHTYsTvWrMubiIAMNS2llXEEgZ1MUOBtXChP3kaFQ==} - engines: {node: '>= 4'} - dev: true - /ignore@5.3.1: resolution: {integrity: sha512-5Fytz/IraMjqpwfd34ke28PTVMjZjJG2MPn5t7OE4eUCUNf8BAa7b5WUS9/Qvr6mwOQS7Mk6vdsMno5he+T8Xw==} engines: {node: '>= 4'} @@ -2603,7 +2808,7 @@ packages: resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==} engines: {node: '>=8'} dependencies: - binary-extensions: 2.2.0 + binary-extensions: 2.3.0 dev: true /is-core-module@2.12.1: @@ -2612,6 +2817,13 @@ packages: has: 1.0.3 dev: true + /is-core-module@2.14.0: + resolution: {integrity: sha512-a5dFJih5ZLYlRtDc0dZWP7RiKr6xIKzmn/oAYCDvdLThadVgyJwlaoQPmRtMSpz+rk0OGAgIu+TcM9HUF0fk1A==} + engines: {node: '>= 0.4'} + dependencies: + hasown: 2.0.2 + dev: true + /is-directory@0.3.1: resolution: {integrity: sha512-yVChGzahRFvbkscn2MlwGismPO12i9+znNruC5gVEntG3qu0xQMzsGg/JFbrsqDOHtHFPci+V5aP5T9I+yeKqw==} engines: {node: '>=0.10.0'} @@ -2711,8 +2923,8 @@ packages: resolution: {integrity: sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==} dev: false - /istanbul-lib-coverage@3.2.0: - resolution: {integrity: sha512-eOeJ5BHCmHYvQK7xt9GkdHuzuCGS1Y6g9Gvnx3Ym33fz/HpLRYxiS0wHNr+m/MBC8B647Xt608vCDEvhl9c6Mw==} + /istanbul-lib-coverage@3.2.2: + resolution: {integrity: sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==} engines: {node: '>=8'} dev: true @@ -2727,10 +2939,10 @@ packages: resolution: {integrity: sha512-BXgQl9kf4WTCPCCpmFGoJkz/+uhvm7h7PFKUYxh7qarQd3ER33vHG//qaE8eN25l07YqZPpHXU9I09l/RD5aGQ==} engines: {node: '>=8'} dependencies: - '@babel/core': 7.22.8 + '@babel/core': 7.24.7 '@istanbuljs/schema': 0.1.3 - istanbul-lib-coverage: 3.2.0 - semver: 6.3.0 + istanbul-lib-coverage: 3.2.2 + semver: 6.3.1 transitivePeerDependencies: - supports-color dev: true @@ -2741,18 +2953,18 @@ packages: dependencies: archy: 1.0.0 cross-spawn: 7.0.3 - istanbul-lib-coverage: 3.2.0 + istanbul-lib-coverage: 3.2.2 p-map: 3.0.0 rimraf: 3.0.2 uuid: 8.3.2 dev: true - /istanbul-lib-report@3.0.0: - resolution: {integrity: sha512-wcdi+uAKzfiGT2abPpKZ0hSU1rGQjUQnLvtY5MpQ7QCTahD3VODhcu4wcfY1YtkGaDD5yuydOLINXsfbus9ROw==} - engines: {node: '>=8'} + /istanbul-lib-report@3.0.1: + resolution: {integrity: sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==} + engines: {node: '>=10'} dependencies: - istanbul-lib-coverage: 3.2.0 - make-dir: 3.1.0 + istanbul-lib-coverage: 3.2.2 + make-dir: 4.0.0 supports-color: 7.2.0 dev: true @@ -2760,19 +2972,43 @@ packages: resolution: {integrity: sha512-n3s8EwkdFIJCG3BPKBYvskgXGoy88ARzvegkitk60NxRdwltLOTaH7CUiMRXvwYorl0Q712iEjcWB+fK/MrWVw==} engines: {node: '>=10'} dependencies: - debug: 4.3.4(supports-color@8.1.1) - istanbul-lib-coverage: 3.2.0 + debug: 4.3.5 + istanbul-lib-coverage: 3.2.2 source-map: 0.6.1 transitivePeerDependencies: - supports-color dev: true - /istanbul-reports@3.1.5: - resolution: {integrity: sha512-nUsEMa9pBt/NOHqbcbeJEgqIlY/K7rVWUX6Lql2orY5e9roQOthbR3vtY4zzf2orPELg80fnxxk9zUyPlgwD1w==} + /istanbul-reports@3.1.7: + resolution: {integrity: sha512-BewmUXImeuRk2YY0PVbxgKAysvhRPUQE0h5QRM++nVWyubKGV0l8qQ5op8+B2DOmwSe63Jivj0BjkPQVf8fP5g==} engines: {node: '>=8'} dependencies: html-escaper: 2.0.2 - istanbul-lib-report: 3.0.0 + istanbul-lib-report: 3.0.1 + dev: true + + /jackspeak@3.4.0: + resolution: {integrity: sha512-JVYhQnN59LVPFCEcVa2C3CrEKYacvjRfqIQl+h8oi91aLYQVWRYbxjPcv1bUiUy/kLmQaANrYfNMCO3kuEDHfw==} + engines: {node: '>=14'} + dependencies: + '@isaacs/cliui': 8.0.2 + optionalDependencies: + '@pkgjs/parseargs': 0.11.0 + dev: true + + /jest-diff@29.7.0: + resolution: {integrity: sha512-LMIgiIrhigmPrs03JHpxUh2yISK3vLFPkAodPeo0+BuF7wA2FoQbkEg1u8gBYBThncu7e1oEDUfIXVuTqLRUjw==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + dependencies: + chalk: 4.1.2 + diff-sequences: 29.6.3 + jest-get-type: 29.6.3 + pretty-format: 29.7.0 + dev: true + + /jest-get-type@29.6.3: + resolution: {integrity: sha512-zrteXnqYxfQh7l5FHyL38jL39di8H8rHoecLH3JNxH3BwOrBsNeabdap5e0I23lD4HHI8W5VFBZqG4Eaq5LNcw==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} dev: true /js-tokens@4.0.0: @@ -2854,7 +3090,7 @@ packages: /jsonfile@6.1.0: resolution: {integrity: sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==} dependencies: - universalify: 2.0.0 + universalify: 2.0.1 optionalDependencies: graceful-fs: 4.2.11 dev: true @@ -2890,8 +3126,8 @@ packages: verror: 1.10.0 dev: false - /just-extend@4.2.1: - resolution: {integrity: sha512-g3UB796vUFIY90VIv/WX3L2c8CS2MdWUww3CNrYmqza1Fg0DURc2K/O4YrnklBdQarSJ/y8JnJYDGc+1iumQjg==} + /just-extend@6.2.0: + resolution: {integrity: sha512-cYofQu2Xpom82S6qD778jBDpwvvy39s1l/hrYij2u9AMdQcGRpaBu6kY4mVhuno5kJVi1DAz4aiphA2WI1/OAw==} dev: true /jwa@1.4.1: @@ -2922,8 +3158,8 @@ packages: resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} dev: true - /lines-and-columns@2.0.3: - resolution: {integrity: sha512-cNOjgCnLB+FnvWWtyRTzmB3POJ+cXxTA81LoW7u8JdmhfXzriropYwpjShnz1QLLWsQwY7nIxoDmcPTwphDK9w==} + /lines-and-columns@2.0.4: + resolution: {integrity: sha512-wM1+Z03eypVAVUCE7QdSqpVIvelbOakn1M0bPDoA4SGWPx3sNDVUiMo3L6To6WWGClB7VyXnhQ4Sn7gxiJbE6A==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} dev: true @@ -3014,10 +3250,22 @@ packages: triple-beam: 1.3.0 dev: false - /loupe@2.3.6: - resolution: {integrity: sha512-RaPMZKiMy8/JruncMU5Bt6na1eftNoo++R4Y+N2FrxkDVTrGvcyzFTsaGif4QTeKESheMGegbhw6iUAq+5A8zA==} + /logform@2.6.0: + resolution: {integrity: sha512-1ulHeNPp6k/LD8H91o7VYFBng5i1BDE7HoKxVbZiGFidS1Rj65qcywLxX+pVfAPoQJEjRdvKcusKwOupHCVOVQ==} + engines: {node: '>= 12.0.0'} dependencies: - get-func-name: 2.0.0 + '@colors/colors': 1.6.0 + '@types/triple-beam': 1.3.5 + fecha: 4.2.3 + ms: 2.1.3 + safe-stable-stringify: 2.4.3 + triple-beam: 1.4.1 + dev: false + + /loupe@2.3.7: + resolution: {integrity: sha512-zSMINGVYkdpYSOBmLi0D1Uo7JU9nVdQKrHxC8eYlV+9YKK9WePqAlL7lSlorG/U2Fw1w0hTBmaa/jrQ3UbPHtA==} + dependencies: + get-func-name: 2.0.2 dev: true /lru-cache@10.2.2: @@ -3052,11 +3300,19 @@ packages: resolution: {integrity: sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==} engines: {node: '>=8'} dependencies: - semver: 6.3.0 + semver: 6.3.1 + dev: true + + /make-dir@4.0.0: + resolution: {integrity: sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==} + engines: {node: '>=10'} + dependencies: + semver: 7.6.2 dev: true /make-error@1.3.6: resolution: {integrity: sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==} + dev: true /marked@1.0.0: resolution: {integrity: sha512-Wo+L1pWTVibfrSr+TTtMuiMfNzmZWiOPeO7rZsQUY5bgsxpHesBEcIWJloWVTFnrMXnf/TL30eTFSGJddmQAng==} @@ -3096,23 +3352,10 @@ packages: resolution: {integrity: sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==} dev: true - /merge2@1.4.1: - resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==} - engines: {node: '>= 8'} - dev: true - /methods@1.1.2: resolution: {integrity: sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==} engines: {node: '>= 0.6'} - /micromatch@4.0.5: - resolution: {integrity: sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==} - engines: {node: '>=8.6'} - dependencies: - braces: 3.0.2 - picomatch: 2.3.1 - dev: true - /mime-db@1.52.0: resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} engines: {node: '>= 0.6'} @@ -3165,8 +3408,8 @@ packages: brace-expansion: 2.0.1 dev: true - /minimatch@8.0.4: - resolution: {integrity: sha512-W0Wvr9HyFXZRGIDgCicunpQ299OKXs9RgZfaukz4qAW/pJhcpUfupc9c+OObPOFueNy8VSrZgEmDtk6Kh4WzDA==} + /minimatch@9.0.4: + resolution: {integrity: sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==} engines: {node: '>=16 || 14 >=14.17'} dependencies: brace-expansion: 2.0.1 @@ -3176,11 +3419,6 @@ packages: resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} dev: true - /minipass@4.2.8: - resolution: {integrity: sha512-fNzuVyifolSLFL4NzpF+wEF4qrgqaaKX0haXPQEdQ7NKAN+WecoKMHV09YcuL/DHxrUsYQOK3MiuDf7Ip2OXfQ==} - engines: {node: '>=8'} - dev: true - /minipass@7.1.2: resolution: {integrity: sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==} engines: {node: '>=16 || 14 >=14.17'} @@ -3197,12 +3435,11 @@ packages: minimist: 1.2.8 dev: true - /mocha@10.0.0: - resolution: {integrity: sha512-0Wl+elVUD43Y0BqPZBzZt8Tnkw9CMUdNYnUsTfOM1vuhJVZL+kiesFYsqwBkEEuEixaiPe5ZQdqDgX2jddhmoA==} + /mocha@10.4.0: + resolution: {integrity: sha512-eqhGB8JKapEYcC4ytX/xrzKforgEc3j1pGlAXVy3eRwrtAy5/nIfT1SvgGzfN0XZZxeLq0aQWkOUAmqIJiv+bA==} engines: {node: '>= 14.0.0'} hasBin: true dependencies: - '@ungap/promise-all-settled': 1.1.2 ansi-colors: 4.1.1 browser-stdout: 1.3.1 chokidar: 3.5.3 @@ -3210,13 +3447,12 @@ packages: diff: 5.0.0 escape-string-regexp: 4.0.0 find-up: 5.0.0 - glob: 7.2.0 + glob: 8.1.0 he: 1.2.0 js-yaml: 4.1.0 log-symbols: 4.1.0 minimatch: 5.0.1 ms: 2.1.3 - nanoid: 3.3.3 serialize-javascript: 6.0.0 strip-json-comments: 3.1.1 supports-color: 8.1.1 @@ -3346,12 +3582,6 @@ packages: hasBin: true dev: true - /nanoid@3.3.3: - resolution: {integrity: sha512-p1sjXuopFs0xg+fPASzQ28agW1oHD7xDsd9Xkf3T15H3c/cifrFHVwrh74PdoklAPi+i7MdRsE47vm2r6JoB+w==} - engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} - hasBin: true - dev: true - /negotiator@0.6.3: resolution: {integrity: sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==} engines: {node: '>= 0.6'} @@ -3369,14 +3599,14 @@ packages: resolution: {integrity: sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==} dev: true - /nise@5.1.4: - resolution: {integrity: sha512-8+Ib8rRJ4L0o3kfmyVCL7gzrohyDe0cMFTBa2d364yIrEGMEoetznKJx899YxjybU6bL9SQkYPSBBs1gyYs8Xg==} + /nise@5.1.9: + resolution: {integrity: sha512-qOnoujW4SV6e40dYxJOb3uvuoPHtmLzIk4TFo+j0jPJoC+5Z9xja5qH5JZobEPsa8+YYphMrOSwnrshEhG2qww==} dependencies: - '@sinonjs/commons': 2.0.0 - '@sinonjs/fake-timers': 10.3.0 + '@sinonjs/commons': 3.0.1 + '@sinonjs/fake-timers': 11.2.2 '@sinonjs/text-encoding': 0.7.2 - just-extend: 4.2.1 - path-to-regexp: 1.8.0 + just-extend: 6.2.0 + path-to-regexp: 6.2.2 dev: true /node-addon-api@3.2.1: @@ -3399,11 +3629,15 @@ packages: whatwg-url: 5.0.0 dev: false - /node-gyp-build@4.6.0: - resolution: {integrity: sha512-NTZVKn9IylLwUzaKjkas1e4u2DLNcV4rdYagA4PWdPwW87Bi7z+BznyKSRwS/761tV/lzCGXplWsiaMjLqP2zQ==} + /node-gyp-build@4.8.1: + resolution: {integrity: sha512-OSs33Z9yWr148JZcbZd5WiAXhh/n9z8TxQcdMhIOlpN9AhWpLfvVFO73+m77bBABQMaY9XSvIa+qk0jlI7Gcaw==} hasBin: true dev: true + /node-machine-id@1.1.12: + resolution: {integrity: sha512-QNABxbrPa3qEIfrE6GOJ7BYIuignnJw7iQ2YPbc3Nla1HzRJjXzZOiikfF8m7eAMfichLt3M4VgLOetqgDmgGQ==} + dev: true + /node-preload@0.2.1: resolution: {integrity: sha512-RM5oyBy45cLEoHqCeh+MNuFAxO0vTFBLskvQbOKnEE7YTTSN4tbN8QWDIPQ6L+WvKsB/qLEGpYe2ZZ9d4W9OIQ==} engines: {node: '>=8'} @@ -3411,8 +3645,8 @@ packages: process-on-spawn: 1.0.0 dev: true - /node-releases@2.0.13: - resolution: {integrity: sha512-uYr7J37ae/ORWdZeQ1xxMJe3NtdmqMC/JZK+geofDrkLUApKRHPd18/TxtBOJ4A0/+uUIliorNrfYV6s1b02eQ==} + /node-releases@2.0.14: + resolution: {integrity: sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw==} dev: true /node-vault@0.9.22: @@ -3479,64 +3713,66 @@ packages: path-key: 3.1.1 dev: true - /nx@16.5.0: - resolution: {integrity: sha512-X95atskaF1ejrF+C80mC4SwFPq0G/yFvxhfeWpPjKj7vUJEy1nZ4SjqlNVMORdN8dKQTE6ss76cIJux3fE7EXw==} + /nx@16.10.0: + resolution: {integrity: sha512-gZl4iCC0Hx0Qe1VWmO4Bkeul2nttuXdPpfnlcDKSACGu3ZIo+uySqwOF8yBAxSTIf8xe2JRhgzJN1aFkuezEBg==} hasBin: true requiresBuild: true peerDependencies: - '@swc-node/register': ^1.4.2 - '@swc/core': ^1.2.173 + '@swc-node/register': ^1.6.7 + '@swc/core': ^1.3.85 peerDependenciesMeta: '@swc-node/register': optional: true '@swc/core': optional: true dependencies: - '@nrwl/tao': 16.5.0 + '@nrwl/tao': 16.10.0 '@parcel/watcher': 2.0.4 '@yarnpkg/lockfile': 1.1.0 '@yarnpkg/parsers': 3.0.0-rc.46 '@zkochan/js-yaml': 0.0.6 - axios: 1.4.0 + axios: 1.7.2 chalk: 4.1.2 cli-cursor: 3.1.0 cli-spinners: 2.6.1 - cliui: 7.0.4 - dotenv: 10.0.0 + cliui: 8.0.1 + dotenv: 16.3.2 + dotenv-expand: 10.0.0 enquirer: 2.3.6 - fast-glob: 3.2.7 figures: 3.2.0 flat: 5.0.2 - fs-extra: 11.1.1 + fs-extra: 11.2.0 glob: 7.1.4 - ignore: 5.2.4 + ignore: 5.3.1 + jest-diff: 29.7.0 js-yaml: 4.1.0 jsonc-parser: 3.2.0 - lines-and-columns: 2.0.3 + lines-and-columns: 2.0.4 minimatch: 3.0.5 + node-machine-id: 1.1.12 npm-run-path: 4.0.1 open: 8.4.2 semver: 7.5.3 string-width: 4.2.3 strong-log-transformer: 2.1.0 tar-stream: 2.2.0 - tmp: 0.2.1 + tmp: 0.2.3 tsconfig-paths: 4.2.0 - tslib: 2.6.0 + tslib: 2.6.3 v8-compile-cache: 2.3.0 yargs: 17.7.2 yargs-parser: 21.1.1 optionalDependencies: - '@nx/nx-darwin-arm64': 16.5.0 - '@nx/nx-darwin-x64': 16.5.0 - '@nx/nx-freebsd-x64': 16.5.0 - '@nx/nx-linux-arm-gnueabihf': 16.5.0 - '@nx/nx-linux-arm64-gnu': 16.5.0 - '@nx/nx-linux-arm64-musl': 16.5.0 - '@nx/nx-linux-x64-gnu': 16.5.0 - '@nx/nx-linux-x64-musl': 16.5.0 - '@nx/nx-win32-arm64-msvc': 16.5.0 - '@nx/nx-win32-x64-msvc': 16.5.0 + '@nx/nx-darwin-arm64': 16.10.0 + '@nx/nx-darwin-x64': 16.10.0 + '@nx/nx-freebsd-x64': 16.10.0 + '@nx/nx-linux-arm-gnueabihf': 16.10.0 + '@nx/nx-linux-arm64-gnu': 16.10.0 + '@nx/nx-linux-arm64-musl': 16.10.0 + '@nx/nx-linux-x64-gnu': 16.10.0 + '@nx/nx-linux-x64-musl': 16.10.0 + '@nx/nx-win32-arm64-msvc': 16.10.0 + '@nx/nx-win32-x64-msvc': 16.10.0 transitivePeerDependencies: - debug dev: true @@ -3555,14 +3791,14 @@ packages: find-up: 4.1.0 foreground-child: 2.0.0 get-package-type: 0.1.0 - glob: 7.2.0 - istanbul-lib-coverage: 3.2.0 + glob: 7.2.3 + istanbul-lib-coverage: 3.2.2 istanbul-lib-hook: 3.0.0 istanbul-lib-instrument: 4.0.3 istanbul-lib-processinfo: 2.0.3 - istanbul-lib-report: 3.0.0 + istanbul-lib-report: 3.0.1 istanbul-lib-source-maps: 4.0.1 - istanbul-reports: 3.1.5 + istanbul-reports: 3.1.7 make-dir: 3.1.0 node-preload: 0.2.1 p-map: 3.0.0 @@ -3583,6 +3819,11 @@ packages: /object-inspect@1.12.3: resolution: {integrity: sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g==} + dev: false + + /object-inspect@1.13.1: + resolution: {integrity: sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==} + dev: true /on-finished@2.4.1: resolution: {integrity: sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==} @@ -3686,6 +3927,10 @@ packages: release-zalgo: 1.0.0 dev: true + /package-json-from-dist@1.0.0: + resolution: {integrity: sha512-dATvCeZN/8wQsGywez1mzHtTlP22H8OEfPrVMLNr4/eGa+ijtLn/6M5f0dY8UKNrC2O9UCU6SSoG3qRKnt7STw==} + dev: true + /parse-json@4.0.0: resolution: {integrity: sha512-aOIos8bujGN93/8Ox/jPLh7RwVnPEysynVFE+fQZyg6jKELEHwzgKdLRFHUgXJL6kylijVSBC4BvN9OmsB48Rw==} engines: {node: '>=4'} @@ -3750,10 +3995,8 @@ packages: resolution: {integrity: sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==} dev: false - /path-to-regexp@1.8.0: - resolution: {integrity: sha512-n43JRhlUKUAlibEJhPeir1ncUID16QnEjNpwzNdO3Lm4ywrBpBZ5oLD0I6br9evr1Y9JTqwRtAh7JLoOzAQdVA==} - dependencies: - isarray: 0.0.1 + /path-to-regexp@6.2.2: + resolution: {integrity: sha512-GQX3SSMokngb36+whdpRXE+3f9V8UzyAorlYvOGx87ufGHehNTn5lCxrKtLyZ4Yl/wEKnNnr98ZzOwwDZV5ogw==} dev: true /pathval@1.1.1: @@ -3764,8 +4007,8 @@ packages: resolution: {integrity: sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==} dev: false - /picocolors@1.0.0: - resolution: {integrity: sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==} + /picocolors@1.0.1: + resolution: {integrity: sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew==} dev: true /picomatch@2.3.1: @@ -3808,6 +4051,21 @@ packages: hasBin: true dev: true + /prettier@3.3.2: + resolution: {integrity: sha512-rAVeHYMcv8ATV5d508CFdn+8/pHPpXeIid1DdrPwXnaAdH7cqjVbpJaT5eq4yRAFU/lsbwYwSF/n5iNrdJHPQA==} + engines: {node: '>=14'} + hasBin: true + dev: true + + /pretty-format@29.7.0: + resolution: {integrity: sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + dependencies: + '@jest/schemas': 29.6.3 + ansi-styles: 5.2.0 + react-is: 18.3.1 + dev: true + /pretty-quick@3.3.1(prettier@2.8.8): resolution: {integrity: sha512-3b36UXfYQ+IXXqex6mCca89jC8u0mYLqFAN5eTQKoXO6oCQYcIVYZEB/5AlBHI7JPYygReM2Vv6Vom/Gln7fBg==} engines: {node: '>=10.13'} @@ -3819,10 +4077,10 @@ packages: find-up: 4.1.0 ignore: 5.3.1 mri: 1.2.0 - picocolors: 1.0.0 + picocolors: 1.0.1 picomatch: 3.0.1 prettier: 2.8.8 - tslib: 2.6.2 + tslib: 2.6.3 dev: true /process-nextick-args@2.0.1: @@ -3894,11 +4152,11 @@ packages: side-channel: 1.0.4 dev: false - /qs@6.11.2: - resolution: {integrity: sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA==} + /qs@6.12.1: + resolution: {integrity: sha512-zWmv4RSuB9r2mYQw3zxQuHWeU+42aKi1wWig/j4ele4ygELZ7PEO6MM7rim9oAQH2A5MWfsAVf/jPvTPgCbvUQ==} engines: {node: '>=0.6'} dependencies: - side-channel: 1.0.4 + side-channel: 1.0.6 dev: true /qs@6.5.3: @@ -3906,10 +4164,6 @@ packages: engines: {node: '>=0.6'} dev: false - /queue-microtask@1.2.3: - resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} - dev: true - /randombytes@2.1.0: resolution: {integrity: sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==} dependencies: @@ -3941,6 +4195,10 @@ packages: unpipe: 1.0.0 dev: false + /react-is@18.3.1: + resolution: {integrity: sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==} + dev: true + /read-pkg@4.0.1: resolution: {integrity: sha512-+UBirHHDm5J+3WDmLBZYSklRYg82nMlz+enn+GMZ22nSR2f4bzxmhso6rzQW/3mT2PVzpzDTiYIZahk8UmZ44w==} engines: {node: '>=6'} @@ -4007,7 +4265,7 @@ packages: resolution: {integrity: sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==} engines: {node: '>= 0.10'} dependencies: - resolve: 1.22.2 + resolve: 1.22.8 dev: true /regenerator-runtime@0.13.11: @@ -4104,6 +4362,15 @@ packages: supports-preserve-symlinks-flag: 1.0.0 dev: true + /resolve@1.22.8: + resolution: {integrity: sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==} + hasBin: true + dependencies: + is-core-module: 2.14.0 + path-parse: 1.0.7 + supports-preserve-symlinks-flag: 1.0.0 + dev: true + /restore-cursor@3.1.0: resolution: {integrity: sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==} engines: {node: '>=8'} @@ -4112,25 +4379,20 @@ packages: signal-exit: 3.0.7 dev: true - /reusify@1.0.4: - resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==} - engines: {iojs: '>=1.0.0', node: '>=0.10.0'} - dev: true - /rimraf@3.0.2: resolution: {integrity: sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==} deprecated: Rimraf versions prior to v4 are no longer supported hasBin: true dependencies: - glob: 7.2.0 + glob: 7.2.3 dev: true - /rimraf@4.4.1: - resolution: {integrity: sha512-Gk8NlF062+T9CqNGn6h4tls3k6T1+/nXdOcSZVikNVtlRdYpA7wRJJMoXmuvOnLW844rPjdQ7JgXCYM6PPC/og==} - engines: {node: '>=14'} + /rimraf@5.0.7: + resolution: {integrity: sha512-nV6YcJo5wbLW77m+8KjH8aB/7/rxQy9SZ0HY5shnwULfS+9nmTtVXAJET5NdZmCzA4fPI/Hm1wo/Po/4mopOdg==} + engines: {node: '>=14.18'} hasBin: true dependencies: - glob: 9.3.5 + glob: 10.4.2 dev: true /run-node@1.0.0: @@ -4139,12 +4401,6 @@ packages: hasBin: true dev: true - /run-parallel@1.2.0: - resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==} - dependencies: - queue-microtask: 1.2.3 - dev: true - /rxjs@6.6.7: resolution: {integrity: sha512-hTdwr+7yYNIT5n4AMYp85KA6yw2Va0FLa3Rguvbpa4W3I5xynaBZo41cM3XM+4Q6fRMj3sBYIR1VAmZMXYJvRQ==} engines: {npm: '>=2.0.0'} @@ -4185,8 +4441,13 @@ packages: resolution: {integrity: sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==} hasBin: true - /semver@6.3.0: - resolution: {integrity: sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==} + /semver@5.7.2: + resolution: {integrity: sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==} + hasBin: true + dev: true + + /semver@6.3.1: + resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==} hasBin: true dev: true @@ -4203,12 +4464,10 @@ packages: lru-cache: 6.0.0 dev: true - /semver@7.5.4: - resolution: {integrity: sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==} + /semver@7.6.2: + resolution: {integrity: sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==} engines: {node: '>=10'} hasBin: true - dependencies: - lru-cache: 6.0.0 dev: true /send@0.18.0: @@ -4260,6 +4519,18 @@ packages: resolution: {integrity: sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==} dev: true + /set-function-length@1.2.2: + resolution: {integrity: sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==} + engines: {node: '>= 0.4'} + dependencies: + define-data-property: 1.1.4 + es-errors: 1.3.0 + function-bind: 1.1.2 + get-intrinsic: 1.2.4 + gopd: 1.0.1 + has-property-descriptors: 1.0.2 + dev: true + /setprototypeof@1.2.0: resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} dev: false @@ -4293,7 +4564,7 @@ packages: engines: {node: '>=4'} hasBin: true dependencies: - glob: 7.2.0 + glob: 7.2.3 interpret: 1.4.0 rechoir: 0.6.2 dev: true @@ -4304,6 +4575,17 @@ packages: call-bind: 1.0.2 get-intrinsic: 1.2.1 object-inspect: 1.12.3 + dev: false + + /side-channel@1.0.6: + resolution: {integrity: sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==} + engines: {node: '>= 0.4'} + dependencies: + call-bind: 1.0.7 + es-errors: 1.3.0 + get-intrinsic: 1.2.4 + object-inspect: 1.13.1 + dev: true /sift@16.0.1: resolution: {integrity: sha512-Wv6BjQ5zbhW7VFefWusVP33T/EM0vYikCaQ2qR8yULbsilAT8/wQaXvuQ3ptGLpoKx+lihJE3y2UTgKDyyNHZQ==} @@ -4313,6 +4595,11 @@ packages: resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==} dev: true + /signal-exit@4.1.0: + resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} + engines: {node: '>=14'} + dev: true + /simple-swizzle@0.2.2: resolution: {integrity: sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==} dependencies: @@ -4328,12 +4615,13 @@ packages: /sinon@15.2.0: resolution: {integrity: sha512-nPS85arNqwBXaIsFCkolHjGIkFo+Oxu9vbgmBJizLAhqe6P2o3Qmj3KCUoRkfhHtvgDhZdWD3risLHAUJ8npjw==} + deprecated: 16.1.1 dependencies: - '@sinonjs/commons': 3.0.0 + '@sinonjs/commons': 3.0.1 '@sinonjs/fake-timers': 10.3.0 '@sinonjs/samsam': 8.0.0 - diff: 5.1.0 - nise: 5.1.4 + diff: 5.2.0 + nise: 5.1.9 supports-color: 7.2.0 dev: true @@ -4483,6 +4771,15 @@ packages: strip-ansi: 6.0.1 dev: true + /string-width@5.1.2: + resolution: {integrity: sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==} + engines: {node: '>=12'} + dependencies: + eastasianwidth: 0.2.0 + emoji-regex: 9.2.2 + strip-ansi: 7.1.0 + dev: true + /string_decoder@0.10.31: resolution: {integrity: sha512-ev2QzSzWPYmy9GuqfIVildA4OdcGLeFZQrq5ys6RtiuF+RQQiZWr8TZNyAcuVXyQRYfEO+MsoB/1BuQVhOJuoQ==} dev: true @@ -4519,6 +4816,13 @@ packages: ansi-regex: 5.0.1 dev: true + /strip-ansi@7.1.0: + resolution: {integrity: sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==} + engines: {node: '>=12'} + dependencies: + ansi-regex: 6.0.1 + dev: true + /strip-bom@3.0.0: resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==} engines: {node: '>=4'} @@ -4554,20 +4858,21 @@ packages: through: 2.3.8 dev: true - /superagent@8.0.9: - resolution: {integrity: sha512-4C7Bh5pyHTvU33KpZgwrNKh/VQnvgtCSqPRfJAUdmrtSYePVzVg4E4OzsrbkhJj9O7SO6Bnv75K/F8XVZT8YHA==} + /superagent@8.1.2: + resolution: {integrity: sha512-6WTxW1EB6yCxV5VFOIPQruWGHqc3yI7hEmZK6h+pyk69Lk/Ut7rLUY6W/ONF2MjBuGjvmMiIpsrVJ2vjrHlslA==} engines: {node: '>=6.4.0 <13 || >=14'} + deprecated: Please upgrade to v9.0.0+ as we have fixed a public vulnerability with formidable dependency. Note that v9.0.0+ requires Node.js v14.18.0+. See https://github.com/ladjs/superagent/pull/1800 for insight. This project is supported and maintained by the team at Forward Email @ https://forwardemail.net dependencies: - component-emitter: 1.3.0 + component-emitter: 1.3.1 cookiejar: 2.1.4 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.5 fast-safe-stringify: 2.1.1 form-data: 4.0.0 formidable: 2.1.2 methods: 1.1.2 mime: 2.6.0 - qs: 6.11.2 - semver: 7.5.4 + qs: 6.12.1 + semver: 7.6.2 transitivePeerDependencies: - supports-color dev: true @@ -4665,7 +4970,7 @@ packages: engines: {node: '>=8'} dependencies: '@istanbuljs/schema': 0.1.3 - glob: 7.2.0 + glob: 7.2.3 minimatch: 3.1.2 dev: true @@ -4698,11 +5003,9 @@ packages: next-tick: 1.1.0 dev: false - /tmp@0.2.1: - resolution: {integrity: sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==} - engines: {node: '>=8.17.0'} - dependencies: - rimraf: 3.0.2 + /tmp@0.2.3: + resolution: {integrity: sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w==} + engines: {node: '>=14.14'} dev: true /to-buffer@1.1.1: @@ -4769,8 +5072,13 @@ packages: resolution: {integrity: sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw==} dev: false - /ts-node@10.9.1(@types/node@20.4.1)(typescript@5.5.2): - resolution: {integrity: sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==} + /triple-beam@1.4.1: + resolution: {integrity: sha512-aZbgViZrg1QNcG+LULa7nhZpJTZSLm/mXnHXnbAbjmN5aSa0y7V+wvv6+4WaBtpISJzThKy+PIPxc1Nq1EJ9mg==} + engines: {node: '>= 14.0.0'} + dev: false + + /ts-node@10.9.2(@types/node@20.14.7)(typescript@5.5.2): + resolution: {integrity: sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==} hasBin: true peerDependencies: '@swc/core': '>=1.2.50' @@ -4784,13 +5092,13 @@ packages: optional: true dependencies: '@cspotcode/source-map-support': 0.8.1 - '@tsconfig/node10': 1.0.9 + '@tsconfig/node10': 1.0.11 '@tsconfig/node12': 1.0.11 '@tsconfig/node14': 1.0.3 '@tsconfig/node16': 1.0.4 - '@types/node': 20.4.1 - acorn: 8.10.0 - acorn-walk: 8.2.0 + '@types/node': 20.14.7 + acorn: 8.12.0 + acorn-walk: 8.3.3 arg: 4.1.3 create-require: 1.1.1 diff: 4.0.2 @@ -4798,9 +5106,9 @@ packages: typescript: 5.5.2 v8-compile-cache-lib: 3.0.1 yn: 3.1.1 - dev: false + dev: true - /ts-node@9.1.1(typescript@5.1.6): + /ts-node@9.1.1(typescript@5.5.2): resolution: {integrity: sha512-hPlt7ZACERQGf03M253ytLY3dHbGNGrAq9qIHWUY9XHYl1z7wYngSr3OQ5xmui8o2AaxsONxIzjafLUiWBo1Fg==} engines: {node: '>=10.0.0'} hasBin: true @@ -4812,7 +5120,7 @@ packages: diff: 4.0.2 make-error: 1.3.6 source-map-support: 0.5.21 - typescript: 5.1.6 + typescript: 5.5.2 yn: 3.1.1 dev: true @@ -4829,15 +5137,11 @@ packages: resolution: {integrity: sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==} dev: true - /tslib@2.6.0: - resolution: {integrity: sha512-7At1WUettjcSRHXCyYtTselblcHl9PJFFVKiCAy/bY97+BPZXSQ2wbq0P9s8tK2G7dFQfNnlJnPAiArVBVBsfA==} + /tslib@2.6.3: + resolution: {integrity: sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==} dev: true - /tslib@2.6.2: - resolution: {integrity: sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==} - dev: true - - /tslint@6.1.3(typescript@5.1.6): + /tslint@6.1.3(typescript@5.5.2): resolution: {integrity: sha512-IbR4nkT96EQOvKE2PW/djGz8iGNeJ4rF2mBfiYaR/nvUWYKJhLwimoJKgjIFEIDibBtOevj7BqCRL4oHeWWUCg==} engines: {node: '>=4.8.0'} deprecated: TSLint has been deprecated in favor of ESLint. Please see https://github.com/palantir/tslint/issues/4534 for more information. @@ -4845,29 +5149,29 @@ packages: peerDependencies: typescript: '>=2.3.0-dev || >=2.4.0-dev || >=2.5.0-dev || >=2.6.0-dev || >=2.7.0-dev || >=2.8.0-dev || >=2.9.0-dev || >=3.0.0-dev || >= 3.1.0-dev || >= 3.2.0-dev || >= 4.0.0-dev' dependencies: - '@babel/code-frame': 7.22.5 + '@babel/code-frame': 7.24.7 builtin-modules: 1.1.1 chalk: 2.4.2 commander: 2.20.3 diff: 4.0.2 - glob: 7.2.0 + glob: 7.2.3 js-yaml: 3.14.1 minimatch: 3.1.2 mkdirp: 0.5.6 - resolve: 1.22.2 - semver: 5.7.1 + resolve: 1.22.8 + semver: 5.7.2 tslib: 1.14.1 - tsutils: 2.29.0(typescript@5.1.6) - typescript: 5.1.6 + tsutils: 2.29.0(typescript@5.5.2) + typescript: 5.5.2 dev: true - /tsutils@2.29.0(typescript@5.1.6): + /tsutils@2.29.0(typescript@5.5.2): resolution: {integrity: sha512-g5JVHCIJwzfISaXpXE1qvNalca5Jwob6FjI4AoPlqMusJ6ftFE7IkkFoMhVLRgK+4Kx3gkzb8UZK5t5yTTvEmA==} peerDependencies: typescript: '>=2.1.0 || >=2.1.0-dev || >=2.2.0-dev || >=2.3.0-dev || >=2.4.0-dev || >=2.5.0-dev || >=2.6.0-dev || >=2.7.0-dev || >=2.8.0-dev || >=2.9.0-dev || >= 3.0.0-dev || >= 3.1.0-dev' dependencies: tslib: 1.14.1 - typescript: 5.1.6 + typescript: 5.5.2 dev: true /tunnel-agent@0.6.0: @@ -4933,7 +5237,7 @@ packages: lunr: 2.3.9 dev: true - /typedoc@0.17.8(typescript@5.1.6): + /typedoc@0.17.8(typescript@5.5.2): resolution: {integrity: sha512-/OyrHCJ8jtzu+QZ+771YaxQ9s4g5Z3XsQE3Ma7q+BL392xxBn4UMvvCdVnqKC2T/dz03/VXSLVKOP3lHmDdc/w==} engines: {node: '>= 8.0.0'} hasBin: true @@ -4950,22 +5254,17 @@ packages: progress: 2.0.3 shelljs: 0.8.5 typedoc-default-themes: 0.10.2 - typescript: 5.1.6 - dev: true - - /typescript@5.1.6: - resolution: {integrity: sha512-zaWCozRZ6DLEWAWFrVDz1H6FVXzUSfTy5FUMWsQlU8Ym5JP9eO4xkTIROFCQvhQf61z6O/G6ugw3SgAnvvm+HA==} - engines: {node: '>=14.17'} - hasBin: true + typescript: 5.5.2 dev: true /typescript@5.5.2: resolution: {integrity: sha512-NcRtPEOsPFFWjobJEtfihkLCZCXZt/os3zf8nTxjVH3RvTSxjrCamJpbExGvYOF+tFHc3pA65qpdwPbzjohhew==} engines: {node: '>=14.17'} hasBin: true + dev: true - /uglify-js@3.17.4: - resolution: {integrity: sha512-T9q82TJI9e/C1TAxYvfb16xO120tMVFZrGA3f9/P4424DNu6ypK103y0GPFVa17yotwSyZW5iYXgjYHkGrJW/g==} + /uglify-js@3.18.0: + resolution: {integrity: sha512-SyVVbcNBCk0dzr9XL/R/ySrmYf0s372K6/hFklzgcp2lBFyXtw4I7BOdDjlLhE1aVqaI/SHWXWmYdlZxuyF38A==} engines: {node: '>=0.8.0'} hasBin: true requiresBuild: true @@ -4976,13 +5275,16 @@ packages: resolution: {integrity: sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==} dev: true + /undici-types@5.26.5: + resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} + /universalify@0.1.2: resolution: {integrity: sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==} engines: {node: '>= 4.0.0'} dev: true - /universalify@2.0.0: - resolution: {integrity: sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==} + /universalify@2.0.1: + resolution: {integrity: sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==} engines: {node: '>= 10.0.0'} dev: true @@ -4991,15 +5293,15 @@ packages: engines: {node: '>= 0.8'} dev: false - /update-browserslist-db@1.0.11(browserslist@4.21.9): - resolution: {integrity: sha512-dCwEFf0/oT85M1fHBg4F0jtLwJrutGoHSQXCh7u4o2t1drG+c0a9Flnqww6XUKSfQMPpJBRjU8d4RXB09qtvaA==} + /update-browserslist-db@1.0.16(browserslist@4.23.1): + resolution: {integrity: sha512-KVbTxlBYlckhF5wgfyZXTWnMn7MMZjMu9XG8bPlliUOP9ThaF4QnhP8qrjrH7DRzHfSk0oQv1wToW+iA5GajEQ==} hasBin: true peerDependencies: browserslist: '>= 4.21.0' dependencies: - browserslist: 4.21.9 - escalade: 3.1.1 - picocolors: 1.0.0 + browserslist: 4.23.1 + escalade: 3.1.2 + picocolors: 1.0.1 dev: true /uri-js@4.4.1: @@ -5029,7 +5331,7 @@ packages: /v8-compile-cache-lib@3.0.1: resolution: {integrity: sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==} - dev: false + dev: true /v8-compile-cache@2.3.0: resolution: {integrity: sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==} @@ -5114,6 +5416,32 @@ packages: triple-beam: 1.3.0 dev: false + /winston-transport@4.7.0: + resolution: {integrity: sha512-ajBj65K5I7denzer2IYW6+2bNIVqLGDHqDw3Ow8Ohh+vdW+rv4MZ6eiDvHoKhfJFZ2auyN8byXieDDJ96ViONg==} + engines: {node: '>= 12.0.0'} + dependencies: + logform: 2.6.0 + readable-stream: 3.6.2 + triple-beam: 1.4.1 + dev: false + + /winston@3.13.0: + resolution: {integrity: sha512-rwidmA1w3SE4j0E5MuIufFhyJPBDG7Nu71RkZor1p2+qHvJSZ9GYDA81AyleQcZbh/+V6HjeBdfnTZJm9rSeQQ==} + engines: {node: '>= 12.0.0'} + dependencies: + '@colors/colors': 1.6.0 + '@dabh/diagnostics': 2.0.3 + async: 3.2.5 + is-stream: 2.0.1 + logform: 2.6.0 + one-time: 1.0.0 + readable-stream: 3.6.2 + safe-stable-stringify: 2.4.3 + stack-trace: 0.0.10 + triple-beam: 1.4.1 + winston-transport: 4.7.0 + dev: false + /winston@3.9.0: resolution: {integrity: sha512-jW51iW/X95BCW6MMtZWr2jKQBP4hV5bIDq9QrIjfDk6Q9QuxvTKEAlpUNAzP+HYHFFCeENhph16s0zEunu4uuQ==} engines: {node: '>= 12.0.0'} @@ -5170,6 +5498,15 @@ packages: strip-ansi: 6.0.1 dev: true + /wrap-ansi@8.1.0: + resolution: {integrity: sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==} + engines: {node: '>=12'} + dependencies: + ansi-styles: 6.2.1 + string-width: 5.1.2 + strip-ansi: 7.1.0 + dev: true + /wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} dev: true @@ -5277,7 +5614,7 @@ packages: engines: {node: '>=10'} dependencies: cliui: 7.0.4 - escalade: 3.1.1 + escalade: 3.1.2 get-caller-file: 2.0.5 require-directory: 2.1.1 string-width: 4.2.3 @@ -5290,7 +5627,7 @@ packages: engines: {node: '>=12'} dependencies: cliui: 8.0.1 - escalade: 3.1.1 + escalade: 3.1.2 get-caller-file: 2.0.5 require-directory: 2.1.1 string-width: 4.2.3 @@ -5301,6 +5638,7 @@ packages: /yn@3.1.1: resolution: {integrity: sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==} engines: {node: '>=6'} + dev: true /yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} From 1dee40a8b479d87e910f68ec7375ce0bfb26028b Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Sat, 27 Jul 2024 15:21:16 -0400 Subject: [PATCH 21/27] Refactor Validation code in preparation for Conditional Restrictions (#215) * Move restrictions and data value types to dedicated files * Add test folders to TS path but exclude from build * Undefined checks where unchecked index access was done * Rename rangeToSymbol as rangeToText, change text priority to describe more restrictive conditions * Range checking utiltiy function with tests It should be mentioned here that the previously committed version of this had a mix up with the exclusive conditions, where the edge cases (value equal to the edge of the range) was incorrectly valdiated in the version imported from js-lectern-client. This is fixed now with tests to check these cases. * Generalize isArray type predicates, move isDefined to common * Add test folders to TS path but exclude from build * FieldRestriction type map and more granular codeList types * Move shared type definition to commonTypes * Use updated restriciton and name types in dictionary defs * Declare separate types for singular and array data record values * Code autoformatting * First pass types for reporting field resriction validation errors * Utility to convert a restriction test on a single value to apply to a whole array * Rewrite restriction validation functions to apply to single fields instead of data records * Simplify type selection * Fix edge case tests and work for isWithinRange * Documentation on process for refactored fieldValidation process * Fix generate script organization within monorepo * Refactor fieldValidation to handle each field individually Original implementation had record processing logic built into the restriction test logic. This refactor sepearates restriction tests to apply to a specific value only, based on the defined restriction rule. This will enable future development to resolve the restrictions on each field based on conditional restrictions and then test based on the resolved conditions. The next step is to add validation testing for full data records, full entity data sets, and then full dictionary data sets. These each loop over the previous element but also introduce new tests. For example, entity data sets enforce unique field and uniqueKey restrictions. Dictionary datasets enforce foreignKey restrictions. Unit tests have been added to fieldValidation function, and to the individual restriction tests. The restriction tests check the detailed set of values vs restriction rules, while the fieldValidation testing is checking that these restrictions are enforced based on schemaField definitions. Note that script validations are not being run in this code, and that the unique constraint cannot be applied to a field. There is additional changes required to move the unique constraint to a field level property (like isArray) so that it can't be affected by conditional restrictions. It will be validated in the Schema level validation (entity dataset). * WIP Documentation for Important Concepts * Move field validation code to `/validateField` * Add validateRecord functionality with basic documentation * Move validateField tests to directory matching source directory name * Tests for validateRecord * Editing TSDoc for validation type utils * Complete pnpm monorepo registration for scripts module * Infer predicates instead of declaring where possible * Create function `validateSchema` to validate a list of records with single schema Applies the `validateRecord` validation to every record, plus runs validations that require knowledge of the entire set of records for this schema: - unique - uniqueKey * Add copyright declaration to top of files * Clean regex test error message by removing the rule * Update test description text for rangeTest to match other tests * Test for validateSchema valid and unique restriction - fixes bug in testUniqueFieldRestriction that was using the wrong field name when generating the hash - adds DataSetHashMap type alias that was missing in previous commit - adds test utility to validate that test fixtures are valid vs the Lectern zod schemas * Add tests for uniqueKey restriction in validateSchema * Add tests for validateSchema uniqueKey with undefined values * Add tests for validateSchema to ensure it tests record and field restrictions * Update uniqueKey invalid test to check error detailed properties * Organize imports * Add `validateDictionary` function plus related tests and fixtures - applies foreign key restrictions - detects unrecognized schemas - collects foreign key errors with other record errors * Standardize test fixture validation message text * Remove range rule from test failure message * Add period to required test failure message * Move `resolveFieldRestrictions` function to its own file * Rename foreign key type and schema to `ForeignKeyRestriction` * Rename testField/restriction specs to match source file names * Remove empty spec files * Add copyright text to test fixtures * Result type can have Failure data, remove `Either` type * Add convertValue functionality - functions for converting values for field, record, schema, and dictionary - update to validation error type names to be shared with convert value types - includes test specs for each convert function exported * Fix simple typos * Update client processing functions to use new validation functions * Remove validation tests from client that are now done in validation package * Unique restriction rules enforced for array fields * Remove deprecated types and test implementations from legacy client * Remove old data conversion implementation from client processing * Remove old data valdiation pipeline from client processing * Remove tests around non-standard default value setting * Test fixtures for unique restrictions with arrays * Update property names in RecordError and TestResult types (#216) * Change `FieldDetails` property `value` to `fieldValue` * Change TestResult invalid property to be `details` * Rename convert functions as parse (#219) --- apps/server/package.json | 2 +- apps/server/tsconfig.build.json | 4 + apps/server/tsconfig.json | 3 +- docs/dictionary-reference.md | 266 ++++++ docs/important-concepts.md | 83 ++ docs/validation/field-validation.md | 53 ++ docs/validation/index.md | 18 + docs/validation/record-validation.md | 10 + nx.json | 2 +- package.json | 2 +- packages/client/package.json | 2 +- .../src/processing/convertDataValueTypes.ts | 122 --- packages/client/src/processing/index.ts | 278 ++---- .../src/processing/processingResultTypes.ts | 67 +- .../src/processing/validationPipelines.ts | 78 -- packages/client/test/processing.spec.ts | 91 -- packages/client/test/validation.spec.ts | 827 ------------------ packages/client/tsconfig.build.json | 4 + packages/client/tsconfig.json | 5 +- packages/common/src/types/defined.ts | 8 + packages/common/src/types/index.ts | 1 + packages/common/src/types/result.ts | 17 +- packages/common/src/types/singular.ts | 11 +- packages/common/src/utils/typeUtils.ts | 13 + packages/dictionary/src/constants.ts | 20 + packages/dictionary/src/index.ts | 1 + packages/dictionary/src/types/commonTypes.ts | 18 + packages/dictionary/src/types/dataTypes.ts | 11 +- packages/dictionary/src/types/dbTypes.ts | 19 + .../dictionary/src/types/dictionaryTypes.ts | 110 +-- packages/dictionary/src/types/index.ts | 2 + .../dictionary/src/types/restrictionsTypes.ts | 94 ++ packages/validation/package.json | 2 +- .../valueTypeValidation.ts | 122 --- .../fieldRestrictions/codeListValidation.ts | 89 -- .../src/fieldRestrictions/rangeValidation.ts | 85 -- .../src/fieldRestrictions/regexValidation.ts | 84 -- .../fieldRestrictions/requiredValidation.ts | 77 -- .../src/fieldRestrictions/scriptValidation.ts | 133 --- packages/validation/src/index.ts | 15 +- .../src/parseValues/ParseValuesResult.ts | 47 + packages/validation/src/parseValues/index.ts | 21 + .../parseValues/matchCodeListFormatting.ts | 42 + .../validation/src/parseValues/parseValues.ts | 283 ++++++ .../foreignKeysValidation.ts | 127 --- .../schemaRestrictions/uniqueKeyValidation.ts | 69 -- .../schemaRestrictions/uniqueValidation.ts | 63 -- packages/validation/src/types/index.ts | 3 +- packages/validation/src/types/testResult.ts | 58 ++ .../src/types/validationErrorTypes.ts | 109 --- .../validation/src/utils/hashDataRecord.ts | 41 + .../validation/src/utils/isValidValueType.ts | 38 + .../validation/src/utils/isWithinRange.ts | 36 + .../{rangeToSymbol.ts => rangeToText.ts} | 17 +- packages/validation/src/utils/typeUtils.ts | 73 +- .../DictionaryValidationError.ts} | 48 +- .../collectSchemaReferenceData.ts | 110 +++ .../src/validateDictionary/index.ts | 21 + .../testForeignKeyRestriction.ts | 63 ++ .../testUnrecognizedSchema.ts | 35 + .../validateDictionary/validateDictionary.ts | 111 +++ .../src/validateField/FieldRestrictionRule.ts | 62 ++ .../FieldRestrictionTest.ts} | 46 +- .../src/validateField/FieldValidationError.ts | 43 + .../validation/src/validateField/index.ts | 24 + .../validateField/resolveFieldRestrictions.ts | 98 +++ .../createFieldRestrictionTestForArrays.ts | 51 ++ .../src/validateField/restrictions/index.ts | 23 + .../restrictions/testCodeList.ts | 50 ++ .../validateField/restrictions/testRange.ts | 44 + .../validateField/restrictions/testRegex.ts | 51 ++ .../restrictions/testRequired.ts | 82 ++ .../src/validateField/validateField.ts | 121 +++ .../validateRecord/RecordValidationError.ts | 40 + .../validation/src/validateRecord/index.ts | 21 + .../src/validateRecord/validateRecord.ts | 80 ++ .../validateSchema/SchemaValidationError.ts | 44 + .../validation/src/validateSchema/index.ts | 21 + .../restrictions/generateDataSetHashMap.ts | 62 ++ .../uniqueField/testUniqueFieldRestriction.ts | 57 ++ .../uniqueKey/getUniqueKeyValues.ts | 31 + .../restrictions/uniqueKey/testUniqueKey.ts | 64 ++ .../src/validateSchema/validateSchema.ts | 87 ++ .../dictionaries/dictionaryFourSchemas.ts | 33 + ...dictionaryMultipleSchemasNoRestrictions.ts | 35 + .../dictionarySingleSchemaNoRestrictions.ts | 15 + ...tionarySingleSchemaRequiredRestrictions.ts | 15 + ...tionarySingleSchemaUniqueKeyRestriction.ts | 15 + .../dictionaryForeignKeyMultiple.ts | 40 + .../foreignKey/dictionaryForeignKeySimple.ts | 28 + .../fieldStringManyRestrictions.ts | 17 + .../fieldBooleanNoRestriction.ts | 7 + .../fieldIntegerNoRestriction.ts | 6 + .../fieldNumberNoRestriction.ts | 6 + .../fieldStringArrayNoRestriction.ts | 7 + .../fieldStringNoRestriction.ts | 6 + .../schemaRestrictions/fieldStringUnique.ts | 7 + .../fieldStringUniqueArray.ts | 8 + .../boolean/fieldBooleanArrayRequired.ts | 29 + .../boolean/fieldBooleanRequired.ts | 9 + .../integer/fieldIntegerArrayRequired.ts | 29 + .../integer/fieldIntegerRequired.ts | 9 + .../number/fieldNumberArrayCodeList.ts | 12 + .../number/fieldNumberRange.ts | 11 + .../number/fieldNumberRequired.ts | 9 + .../string/fieldStringArrayCodeList.ts | 31 + .../string/fieldStringArrayRequired.ts | 11 + .../string/fieldStringCodeList.ts | 11 + .../string/fieldStringRegex.ts | 14 + .../string/fieldStringRequired.ts | 10 + .../restrictions/codeListsFixtures.ts | 5 + .../fixtures/restrictions/rangeFixtures.ts | 6 + .../fixtures/restrictions/regexFixtures.ts | 5 + .../schema/schemaAllDataMixedRestrictions.ts | 17 + .../fixtures/schema/schemaAllDataTypes.ts | 14 + .../schema/schemaAllDataTypesRequired.ts | 14 + .../fixtures/schema/schemaSingleString.ts | 11 + .../schema/schemaSingleStringRequired.ts | 11 + .../test/fixtures/schema/schemaUniqueKey.ts | 27 + .../schema/schemaUniqueKeyWithArray.ts | 27 + .../fixtures/schema/schemaUniqueString.ts | 12 + .../schema/schemaUniqueStringArray.ts | 13 + .../test/parseValues/parseDictionary.spec.ts | 132 +++ .../test/parseValues/parseField.spec.ts | 304 +++++++ .../test/parseValues/parseRecord.spec.ts | 109 +++ .../test/parseValues/parseSchema.spec.ts | 119 +++ packages/validation/test/placeholder.spec.ts | 3 - .../test/testUtils/expectAllValuesMatch.ts | 25 + .../test/testUtils/validateFixture.ts | 34 + .../test/utils/hashDataRecord.spec.ts | 70 ++ .../validation/test/utils/rangeTest.spec.ts | 119 +++ .../validateDictionary.spec.ts | 395 +++++++++ .../restrictions/testCodeList.spec.ts | 126 +++ .../restrictions/testRange.spec.ts | 0 .../restrictions/testRegex.spec.ts | 89 ++ .../restrictions/testRequired.spec.ts | 73 ++ .../test/validateField/validateField.spec.ts | 399 +++++++++ .../validateRecord/validateRecord.spec.ts | 204 +++++ .../generateDataSetHashMap.spec.ts | 123 +++ .../validateSchema/validateSchema.spec.ts | 378 ++++++++ packages/validation/tsconfig.build.json | 4 + packages/validation/tsconfig.json | 3 +- pnpm-lock.yaml | 10 +- pnpm-workspace.yaml | 3 +- scripts/{ => generate}/package.json | 1 + .../{ => generate}/src/generateMetaSchema.ts | 0 146 files changed, 6303 insertions(+), 2478 deletions(-) create mode 100644 apps/server/tsconfig.build.json create mode 100644 docs/dictionary-reference.md create mode 100644 docs/important-concepts.md create mode 100644 docs/validation/field-validation.md create mode 100644 docs/validation/index.md create mode 100644 docs/validation/record-validation.md delete mode 100644 packages/client/src/processing/convertDataValueTypes.ts delete mode 100644 packages/client/src/processing/validationPipelines.ts delete mode 100644 packages/client/test/processing.spec.ts delete mode 100644 packages/client/test/validation.spec.ts create mode 100644 packages/client/tsconfig.build.json create mode 100644 packages/common/src/types/defined.ts create mode 100644 packages/dictionary/src/constants.ts create mode 100644 packages/dictionary/src/types/commonTypes.ts create mode 100644 packages/dictionary/src/types/restrictionsTypes.ts delete mode 100644 packages/validation/src/dataRecordValidation/valueTypeValidation.ts delete mode 100644 packages/validation/src/fieldRestrictions/codeListValidation.ts delete mode 100644 packages/validation/src/fieldRestrictions/rangeValidation.ts delete mode 100644 packages/validation/src/fieldRestrictions/regexValidation.ts delete mode 100644 packages/validation/src/fieldRestrictions/requiredValidation.ts delete mode 100644 packages/validation/src/fieldRestrictions/scriptValidation.ts create mode 100644 packages/validation/src/parseValues/ParseValuesResult.ts create mode 100644 packages/validation/src/parseValues/index.ts create mode 100644 packages/validation/src/parseValues/matchCodeListFormatting.ts create mode 100644 packages/validation/src/parseValues/parseValues.ts delete mode 100644 packages/validation/src/schemaRestrictions/foreignKeysValidation.ts delete mode 100644 packages/validation/src/schemaRestrictions/uniqueKeyValidation.ts delete mode 100644 packages/validation/src/schemaRestrictions/uniqueValidation.ts create mode 100644 packages/validation/src/types/testResult.ts delete mode 100644 packages/validation/src/types/validationErrorTypes.ts create mode 100644 packages/validation/src/utils/hashDataRecord.ts create mode 100644 packages/validation/src/utils/isValidValueType.ts create mode 100644 packages/validation/src/utils/isWithinRange.ts rename packages/validation/src/utils/{rangeToSymbol.ts => rangeToText.ts} (85%) rename packages/validation/src/{types/validationFunctionTypes.ts => validateDictionary/DictionaryValidationError.ts} (52%) create mode 100644 packages/validation/src/validateDictionary/collectSchemaReferenceData.ts create mode 100644 packages/validation/src/validateDictionary/index.ts create mode 100644 packages/validation/src/validateDictionary/testForeignKeyRestriction.ts create mode 100644 packages/validation/src/validateDictionary/testUnrecognizedSchema.ts create mode 100644 packages/validation/src/validateDictionary/validateDictionary.ts create mode 100644 packages/validation/src/validateField/FieldRestrictionRule.ts rename packages/validation/src/{dataRecordValidation/fieldNamesValidation.ts => validateField/FieldRestrictionTest.ts} (55%) create mode 100644 packages/validation/src/validateField/FieldValidationError.ts create mode 100644 packages/validation/src/validateField/index.ts create mode 100644 packages/validation/src/validateField/resolveFieldRestrictions.ts create mode 100644 packages/validation/src/validateField/restrictions/createFieldRestrictionTestForArrays.ts create mode 100644 packages/validation/src/validateField/restrictions/index.ts create mode 100644 packages/validation/src/validateField/restrictions/testCodeList.ts create mode 100644 packages/validation/src/validateField/restrictions/testRange.ts create mode 100644 packages/validation/src/validateField/restrictions/testRegex.ts create mode 100644 packages/validation/src/validateField/restrictions/testRequired.ts create mode 100644 packages/validation/src/validateField/validateField.ts create mode 100644 packages/validation/src/validateRecord/RecordValidationError.ts create mode 100644 packages/validation/src/validateRecord/index.ts create mode 100644 packages/validation/src/validateRecord/validateRecord.ts create mode 100644 packages/validation/src/validateSchema/SchemaValidationError.ts create mode 100644 packages/validation/src/validateSchema/index.ts create mode 100644 packages/validation/src/validateSchema/restrictions/generateDataSetHashMap.ts create mode 100644 packages/validation/src/validateSchema/restrictions/uniqueField/testUniqueFieldRestriction.ts create mode 100644 packages/validation/src/validateSchema/restrictions/uniqueKey/getUniqueKeyValues.ts create mode 100644 packages/validation/src/validateSchema/restrictions/uniqueKey/testUniqueKey.ts create mode 100644 packages/validation/src/validateSchema/validateSchema.ts create mode 100644 packages/validation/test/fixtures/dictionaries/dictionaryFourSchemas.ts create mode 100644 packages/validation/test/fixtures/dictionaries/dictionaryMultipleSchemasNoRestrictions.ts create mode 100644 packages/validation/test/fixtures/dictionaries/dictionarySingleSchemaNoRestrictions.ts create mode 100644 packages/validation/test/fixtures/dictionaries/dictionarySingleSchemaRequiredRestrictions.ts create mode 100644 packages/validation/test/fixtures/dictionaries/dictionarySingleSchemaUniqueKeyRestriction.ts create mode 100644 packages/validation/test/fixtures/dictionaries/foreignKey/dictionaryForeignKeyMultiple.ts create mode 100644 packages/validation/test/fixtures/dictionaries/foreignKey/dictionaryForeignKeySimple.ts create mode 100644 packages/validation/test/fixtures/fields/multipleRestrictions/fieldStringManyRestrictions.ts create mode 100644 packages/validation/test/fixtures/fields/noRestrictions/fieldBooleanNoRestriction.ts create mode 100644 packages/validation/test/fixtures/fields/noRestrictions/fieldIntegerNoRestriction.ts create mode 100644 packages/validation/test/fixtures/fields/noRestrictions/fieldNumberNoRestriction.ts create mode 100644 packages/validation/test/fixtures/fields/noRestrictions/fieldStringArrayNoRestriction.ts create mode 100644 packages/validation/test/fixtures/fields/noRestrictions/fieldStringNoRestriction.ts create mode 100644 packages/validation/test/fixtures/fields/schemaRestrictions/fieldStringUnique.ts create mode 100644 packages/validation/test/fixtures/fields/schemaRestrictions/fieldStringUniqueArray.ts create mode 100644 packages/validation/test/fixtures/fields/simpleRestrictions/boolean/fieldBooleanArrayRequired.ts create mode 100644 packages/validation/test/fixtures/fields/simpleRestrictions/boolean/fieldBooleanRequired.ts create mode 100644 packages/validation/test/fixtures/fields/simpleRestrictions/integer/fieldIntegerArrayRequired.ts create mode 100644 packages/validation/test/fixtures/fields/simpleRestrictions/integer/fieldIntegerRequired.ts create mode 100644 packages/validation/test/fixtures/fields/simpleRestrictions/number/fieldNumberArrayCodeList.ts create mode 100644 packages/validation/test/fixtures/fields/simpleRestrictions/number/fieldNumberRange.ts create mode 100644 packages/validation/test/fixtures/fields/simpleRestrictions/number/fieldNumberRequired.ts create mode 100644 packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringArrayCodeList.ts create mode 100644 packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringArrayRequired.ts create mode 100644 packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringCodeList.ts create mode 100644 packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringRegex.ts create mode 100644 packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringRequired.ts create mode 100644 packages/validation/test/fixtures/restrictions/codeListsFixtures.ts create mode 100644 packages/validation/test/fixtures/restrictions/rangeFixtures.ts create mode 100644 packages/validation/test/fixtures/restrictions/regexFixtures.ts create mode 100644 packages/validation/test/fixtures/schema/schemaAllDataMixedRestrictions.ts create mode 100644 packages/validation/test/fixtures/schema/schemaAllDataTypes.ts create mode 100644 packages/validation/test/fixtures/schema/schemaAllDataTypesRequired.ts create mode 100644 packages/validation/test/fixtures/schema/schemaSingleString.ts create mode 100644 packages/validation/test/fixtures/schema/schemaSingleStringRequired.ts create mode 100644 packages/validation/test/fixtures/schema/schemaUniqueKey.ts create mode 100644 packages/validation/test/fixtures/schema/schemaUniqueKeyWithArray.ts create mode 100644 packages/validation/test/fixtures/schema/schemaUniqueString.ts create mode 100644 packages/validation/test/fixtures/schema/schemaUniqueStringArray.ts create mode 100644 packages/validation/test/parseValues/parseDictionary.spec.ts create mode 100644 packages/validation/test/parseValues/parseField.spec.ts create mode 100644 packages/validation/test/parseValues/parseRecord.spec.ts create mode 100644 packages/validation/test/parseValues/parseSchema.spec.ts delete mode 100644 packages/validation/test/placeholder.spec.ts create mode 100644 packages/validation/test/testUtils/expectAllValuesMatch.ts create mode 100644 packages/validation/test/testUtils/validateFixture.ts create mode 100644 packages/validation/test/utils/hashDataRecord.spec.ts create mode 100644 packages/validation/test/utils/rangeTest.spec.ts create mode 100644 packages/validation/test/validateDictionary/validateDictionary.spec.ts create mode 100644 packages/validation/test/validateField/restrictions/testCodeList.spec.ts create mode 100644 packages/validation/test/validateField/restrictions/testRange.spec.ts create mode 100644 packages/validation/test/validateField/restrictions/testRegex.spec.ts create mode 100644 packages/validation/test/validateField/restrictions/testRequired.spec.ts create mode 100644 packages/validation/test/validateField/validateField.spec.ts create mode 100644 packages/validation/test/validateRecord/validateRecord.spec.ts create mode 100644 packages/validation/test/validateSchema/restrictions/generateDataSetHashMap.spec.ts create mode 100644 packages/validation/test/validateSchema/validateSchema.spec.ts create mode 100644 packages/validation/tsconfig.build.json rename scripts/{ => generate}/package.json (95%) rename scripts/{ => generate}/src/generateMetaSchema.ts (100%) diff --git a/apps/server/package.json b/apps/server/package.json index faa69a8..5d87de4 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -3,7 +3,7 @@ "version": "2.0.0-next.0", "description": "Overture Data Dictionary Management", "scripts": { - "build": "tsc", + "build": "tsc -p tsconfig.build.json", "debug": "nodemon --watch 'src/**/*.ts' --ignore 'src/**/*.spec.ts' --exec node --inspect -r ts-node/register ./src/server.ts", "start": "NODE_ENV=production ts-node -T src/server.ts", "test": "nyc mocha" diff --git a/apps/server/tsconfig.build.json b/apps/server/tsconfig.build.json new file mode 100644 index 0000000..df0d74a --- /dev/null +++ b/apps/server/tsconfig.build.json @@ -0,0 +1,4 @@ +{ + "extends": "./tsconfig.json", + "exclude": ["test/**/*.ts"] +} diff --git a/apps/server/tsconfig.json b/apps/server/tsconfig.json index a0432e8..57c9c2b 100644 --- a/apps/server/tsconfig.json +++ b/apps/server/tsconfig.json @@ -11,8 +11,7 @@ "esModuleInterop": true, "noEmit": true, "outDir": "dist", - "baseUrl": "./src", "skipLibCheck": true }, - "include": ["./src/**/*.ts"] + "include": ["./src/**/*.ts", "./test/**/*.ts"] } diff --git a/docs/dictionary-reference.md b/docs/dictionary-reference.md new file mode 100644 index 0000000..64bf017 --- /dev/null +++ b/docs/dictionary-reference.md @@ -0,0 +1,266 @@ +# Lectern Dictionary Meta-Schema Refernce + +For a high level description of the component parts of a Lectern Dictionary see [Important Concepts - Dictionary Model](./important-concepts.md#dictionary-model). + +## Dictionary Structure + +A Lectern Dictionary is a collection of Lectern Schemas. Each schema describes the structure of a TSV file, providing a list of the columns for that file and the data types and restrictions on the content of those columns. + +In addition to schemas, a Lectern Dictionary can contain reference values that can be reused throughout the schema definitions to define property restrictions with shared rules. + +> **Dictionary Structure Example** +> ```json +> { +> "name": "example_dictionary", +> "description": "Collection of schemas to demonstrate Lectern functionality", +> "meta": { /* Custom meta data about the dictionary here */ }, +> +> "version": "1.0", +> +> "schemas": [ /* Schemas Here */ ], +> "references": { /* Reference Variables Here */ } +> } +> ``` + +| Property | Type | Required | Description | Example | +| ------------- | -------------------------------------------------------------- | -------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------- | +| `name` | `string` | Required | Display name of the dictionary | `"Example Lectern Dictionary"` | +| `version` | `string`, as a semantic version number `major`.`minor`.`patch` | Required | Version of the dictionary. | `"1.23.4"` | +| `schemas` | `Array<`[`LecternSchema`](#dictionary-schema-structure)`>` | Required | An array containing Lectern Schemas. Minimum of 1 Schema is required. | [Dictionary Schema Structure](#dictionary-schema-structure) | +| `description` | `string` | Optional | Free text description of the schema, for use as a reference for users of the dictionary. This description is not used by Lcetern for dictionary validation. | `"Collection of schemas to demonstrate Lectern functionality"` | +| `meta` | [MetaData](#meta-data-structure) object | Optional | Schema implementor defined fields to capture any additional properties not defined in standard Lectern dictionaries. These properties are not used by Lctern for dictionary validation | `{ "author": "Guy Incognito" }` | +| `references` | [References](#references-structure) object | Optional | Reference values that can be referenced throughout the dictionary. | `{ "customRegex": { "ncitIds": "^NCIT:C\d+$" } }` | + +### Dictionary Schema Structure +> **Dictionary Schema Example** +> ```json +> { +> "name": "example-schema", +> "description": "Demonstrating structure of Lectern Schema", +> "meta": { /* Custom meta data about the schema here */ }, +> +> "fields": [ /* Fields Here */ ] +> } +> ``` + + +| Property | Type | Required | Default | Description | Example | +| ------------- | ------------------------------------------------------- | -------- | ------- | :------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------- | +| `name` | NameString (no whitespace or `.`) | Required | | Name of the schema. This will be used in paths that reference this schema, and for identifying files containing records for this schema. | `"example-schema"` | +| `fields` | `Array<`[`LecterField`](#dictionary-field-structure)`>` | Required | | List of fields contained in this Schema. | [Dictionary Field Structure](#dictionary-field-structure) | +| `description` | `string` | Optional | None | Free text description of the schema, for use as a reference for users of the schema. This description is not used in dictionary validation. | `"Demonstrating structure of Lectern Schema"` | +| `meta` | [`MetaData`](#meta-data-structure) | Optional | None | Schema implementor defined fields to capture any additional properties not defined in standard Lectern schemas. | [Meta Data Structure](#meta-data-structure) | + +### Dictionary Field Structure +> **Example Dictionary Field Definition** +> ```json +> { +> "name": "example_field", +> "description": "Shows a string field with a required restriction", +> "meta": { /* Custom meta data abou the field here */ }, +> "isArray": false, +> +> "valueType": "string", +> "restrictions": { +> "required": true +> } +> } +> ``` + +| Property | Required | Default | Type | Description | Example | +| ---------------- | -------- | ---------------------- | --------------------------------------------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------- | +| `name` | Required | | NameString (no whitespace or `.`) | Name of the field. This will be used as the header in TSV files in this field's schema, and in any paths referencing this field. | `"example_field` | +| `valueType` | Required | | [Field Data Type](#field-data-types) | Type of value stored in this field | `"string"` | +| `arrayDelimiter` | Optional | `\|` | `string` | Character or string that will be used to split multiple values into an array. The default delimiter is the `\|` character. | | +| `description` | Optional | `""` No value | `string` | Free text description of the field, for use as a reference for users of the schema. This description is not used in dictionary validation. | `"Shows a string field with a required restriction"` | +| `meta` | Optional | Empty object, no value | [`MetaData`](#meta-data-structure) object | Schema implementor defined fields to capture any additional properties not defined in standard Lectern fields. | `{ "displayName": "Example Field" }` | +| `isArray` | Optional | `false` | `boolean` | Type of value stored in this field | | +| `restrictions` | Optional | No Restrictions | `RestrictionsObject` or `Array` | An object containing all validation rules for this field. This can be a single object containing all [restrictions](#field-restrictions) applied to this field or a list of objects whose restrictions will be combined. [Conditional restrictions](#conditional-restrictions) can also be used to apply validation rules based on values of other fields in the record. | `{ "required": true }` | + + + +#### Field Data Types + +| valueType | Description | Examples | +| --------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------- | +| `boolean` | Boolean value, either `true` or `false`. Accepts values with any letter casing, for example `true`, `True`, and `TRUE` will all be interpretted as `true` | `true`, `false` | +| `integer` | Numeric integer value. Will accept positive and negative values (ex. `21` or `-8`) but will reject any decimals (ex. `1.23`) | `21`, `-8` | +| `number` | Numeric value. Will accept any numeric value, including those with decimals. | `1.23`, `-4.567` | +| `string` | String fields. Value can have any length and use any character other than the file delimiter (by default `tab`) or the array delimiter for an array field (by default ` \| `) | `asdf`, `Hello World`, `Another longer example of a string` | + +#### Field Restrictions + +Restrictions on a field are a list of rules that all values for this field must adhere to, these are the list of validations on the contents of each field. Two examples of restrictions are that a value is `required`, and that a value must take a value from a list of available options (`codeList`). The full list of available restrictions are described in the table below. + +The restrictions property of a field can have a value that is either a single restrictions object, or an array with any number of restrictions objects. If an array of restriction objects is provided, each set of restrictions will be applied in turn - for data to be valid, all restrictions in the array must pass. A restrictions object can either contain a set of restrictions from the table below, or be a [conditional restriction](#conditional-restrictions). + +The full list of available restrictions are: + +| Restriction | Used with Field Types | Type | Description | Examples | +| ----------- | ----------------------------- | --------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `codeList` | `integer`, `number`, `string` | Array of type of the field | An array of values of the type matching this field. Data provided for this field must have one of the values in this list. | `["Weak", "Average", "Strong"]` | +| `compare` | all | [ComparedFieldsRule](#comparedfieldsrule-data-structure) object | Enforces that this field has a value based on the provided value in another field. Examples would be to ensure that the two values are not equal, or for numeric values ensure one is greater than the other. | `{ "fields": ["age_at_diagnosis"], "relation": "greaterThanOrEqual" }` Ensure that a field such as `age_at_death` is greater than the provided `age_at_diagnosis` | +| `count` | Array fields of all types | `integer` or [`RangeRule`](#rangerule-data-structure) object | Enfroces the number of entries in an array. Can specify an exact array size, or provide range rules that set maximum and minimum counts. | `7` or `{"min": 5, "max": 10}` | +| `empty` | all | | Requires that no value is provided. This is useful when used on a [conditional restriction](#conditional-restrictions) in order to prevent a value from being given when the condition is `true`. For an array field with this restriction, an empty array is a valid value for this restriction. | n/a | +| `range` | `integer`, `number` | | Uses a [RangeRule](#rangerule-data-structure) object to define minimum and/or maximum values for this field | `{"min": 5}`, `{"exclusiveMax": 50}`, `{"exclusiveMin": 5, "max": 50}` | +| `regex` | `string` | | A regular expression that all values must match. | `^[a-z0-9]+$` | +| `required` | all | | A value must be provided, missing/undefined values will fail validation. Empty strings will not be accepted, though `0` (for `number` and `int` fields) and `false` (for `boolean` fields) are accepted. An array field with this restriction must have at least one entry. | `true`, `false` | +| `unique` | all | | When a field has the `unique` restriction, each record must have a distinct value for this field. Uniqueness tests are case sensitive, so `Abc` and `abc` are both distinct values. This restriction is only applied when a collection of records are tested together, ensuring that no two records in that collection share a value. | `true`, `false` | + +#### Conditional Restrictions + +Restrictions can be added with conditions so that the validations are only applied based on the values provided to other fields within a record. + +A conditional restriction uses an if/then/else style syntax: + +The `if` property will be an object containing an array of `conditions` that look at other fields on the same record and apply matching rules to their values. When those field values match the rules in the condition than the condition passes. An optional `case` property can be added to the `if` object that defines how many of the `conditions` have to pass in order for the whole condition block to resolve as `true` - default is `all`, requiring all conditions to be met. + +The `then` object contains the restrictions that will be applied when the `if` condition is `true`, and the `else` condition contains restrictions to apply when the `if` condition is `false`. The `then` property is required but using an `else` property is optional. + +| Property | Required | Default | Type | Description | Example | +| -------- | -------- | ----------------------------- | --------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | +| `if` | Required | | `RequirementsConditions` | Contains the conditional cases that will be checked before applying this object's restrictions. This object contains a list of `conditions` and a `case` that indicates how many of the conditions need to be found `true` for the entire conditions block to be considered `true`. The case options are `any`, `all`, and `none`, with `all` being default (if case is not provided). | `{ "conditions": [ { "field": "another_field", "match": { "value": "Some Value" }} ], "case": "all" }` | +| `then` | Required | | `RestrictionsObject` or `Array` | The restriction rules to apply when the `if` condition is found to be `true`. | `{ "required": true}` | +| `else` | Optional | Empty object, no restrictions | `RestrictionsObject` or `Array` | The restriction rules to apply when the `if` condition is found to be `false`. | `{ "empty": true}` | + +```json +{ + "if": { + "conditions": [ /* Restriction conditions */ ], + "case": "all" + }, + "then": {/* Restrictons */} OR [ /* Restrictions objects (restriction values or nested conditional restrictions */ ], + "else": {/* Restrictons */} OR [ /* Restrictions objects (restriction values or nested conditional restrictions */ ] +} +``` + +##### Conditions Structure + +A requirement condition is defined by providing a field name or list of field names from this schema, and the matching rules that satisfy this condition. If multiple field names are provided, a `case` property can be added to specify how many of their values must pass the matching rules (`all`, `any`, or `none` of them). + +| Property | Required | Default | Type | Description | Example | +| ---------------- | -------- | ------- | -------------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------- | +| `fields` | Required | | `Array` | Names of fields from the same schema. This match rule will be applied to all fields listed - see `case` to determine the rules for how many of these fields must match. All specified fields must store values of the same type. | `["some_field"]` | +| `match` | Required | | `MatchRules` object | Matching rules for the values of the `fields`. All rules included in this object will be tested and all must be pass - this is not affected by the `case` property. [Conditional Match Rules](#conditional-match-rules) | `{ "value": "Hello World" }` | +| `arrayFieldCase` | Optional | `all` | `all`, `any`, `none` | When a specified field is an array type, the `arrayFieldCase` dictates how many of the values in the array must pass the matching rules. `all` requires all values in the array to pass the matching rule. `any` requires at least one value in the array to match. `non` requires that none of the values in the array match. | `any` | +| `case` | Optional | `all` | `all`, `any`, `none` | Defines how many of the listed `fields` must have a value that matches the `match` rules. `all` requires all fields values to have matching values. `any` requires at least one field to have a matching value. `none` requires that there none of the specified fields have values that match. | `any` | + +> **Example Conditional Restriction**: match single value +> +> Condition where `shirt_size` is `Small` +> ```json +> { +> "fields": ["shirt_size"], +> "match": { +> "value": "Small" +> } +> } +> ``` + +> **Example Conditional Restriction**: match value from list +> +> Condition where `shirt_size` is any value in a list (`Medium` or `Large`) +> ```json +> { +> "fields": ["shirt_size"], +> "match": { +> "codeList": ["Medium", "Large`"] +> } +> } +> ``` + +##### Conditional Match Rules +| Property | Used with Field Types | Type | Description | Example | +| ---------- | --------------------- | -------------------------------------------------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | +| `codeList` | all | Array of type of specified fields | A list of values that the field could match. This rule passes when the specified field's value can be found in this list. | `["value_one", "value_two"]` | +| `compare` | all | [ComparedFieldsRule](#comparedfieldsrule-data-structure) | Compare the value of the specified fields to values of another field (or set of fields). This can be configured to check if the fields match, dont match, and for numeric fields can check if the field is Greater or Lesser than. | `{ "fields": ["compared_to_field"], "relation": "equal" }` | +| `count` | Array type fields | Integer, or [RangeRule](#rangerule-data-structure) | Matches the number of values in an array field. This condition can be provided as a number, in which case this condition matches if the array is that exact length. This condition can be provided as a Range object as well, in which case it will match if the number of elements in the array pass the minimum and maximum conditions provided in the condition. | `2` - Field must have exactly 2 elements.
`{ max: 10 }` - Field must have no more than 10 items. | +| `exists` | all | Boolean | This condition requires a field to either have a value or have no value. When the `exists` condition is set to `true`, the field must have a value. When `exists` is sdet to `false`, the field must have no value. For array fields, `exists=false` only matches when the array is completely empty, and `exists=true` passes if the array has 1 or more values - `arrayCase` has no interaction with the `exists` condition. | `true` | +| `range` | `number`, `integer` | [RangeRule](#rangerule-data-structure) | Maximum and minimum value conditions that a numeric field must pass. | `{ min: 5, exclusiveMax: 10 }` Represents an integer from 5-9. | +| `regex` | `string` | String (Regular Expression) | A regular expression pattern that the value must match. | `^NCIT:C\d+$` Value must match an NCI Thesaurus ID. | +| `value` | all | Type of specified fields | Field value exactly matches the value of the specified field. Strings are matched case sensitive. | `some_value` | + +### Meta Data Structure + +> **Meta Example** +> ```json +> { +> "displayName": "Nicely Formatted Name", +> "externalReferenceId": "ABCD:1234", +> "exampleBooleanPropery": true, +> "exampleNumericProperty": 123 +> } +> ``` + +A `meta` object is available to allow the dictionary creator to add custom properties to the Lectern Dictionary. The `meta` property is available to all Dictionary, Schema, and Field objects. Providing a `meta` value is optional. If provided the `meta` value is a JSON object. There are no restrictions on the field names that can be added to the `meta` object other than they must be valid JSON. The values for properties of the `meta` can either be another nested meta object, or are one of the allowed value types: + - `string` + - `number` + - `boolean` + - `Array` + - `Array` + +### References Structure + +References are defined at the dictionary level so they can be reused across schemas. References can be used to store values that can be used in `meta` or `restrictions` + +#### Using References +Reference variables can be used in a `meta` object or a `restrictions` object as either a restriction value or a conditional match value. + +To use a reference, replace the value in the value of the meta or restriction property with a string containing a `ReferenceTag`. A `ReferenceTags` + +### RangeRule Data Structure + +> **RangeRule Example** +> ```json +> { +> "min": 5, +> "exclusiveMax": 10 +> } +> ``` + +`RangeRule` objects are used to define restrictions and conditions where a numeric minimum or maximum needs to be defined. This object must define at least 1 property (ie. could define a minimum but not maximum, or vice-versa). + +There is an inclusive and an exclusive version of the minimum and maximum properties. `min` and `max` are _inclusive_, and the alternate form `exclusiveMin` and `exclusiveMax` are _exclusive_. By example, `{ "min":5 }` allows the value `5` and greater, while `{ "exclusiveMin": 5 }` allows only values greater than `5` but not `5` itself. + +A `RangeRule` cannot include but an inclusive and exclusive version of min, or of max (ie. it cannot have `min` and `exclusiveMin`.) + +| Property | Description | +| -------------- | :---------------------------------------------------------------- | +| `exclusiveMax` | Allows values less than this value, but not this value itself. | +| `exclusiveMin` | Allows values greater than this value, but not this value itself. | +| `max` | Allows this value and values lesser than this value. | +| `min` | Allows this value and values greater than this value. | + +### ComparedFieldsRule Data Structure + +> **ComparedFieldsRule** Example +> +> ```json +> { +> "fields": "some_field", +> "relation": "equal", +> } +> ``` + +| Property | Required | Default | Type | Description | +| ---------- | -------- | -------------------- | ---------------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `fields` | Required | | `string` or `Array` | The field(s) that the values of will be compared to. These fields will be refered to throughout this section as _compared to_ fields. All these fields need to be the same type as the field(s) they will be compared to. | +| `relation` | Required | | `equal`, `notEqual`, `contains`, `containedIn`, `greaterThan`, `greaterThanOrEqual`, `lesserThan`, `lesserThanOrEqual` | The relation between the values of the test field and the compared to fields. See [ComparedFieldsRule Relations](#comparedfieldsrule-relations). | +| `case` | Optional | `all`, `any`, `none` | MatchCase (RangeRule or one of: `all`, `any`, `none`) | How many of the _compared to_ fields must pass the comparison for this rule to pass. | + +#### ComparedFieldsRule Relations + +| Relation Value | Allowable Field Types | Description | +| ------------------------ | --------------------- | :--------------------------------------------------------------------------------------------------------- | +| **`equal`**: | all | Checks that the current field and the comapred field(s) have the same value | +| **`notEqual`**: | all | Checks that the current field and the comapred field(s) do not have the same value | +| **`contains`** | `string` | Checks that the value of the current field completely contains the value of the compared field(s) | +| **`containedIn`** | `string` | Checks that the value of the current field is completely contained in the value of the compared field(s) | +| **`greaterThan`** | `number`, `integer` | Checks that the value of the current field is greater than (exclusive) the value of the compared field(s). | +| **`greaterThanOrEqual`** | `number`, `integer` | Checks that the value of the current field is greater than or equal to the value of the compared field(s). | +| **`lesserThan`** | `number`, `integer` | Checks that the value of the current field is lesser than (exclusive) the value of the compared field(s). | +| **`lesserThanOrEqual`** | `number`, `integer` | Checks that the value of the current field is lesser than or equal to the value of the compared field(s). | + +## Source Code Reference + +Source code for the Lectern Dictionary meta-schema is made available through the package [@overture-stack/lectern-dictionary](../packages/dictionary/). The meta-schema is formally defined in TypeScript and exported as the type `Dictionary` from the file [`dictionary/src/types/dictionaryTypes.ts`](../packages/dictionary/src/types/dictionaryTypes.ts). This definition is created using [`Zod`] schemas, which are also exported from this package and available for use to confirm a given object is a valid Lectern Dictionary. \ No newline at end of file diff --git a/docs/important-concepts.md b/docs/important-concepts.md new file mode 100644 index 0000000..9e45fdb --- /dev/null +++ b/docs/important-concepts.md @@ -0,0 +1,83 @@ +# Important Concepts + +This document is a reference of commonly used terms and definitions. + +## Dictionary Model + +Lectern provides a "meta-schema" which describes a syntax for creating Data Dictionaries. This meta-schema is a set of rules for a JSON document, and any JSON document that correctly applies these rules represents a valid Lectern Dictionary. The meta-schema is defined through code rules in the [@overture-stack/lectern-dictionary](../packages/dictionary) package. + +A [JSON-schema version of this meta-schema](../generated/DictionaryMetaSchema.json) has been generated and is included in this code base. + +This section describes at a high level the component parts of a Lectern Dictionary and the terms used when discussing those parts. The terms defined here are used throughout the documentation and the type system of the Lectern codebase. If you are writing a Lectern Dictionary, you may instead be looking for the [reference documentation for Lectern Dictionaries](). + +### Dictionary + +Placeholder + +### Schema + +Placeholder + +### Field + +Placeholder + +#### Data Type + +Placeholder + +### Restrictions + +Placeholder + +#### Schema Restriction + +Placeholder + +#### Field Restriction + +Placeholder + +##### Conditional Field Restriction + +Placeholder + +### Meta + +Placeholder + +## Common Types + +### TestResult + +Reference: [validation/src/types/testResult.ts](../packages/validation/src/types/testResult.ts) + +A `TestResult` represents the outcome of a validation test applied to an object. For example, a [`Field`](#field) will be validated based on its [Restrictions](#field-restriction) and this will generate a `TestResult` that will contain a list of `ValidationTestErrors` if the field is found to be invalid for one or more reasons. + +The TestResult object indicates if the object tested was **valid** or **invalid** (`valid: true`) for the given test. When the value is **invalid**, additional information should be included in the result that describes the conditions that the test failed. + +Example Valid `TestResult`: + +```ts +{ + valid: true +} +``` + +Example Invalid `TestResult`, for a field validation that failed due to CodeList restriction: + +```ts +{ + valid: false, + details: { + reason: "INVALID_BY_RESTRICTION", + value: "Atari", + errors: [ + { + type: "codeList", + rule: ["Nintendo", "Sega", "PlayStation", "Xbox"] + } + ] + } +} +``` \ No newline at end of file diff --git a/docs/validation/field-validation.md b/docs/validation/field-validation.md new file mode 100644 index 0000000..5fc2906 --- /dev/null +++ b/docs/validation/field-validation.md @@ -0,0 +1,53 @@ +# Field Validation + +To validate a field, each restriction from the field definition must be applied to the value in turn. The outcome of a field validation test will then be a [`TestResult`](../important-concepts.md#testresult) that if invalid will contain an array of type `FieldValidationError` that contains all the [reasons why the validation failed](#field-validation-failure-reasons). + +## Inputs + +- **Value** - The value to be validated. This is not the raw string from a TSV, this is the final value coerced to the correct type. The type of the value will be confirmed in this process. +- **Field Definition** - The definition for the field being validated, taken from the Schema. This will provide validation information including valueType and all restrictions to apply. +- **Data Record** - The data record that this value belongs to. This is used to resolve [Conditional Restrictions](../important-concepts.md#conditional-field-restriction). + +## Sequence Diagram + +The following sequence diagram describes the logical processes that will be performed to determine if a field is valid. Some of the tasks listed here are described beneath the diagram. + +```mermaid +flowchart TD + Start([Start]) --> ValueTest[Test if Value matches ValueType] + ValueTest --Value is invalid type--> EndInvalidValue([Return invalid: INVALID_VALUE_TYPE]) + ValueTest --Value is valid type--> Restrictions[[Resolve Conditional Restrictions]] + ValueTest --Value is 'undefined'--> Restrictions + + Restrictions --> TestRestrictions[[Test value for every Restriction]] + + + TestRestrictions --> CheckTestResult{{Check if invalid for any Restrictions}} + + CheckTestResult --Value valid for all tests--> Valid + CheckTestResult --Invalid for one or more tests--> EndInvalidTests([Return invalid: INVALID_BY_RESTRICTION]) +``` + +### Resolve Conditional Restrictions + +Collect all restrictions that apply to this field by resolving conditional restrictions that are defined in the field definition. Each conditional restriction defines a check on some other field(s) in the data record, this process will make these checks and determine which restrictions should be applied. + +## Field Validation Failure Reasons +The validation test can fail for the following reasons: + +1. `INVALID_VALUE_TYPE` - The data type was invalid for the field's defined `valueType`. +2. `INVALID_BY_RESTRICTION` - If the field value type is correct and the value is present, then the value will be tested for all applicable restrictions. All restrictions will be tested, even if some have already failed: the invalid result will include an array of errors that include all reasons why the value is invalid. + + +## Implementation Details +### Field Restriction Tests + +Every type of restriction defined in the Lectern meta-schema needs to have a function written that can take a `DataRecordValue` and return a `TestResult` indicating if the value passes the restriction. When defining a restriction test, it would be ideal to define a function that only accepts data values that match the value types that this restriction can apply to. However, when actually validating a field, it is incredibly cumbersome to handle all the type variables (field valueType, field isArray, restrictions available for that type and array, validation function for that restriction for those types). There are definitely ways to do this buy they become very verbose and have a lot of moving parts to modify whenever a new restriction is added. + +In place of a perfect one to one mapping of restrictions and types to validation functions, we have adopted a system where a field validation function has a common signature that will take any DataRecordValue, and a RestrictionRule data type, and will then return a TestResult with details specific to that restriction type. It is the responsibility of the test to ensure that it is only applied to data that is the correct type. We aren't given the field definition in this function, so we don't need to compare if the value we have been given is the correct type based on the field definition, that can happen elsewhere. The field validation function only needs to check the type of the value it was given, and apply its test to that. + +When a field validation function is given a value that is untestable - for example a regex restriction is given a value with type `number` - then the validation function should return that the test is **valid**. We simply do not apply the test when the type is wrong. Importantly, this usually applies to `undefined` values. For example, a range restriction will be **valid** if given an `undefined` value. A required restriction will return **invalid** for `undefined` values, but can return **valid** for everything else. + +Types are defined for these test functions, named [`FieldRestrictionTest`](../../packages/validation/src/fieldValidation/FieldRestrictionTest.ts). There are versions of the test for single values and for array values - the array version returns additional information in order to indicate which items in the array are invalid. The function [`createFieldRestrictionTestForArrays`](../../packages/validation/src/fieldValidation/restrictions/createFieldRestrictionTestForArrays.ts) allows a test function for a single value to be applied to an entire array using a standard process, including properly formatted errors. This results in a common format for defining a field restriction test file that can define a single and array function and combine them for the final test function. + +See the [`rangeValidation`](../../packages/validation/src/fieldValidation/restrictions/rangeValidation.ts) file as a standard templated example of a Field Restriction Test file. \ No newline at end of file diff --git a/docs/validation/index.md b/docs/validation/index.md new file mode 100644 index 0000000..e013c9a --- /dev/null +++ b/docs/validation/index.md @@ -0,0 +1,18 @@ +# Validating Data with Lectern + +- Uses validation library + - Available through: + - Lectern client NPM package + - Lectern server via web request (pending) +- Can validate entire data sets, or subsets of the data. These Different validations apply subsets of the dictionary restrictions based on the amount of data provided. + - Individual field from a single data record vs Field + - Includes all field level restrictions and data value type checking + - Can be provided Data Record information to resolve conditional restrictions. + - Single data record vs Schema + - Includes all field level restricitons for every field in the record + - Always resolves conditional restrictions + - Includes check for unrecognized fields + - Data of a single entity type vs Schema + - includes unique and uniqueKey restrictions + - Entire dataset vs Dictionary + - inlude foreignKey restrictions \ No newline at end of file diff --git a/docs/validation/record-validation.md b/docs/validation/record-validation.md new file mode 100644 index 0000000..3ff1c67 --- /dev/null +++ b/docs/validation/record-validation.md @@ -0,0 +1,10 @@ +# Record Validation + +This process validates a single [data record](../important-concepts.md/#datarecord) using a [Schema](../important-concepts.md#schema) definition. This will check each field in the record against the field definition from the schema. he outcome of a Record validation test will then be a [`TestResult`](./important-concepts.md#testresult) that if invalid will contain an array of type `RecordValidationError` that contains all the [reasons why the validation failed](#record-validation-failure-reasons). + +## Record Validation Failure Reasons + +Record validation has only a single failure case which occurs if one or more of the field values in the data record are invalid. Therefore the information provided to an invalid Record validation test is an array of errors for each field that failed validation. Each of these errors are an object that will have a `reason` property to indicate the source of the error: + +1. `INVALID_FIELD_VALUE`: The field is defined in the schema and the value was tested and found to be in error. The causes for this validation failure are defined in [`Field Validation Error Reasons`](./field-validation.md#field-validation-failure-reasons). +2. `UNRECOGNIZED_FIELD`: The field in the record is not found in the Schema definition, therefore it should not be in this data record. There is no additional information given other than the name of the field. \ No newline at end of file diff --git a/nx.json b/nx.json index 03619a0..9249f41 100644 --- a/nx.json +++ b/nx.json @@ -3,7 +3,7 @@ "default": { "runner": "nx/tasks-runners/default", "options": { - "cacheableOperations": ["build", "lint", "test"] + "cacheableOperations": ["build", "generate", "lint", "test"] } } }, diff --git a/package.json b/package.json index 2cf4881..9398856 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "test:validation": "pnpm nx test @overture-stack/lectern-validation", "test:server": "pnpm nx test server", "test:client": "pnpm nx test @overture-stack/lectern-client", - "generate": "pnpm build:dictionary && pnpm -C scripts generate" + "generate": "pnpm build:dictionary && pnpm -C generate generate" }, "keywords": [], "author": "Ontario Institute for Cancer Research", diff --git a/packages/client/package.json b/packages/client/package.json index 1bb4c2d..3f21f95 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -8,7 +8,7 @@ "typings": "./dist/index.d.ts", "description": "TypeScript client to interact with Lectern servers and perform data validation versus Lectern dictionaries.", "scripts": { - "build": "rimraf dist && tsc", + "build": "rimraf dist && tsc -p tsconfig.build.json", "test": "nyc mocha --exit --timeout 5000 -r ts-node/register test/**.spec.ts", "lint": "tslint -c tslint.json -e node_modules -p tsconfig.json" }, diff --git a/packages/client/src/processing/convertDataValueTypes.ts b/packages/client/src/processing/convertDataValueTypes.ts deleted file mode 100644 index e3c4a2a..0000000 --- a/packages/client/src/processing/convertDataValueTypes.ts +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved - * - * This program and the accompanying materials are made available under the terms of - * the GNU Affero General Public License v3.0. You should have received a copy of the - * GNU Affero General Public License along with this program. - * If not, see . - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT - * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER - * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -import _ from 'lodash'; - -import { Singular } from 'common'; -import { - DataRecord, - DataRecordValue, - UnprocessedDataRecord, - Schema, - SchemaField, - SchemaFieldValueType, -} from 'dictionary'; - -import { convertToArray, isEmpty } from '../utils'; -import { SchemaValidationError, SchemaValidationErrorTypes } from '@overture-stack/lectern-validation'; - -/** - * Warning: - * This needs to be provided records that have already had their types validated, there is little to no checking that the values - * this converts will be correct, and no errors are thrown by failed conversions. - * - * @param schemaDef - * @param record - * @param index - * @param recordErrors - * @returns - */ -export const convertFromRawStrings = ( - schemaDef: Schema, - record: UnprocessedDataRecord, - index: number, - recordErrors: ReadonlyArray, -): DataRecord => { - const mutableRecord: DataRecord = { ...record }; - schemaDef.fields.forEach((field) => { - // if there was an error for this field don't convert it. this means a string was passed instead of number or boolean - // this allows us to continue other validations without hiding possible errors downstream. - - if ( - recordErrors.find( - (er) => er.errorType === SchemaValidationErrorTypes.INVALID_FIELD_VALUE_TYPE && er.fieldName === field.name, - ) - ) { - return undefined; - } - - /* - * if the field is missing from the records don't set it to undefined - */ - if (!_.has(record, field.name)) { - return; - } - - // need to check how it behaves for record[field.name] === "" - if (isEmpty(record[field.name])) { - mutableRecord[field.name] = undefined; - return; - } - - const rawValue = record[field.name]; - - if (field.isArray) { - const rawValueAsArray = convertToArray(rawValue); - // TODO: Keeping this type assertion for during the type refactoring process. We need to refactor how values are validated as matching their corresponding types - // refactoring type checking/conversion will result in combining the conversion and type checking code into a single place. Right now its possible to run the converter - // on values that have not been properly validated. - // This type assertion is needed because the code as is results in teh type `(string | number | boolean | undefined)[]` instead of `string[] | number[] | boolean[] | undefined` - mutableRecord[field.name] = rawValueAsArray.map((value) => getTypedValue(field, value)) as DataRecordValue; - } else { - const rawValueAsString = Array.isArray(rawValue) ? rawValue.join('') : rawValue; - mutableRecord[field.name] = getTypedValue(field, rawValueAsString); - } - }); - return mutableRecord; -}; - -const getTypedValue = (field: SchemaField, rawValue: string): Singular => { - switch (field.valueType) { - case SchemaFieldValueType.Values.boolean: { - return Boolean(rawValue.toLowerCase()); - } - case SchemaFieldValueType.Values.integer: { - return Number(rawValue); - } - case SchemaFieldValueType.Values.number: { - return Number(rawValue); - } - case SchemaFieldValueType.Values.string: { - // For string fields with a codeList restriction: - // we want to format the value with the same letter cases as is defined in the codeList - if (field.restrictions?.codeList && Array.isArray(field.restrictions.codeList)) { - const formattedField = field.restrictions.codeList.find( - (codeListOption) => codeListOption.toString().toLowerCase() === rawValue.toString().toLowerCase(), - ); - if (formattedField) { - return formattedField; - } - } - - // Return original string - return rawValue; - } - } -}; diff --git a/packages/client/src/processing/index.ts b/packages/client/src/processing/index.ts index 0baea5d..942ebad 100644 --- a/packages/client/src/processing/index.ts +++ b/packages/client/src/processing/index.ts @@ -17,224 +17,120 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import type { SchemaValidationError } from '@overture-stack/lectern-validation'; import * as validation from '@overture-stack/lectern-validation'; -import { NotFoundError } from 'common'; import { DataRecord, Dictionary, Schema, UnprocessedDataRecord } from 'dictionary'; -import _ from 'lodash'; import { loggerFor } from '../logger'; -import { convertToArray, isEmpty, isNotAbsent, isString, isStringArray, notEmpty } from '../utils'; -import { convertFromRawStrings } from './convertDataValueTypes'; -import { BatchProcessingResult, FieldNamesByPriorityMap, SchemaProcessingResult } from './processingResultTypes'; -import * as pipelines from './validationPipelines'; +import { + SchemaProcessingResult, + type DictionaryProcessingResult, + type RecordProcessingResult, +} from './processingResultTypes'; const L = loggerFor(__filename); -export const processSchemas = ( +export const processDictionary = ( + data: Record, dictionary: Dictionary, - schemasData: Record, -): Record => { - const results: Record = {}; - - Object.keys(schemasData).forEach((schemaName) => { - // Run validations at the record level - const recordLevelValidationResults = processRecords(dictionary, schemaName, schemasData[schemaName]); - - // Run cross-schema validations - const schemaDef = getNotNullSchemaDefinitionFromDictionary(dictionary, schemaName); - const crossSchemaLevelValidationResults = pipelines - .runCrossSchemaValidationPipeline(schemaDef, schemasData, [validation.validateForeignKeys]) - .filter(notEmpty); - - const allErrorsBySchema: validation.SchemaValidationError[] = [ - ...recordLevelValidationResults.validationErrors, - ...crossSchemaLevelValidationResults, - ]; - - results[schemaName] = { - validationErrors: allErrorsBySchema, - processedRecords: recordLevelValidationResults.processedRecords, - }; - }); +): DictionaryProcessingResult => { + const parsedResult = validation.parseDictionaryValues(data, dictionary); - return results; -}; + if (!parsedResult.success) { + return { + status: 'ERROR_PARSING', + data: parsedResult.data, + }; + } -export const processRecords = ( - dictionary: Dictionary, - definition: string, - records: UnprocessedDataRecord[], -): BatchProcessingResult => { - const schemaDef = getNotNullSchemaDefinitionFromDictionary(dictionary, definition); - - let validationErrors: SchemaValidationError[] = []; - const processedRecords: DataRecord[] = []; - - records.forEach((dataRecord, index) => { - const result = process(dictionary, definition, dataRecord, index); - validationErrors = validationErrors.concat(result.validationErrors); - processedRecords.push(_.cloneDeep(result.processedRecord)); - }); - // Record set level validations - const newErrors = validateRecordsSet(schemaDef, processedRecords); - validationErrors.push(...newErrors); - L.debug( - `done processing all rows, validationErrors: ${validationErrors.length}, validRecords: ${processedRecords.length}`, + const parsedData = Object.entries(parsedResult.data).reduce>( + (output, [schemaName, schemaConversionResult]) => { + // Expect all schemaResults to be successful, otherwise the conversion result would have failed. + output[schemaName] = schemaConversionResult.data.records; + return output; + }, + {}, ); + const validationResult = validation.validateDictionary(parsedData, dictionary); + + if (!validationResult.valid) { + return { + status: 'ERROR_VALIDATION', + data: parsedData, + errors: validationResult.details, + }; + } return { - validationErrors, - processedRecords, + status: 'SUCCESS', + data: parsedData, }; }; -export const process = ( - dictionary: Dictionary, - schemaName: string, - data: Readonly, - index: number, -): SchemaProcessingResult => { - const schemaDef = dictionary.schemas.find((e) => e.name === schemaName); - - if (!schemaDef) { - throw new Error(`no schema found for : ${schemaName}`); - } - - let validationErrors: SchemaValidationError[] = []; +/** + * Process a list of records for a single schema. + * + * Parse and then validate each record in the list. + * @param dictionary + * @param definition + * @param records + * @returns + */ +export const processSchema = (records: UnprocessedDataRecord[], schema: Schema): SchemaProcessingResult => { + const parseResult = validation.parseSchemaValues(records, schema); - const defaultedRecord = populateDefaults(schemaDef, data, index); - L.debug(`done populating defaults for record #${index}`); - const result = validateUnprocessedRecord(schemaDef, defaultedRecord, index); - L.debug(`done validation for record #${index}`); - if (result && result.length > 0) { - L.debug(`${result.length} validation errors for record #${index}`); - validationErrors = validationErrors.concat(result); + if (!parseResult.success) { + return { + status: 'ERROR_PARSING', + ...parseResult.data, + }; } - const convertedRecord = convertFromRawStrings(schemaDef, defaultedRecord, index, result); - L.debug(`converted row #${index} from raw strings`); - const postTypeConversionValidationResult = validateAfterTypeConversion( - schemaDef, - _.cloneDeep(convertedRecord), - index, - ); - if (postTypeConversionValidationResult && postTypeConversionValidationResult.length > 0) { - validationErrors = validationErrors.concat(postTypeConversionValidationResult); - } + const parsedRecords = parseResult.data.records; + const validationResult = validation.validateSchema(parsedRecords, schema); - L.debug(`done processing all rows, validationErrors: ${validationErrors.length}, validRecords: ${convertedRecord}`); + if (!validationResult.valid) { + return { + status: 'ERROR_VALIDATION', + records: parsedRecords, + errors: validationResult.details, + }; + } return { - validationErrors, - processedRecord: convertedRecord, + status: 'SUCCESS', + records: parsedRecords, }; }; -const getNotNullSchemaDefinitionFromDictionary = (dictionary: Dictionary, schemaName: string): Schema => { - const schemaDef = dictionary.schemas.find((e) => e.name === schemaName); - if (!schemaDef) { - throw new Error(`no schema found for : ${schemaName}`); - } - return schemaDef; -}; +/** + * Process a single data record. + * + * Parse and then validate a data record. If there are errors found during conversion, + * those errors will be returned and validation will be skipped. The final result will indicate if the + * data processing attempt was successful, or failed due to errors in conversion or validation. + */ +export const processRecord = (schema: Schema, data: UnprocessedDataRecord): RecordProcessingResult => { + const parseResult = validation.parseRecordValues(data, schema); -export const getSchemaFieldNamesWithPriority = (schema: Dictionary, definition: string): FieldNamesByPriorityMap => { - const schemaDef = schema.schemas.find((schema) => schema.name === definition); - if (!schemaDef) { - throw new NotFoundError(`no schema found for : ${definition}`); + if (!parseResult.success) { + return { + status: 'ERROR_PARSING', + ...parseResult.data, + }; } - const fieldNamesMapped: FieldNamesByPriorityMap = { required: [], optional: [] }; - schemaDef.fields.forEach((field) => { - if (field.restrictions?.required) { - fieldNamesMapped.required.push(field.name); - } else { - fieldNamesMapped.optional.push(field.name); - } - }); - return fieldNamesMapped; -}; -/** - * Populate the passed records with the default value based on the field name if the field is - * missing from the records it will NOT be added. - * @param definition the name of the schema definition to use for these records - * @param records the list of records to populate with the default values. - */ -const populateDefaults = (schemaDef: Schema, record: UnprocessedDataRecord, index: number): UnprocessedDataRecord => { - const clonedRecord = _.cloneDeep(record); - schemaDef.fields.forEach((field) => { - const defaultValue = field.meta && field.meta.default; - if (isEmpty(defaultValue)) return undefined; - - const value = record[field.name]; - - // data record value is (or is expected to be) just one string - if (isString(value) && !field.isArray) { - if (isNotAbsent(value) && value.trim() === '') { - L.debug(`populating Default: "${defaultValue}" for "${field.name}" of record at index ${index}`); - clonedRecord[field.name] = `${defaultValue}`; - } - return undefined; - } - - // data record value is (or is expected to be) array of string - if (isStringArray(value) && field.isArray) { - if (notEmpty(value) && value.every((v) => v.trim() === '')) { - L.debug(`populating Default: "${defaultValue}" for ${field.name} of record at index ${index}`); - const arrayDefaultValue = convertToArray(defaultValue); - clonedRecord[field.name] = arrayDefaultValue.map((v) => `${v}`); - } - return undefined; - } - }); - - return _.cloneDeep(clonedRecord); -}; + const record = parseResult.data.record; + const validationResult = validation.validateRecord(record, schema); -/** - * Run schema validation pipeline for a schema defintion on the list of records provided. - * @param definition the schema definition name. - * @param record the records to validate. - */ -const validateUnprocessedRecord = ( - schemaDef: Schema, - record: UnprocessedDataRecord, - index: number, -): ReadonlyArray => { - const majorErrors = pipelines - .runUnprocessedRecordValidationPipeline(record, index, schemaDef.fields, [ - validation.validateFieldNames, - validation.validateNonArrayFields, - validation.validateRequiredFields, - validation.validateValueTypes, - ]) - .filter(notEmpty); - return [...majorErrors]; -}; + if (!validationResult.valid) { + return { + status: 'ERROR_VALIDATION', + record, + errors: validationResult.details, + }; + } -const validateAfterTypeConversion = ( - schemaDef: Schema, - record: DataRecord, - index: number, -): ReadonlyArray => { - const validationErrors = pipelines - .runRecordValidationPipeline(record, index, schemaDef.fields, [ - validation.validateRegex, - validation.validateRange, - validation.validateCodeList, - validation.validateScript, - ]) - .filter(notEmpty); - - return [...validationErrors]; + return { + status: 'SUCCESS', + record, + }; }; - -function validateRecordsSet(schemaDef: Schema, processedRecords: DataRecord[]) { - const validationErrors = pipelines - .runDatasetValidationPipeline(processedRecords, schemaDef, [ - validation.validateUnique, - validation.validateUniqueKey, - ]) - .filter(notEmpty); - return validationErrors; -} diff --git a/packages/client/src/processing/processingResultTypes.ts b/packages/client/src/processing/processingResultTypes.ts index 8360998..4769b9c 100644 --- a/packages/client/src/processing/processingResultTypes.ts +++ b/packages/client/src/processing/processingResultTypes.ts @@ -17,20 +17,73 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { SchemaValidationError } from '@overture-stack/lectern-validation'; +import { + SchemaValidationError, + type ParseRecordFailureData, + type DictionaryValidationError, + type RecordValidationError, + type ParseSchemaFailureData, + type ParseDictionaryData, +} from '@overture-stack/lectern-validation'; import { DataRecord, Schema } from 'dictionary'; export type ProcessingFunction = (schema: Schema, rec: Readonly, index: number) => any; -export type SchemaProcessingResult = { - validationErrors: SchemaValidationError[]; - processedRecord: DataRecord; +type RecordProcessingErrorSuccess = { + status: 'SUCCESS'; + record: DataRecord; }; +type RecordProcessingErrorParsing = ParseRecordFailureData & { + status: 'ERROR_PARSING'; +}; +type RecordProcessingErrorValidation = { + status: 'ERROR_VALIDATION'; + record: DataRecord; + errors: RecordValidationError[]; +}; +export type RecordProcessingResult = + | RecordProcessingErrorSuccess + | RecordProcessingErrorParsing + | RecordProcessingErrorValidation; + +type SchemaProcessingErrorSuccess = { + status: 'SUCCESS'; + records: DataRecord[]; +}; +type SchemaProcessingErrorParsing = ParseSchemaFailureData & { + status: 'ERROR_PARSING'; +}; +type SchemaProcessingErrorValidation = { + status: 'ERROR_VALIDATION'; + records: DataRecord[]; + errors: SchemaValidationError[]; +}; +export type SchemaProcessingResult = + | SchemaProcessingErrorSuccess + | SchemaProcessingErrorParsing + | SchemaProcessingErrorValidation; -export type BatchProcessingResult = { - validationErrors: SchemaValidationError[]; - processedRecords: DataRecord[]; +type DictionaryProcessingErrorSuccess = { + status: 'SUCCESS'; + data: Record; +}; +type DictionaryProcessingErrorParsing = { + status: 'ERROR_PARSING'; + data: ParseDictionaryData; +}; + +/** + * Includes all converted data plus the list of errors that occurred in validation. + */ +type DictionaryProcessingErrorValidation = { + status: 'ERROR_VALIDATION'; + data: Record; + errors: DictionaryValidationError[]; }; +export type DictionaryProcessingResult = + | DictionaryProcessingErrorSuccess + | DictionaryProcessingErrorParsing + | DictionaryProcessingErrorValidation; export interface FieldNamesByPriorityMap { required: string[]; diff --git a/packages/client/src/processing/validationPipelines.ts b/packages/client/src/processing/validationPipelines.ts deleted file mode 100644 index a5fe2f5..0000000 --- a/packages/client/src/processing/validationPipelines.ts +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved - * - * This program and the accompanying materials are made available under the terms of - * the GNU Affero General Public License v3.0. You should have received a copy of the - * GNU Affero General Public License along with this program. - * If not, see . - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT - * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER - * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -import { DataRecord, UnprocessedDataRecord, Schema, SchemaField } from 'dictionary'; - -import { SchemaValidationError } from '@overture-stack/lectern-validation'; -import { - CrossSchemaValidationFunction, - DatasetValidationFunction, - UnprocessedRecordValidationFunction, - ValidationFunction, -} from '@overture-stack/lectern-validation'; - -export const runUnprocessedRecordValidationPipeline = ( - record: UnprocessedDataRecord, - index: number, - fields: ReadonlyArray, - validationFunctions: Array, -) => { - let result: Array = []; - for (const validationFunction of validationFunctions) { - result = result.concat(validationFunction(record, index, getValidFields(result, fields))); - } - return result; -}; - -export const runRecordValidationPipeline = ( - record: DataRecord, - index: number, - fields: ReadonlyArray, - validationFunctions: Array, -) => { - let result: Array = []; - for (const validationFunction of validationFunctions) { - result = result.concat(validationFunction(record, index, getValidFields(result, fields))); - } - return result; -}; - -export const runDatasetValidationPipeline = ( - data: DataRecord[], - schema: Schema, - validationFunctions: Array, -) => validationFunctions.flatMap((validationFunction) => validationFunction(data, schema)); - -export const runCrossSchemaValidationPipeline = ( - schema: Schema, - data: Record, - validationFunctions: Array, -) => { - let result: Array = []; - for (const validationFunction of validationFunctions) { - result = result.concat(validationFunction(schema, data)); - } - return result; -}; - -const getValidFields = (errs: ReadonlyArray, fields: ReadonlyArray) => { - return fields.filter((field) => { - return !errs.find((e) => e.fieldName === field.name); - }); -}; diff --git a/packages/client/test/processing.spec.ts b/packages/client/test/processing.spec.ts deleted file mode 100644 index ccd84d1..0000000 --- a/packages/client/test/processing.spec.ts +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved - * - * This program and the accompanying materials are made available under the terms of - * the GNU Affero General Public License v3.0. You should have received a copy of the - * GNU Affero General Public License along with this program. - * If not, see . - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT - * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER - * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -import { SchemaValidationErrorTypes } from '@overture-stack/lectern-validation'; -import chai from 'chai'; -import { functions as schemaService } from '../src'; -import { loggerFor } from '../src/logger'; -import dictionary from './fixtures/registrationSchema'; -const L = loggerFor(__filename); - -chai.should(); - -const VALUE_NOT_ALLOWED = 'The value is not permissible for this field.'; -const PROGRAM_ID_REQ = 'program_id is a required field.'; - -describe('processing', () => { - it('should populate records based on default value ', () => { - const result = schemaService.processRecords(dictionary, 'registration', [ - { - program_id: 'PEME-CA', - submitter_donor_id: 'OD1234', - gender: '', - submitter_specimen_id: '87813', - specimen_type: 'Skin', - tumour_normal_designation: 'Normal', - submitter_sample_id: 'MAS123', - sample_type: 'ctDNA', - }, - { - program_id: 'PEME-CA', - submitter_donor_id: 'OD1234', - gender: '', - submitter_specimen_id: '87812', - specimen_type: 'Skin', - tumour_normal_designation: 'Normal', - submitter_sample_id: 'MAS1234', - sample_type: 'ctDNA', - }, - ]); - chai.expect(result.processedRecords[0].gender).to.eq('Other'); - chai.expect(result.processedRecords[1].gender).to.eq('Other'); - }); - - it('should NOT populate missing columns based on default value ', () => { - const result = schemaService.processRecords(dictionary, 'registration', [ - { - program_id: 'PEME-CA', - submitter_donor_id: 'OD1234', - gendr: '', - submitter_specimen_id: '87813', - specimen_type: 'Skin', - tumour_normal_designation: 'Normal', - submitter_sample_id: 'MAS123', - sample_type: 'ctDNA', - }, - { - program_id: 'PEME-CA', - submitter_donor_id: 'OD1234', - gender: '', - submitter_specimen_id: '87812', - specimen_type: 'Skin', - tumour_normal_designation: 'Normal', - submitter_sample_id: 'MAS1234', - sample_type: 'ctDNA', - }, - ]); - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.MISSING_REQUIRED_FIELD, - fieldName: 'gender', - index: 0, - info: {}, - message: 'gender is a required field.', - }); - }); -}); diff --git a/packages/client/test/validation.spec.ts b/packages/client/test/validation.spec.ts deleted file mode 100644 index 36f72d7..0000000 --- a/packages/client/test/validation.spec.ts +++ /dev/null @@ -1,827 +0,0 @@ -/* - * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved - * - * This program and the accompanying materials are made available under the terms of - * the GNU Affero General Public License v3.0. You should have received a copy of the - * GNU Affero General Public License along with this program. - * If not, see . - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT - * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER - * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -import { SchemaValidationErrorTypes } from '@overture-stack/lectern-validation'; -import chai from 'chai'; -import { functions as schemaService } from '../src'; -import { loggerFor } from '../src/logger'; -import dictionary from './fixtures/registrationSchema'; - -chai.should(); - -const VALUE_NOT_ALLOWED = 'The value is not permissible for this field.'; -const PROGRAM_ID_REQ = 'program_id is a required field.'; - -describe('validation', () => { - it('should validate required', () => { - const result = schemaService.processRecords(dictionary, 'registration', [ - { - submitter_donor_id: 'OD1234', - gender: 'Female', - submitter_specimen_id: '87813', - specimen_type: 'Skin', - tumour_normal_designation: 'Normal', - submitter_sample_id: 'MAS123', - sample_type: 'ctDNA', - }, - ]); - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.MISSING_REQUIRED_FIELD, - fieldName: 'program_id', - index: 0, - info: {}, - message: PROGRAM_ID_REQ, - }); - }); - - it('should validate value types', () => { - const result = schemaService.processRecords(dictionary, 'address', [ - { - country: 'US', - unit_number: 'abc', - postal_code: '12345', - }, - ]); - - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.INVALID_FIELD_VALUE_TYPE, - fieldName: 'unit_number', - index: 0, - info: { value: ['abc'] }, - message: VALUE_NOT_ALLOWED, - }); - }); - - it('should convert string to integer after processing', () => { - const result = schemaService.processRecords(dictionary, 'address', [ - { - country: 'US', - unit_number: '123', - postal_code: '12345', - }, - ]); - chai.expect(result.processedRecords).to.deep.include({ - country: 'US', - unit_number: 123, - postal_code: '12345', - }); - }); - - it('should validate regex', () => { - const result = schemaService.processRecords(dictionary, 'registration', [ - { - program_id: 'PEME-CAA', - submitter_donor_id: 'OD1234', - gender: 'Female', - submitter_specimen_id: '87813', - specimen_type: 'Skin', - tumour_normal_designation: 'Normal', - submitter_sample_id: 'MAS123', - sample_type: 'ctDNA', - }, - ]); - chai.expect(result.validationErrors[0]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_REGEX, - fieldName: 'program_id', - index: 0, - info: { - examples: 'PACA-CA, BASHAR-LA', - regex: '^[A-Z1-9][-_A-Z1-9]{2,7}(-[A-Z][A-Z])$', - value: ['PEME-CAA'], - }, - message: - 'The value is not a permissible for this field, it must meet the regular expression: "^[A-Z1-9][-_A-Z1-9]{2,7}(-[A-Z][A-Z])$". Examples: PACA-CA, BASHAR-LA', - }); - }); - - it('should validate range', () => { - const result = schemaService.processRecords(dictionary, 'address', [ - { - country: 'US', - postal_code: '12345', - unit_number: '-1', - }, - { - country: 'US', - postal_code: '12345', - unit_number: '223', - }, - { - country: 'US', - postal_code: '12345', - unit_number: '500000', - }, - ]); - - const info1 = { - exclusiveMax: 999, - min: 0, - value: [-1], - }; - - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.INVALID_BY_RANGE, - fieldName: 'unit_number', - index: 0, - info: info1, - message: `Value is out of permissible range, it must be >= 0 and < 999.`, - }); - - const info2 = { - exclusiveMax: 999, - min: 0, - value: [500000], - }; - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.INVALID_BY_RANGE, - fieldName: 'unit_number', - index: 2, - info: info2, - message: `Value is out of permissible range, it must be >= 0 and < 999.`, - }); - }); - - it('should validate script', () => { - const result = schemaService.processRecords(dictionary, 'address', [ - { - country: 'US', - postal_code: '12', - }, - { - country: 'CANADA', - postal_code: 'ABC', - }, - { - country: 'US', - postal_code: '15523', - }, - ]); - - chai.expect(result.validationErrors.length).to.eq(2); - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.INVALID_BY_SCRIPT, - fieldName: 'postal_code', - index: 0, - info: { message: 'invalid postal code for US', value: '12' }, - message: 'invalid postal code for US', - }); - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.INVALID_BY_SCRIPT, - fieldName: 'postal_code', - index: 1, - info: { message: 'invalid postal code for CANADA', value: 'ABC' }, - message: 'invalid postal code for CANADA', - }); - }); - - it('should validate if non-required feilds are not provided', () => { - const result = schemaService.processRecords(dictionary, 'donor', [ - // optional enum field not provided - { - program_id: 'PACA-AU', - submitter_donor_id: 'ICGC_0004', - gender: 'Female', - ethnicity: 'black or african american', - vital_status: 'alive', - }, - // optional enum field provided with proper value - { - program_id: 'PACA-AU', - submitter_donor_id: 'ICGC_0002', - gender: 'Male', - ethnicity: 'asian', - vital_status: 'deceased', - cause_of_death: 'died of cancer', - survival_time: '124', - }, - // optional enum field provided with no value - { - program_id: 'PACA-AU', - submitter_donor_id: 'ICGC_0002', - gender: 'Male', - ethnicity: 'asian', - vital_status: 'deceased', - cause_of_death: '', - survival_time: '124', - }, - ]); - chai.expect(result.validationErrors.length).to.eq(0); - }); - - it('should error if integer fields are not valid', () => { - const result = schemaService.processRecords(dictionary, 'donor', [ - { - program_id: 'PACA-AU', - submitter_donor_id: 'ICGC_0002', - gender: 'Other', - ethnicity: 'asian', - vital_status: 'deceased', - cause_of_death: 'died of cancer', - survival_time: '0.5', - }, - ]); - - chai.expect(result.validationErrors.length).to.eq(1); - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.INVALID_FIELD_VALUE_TYPE, - fieldName: 'survival_time', - index: 0, - info: { value: ['0.5'] }, - message: VALUE_NOT_ALLOWED, - }); - }); - - it('should validate case insensitive enums, return proper format', () => { - const result = schemaService.processRecords(dictionary, 'registration', [ - { - program_id: 'PACA-AU', - submitter_donor_id: 'OD1234', - gender: 'feMale', - submitter_specimen_id: '87813', - specimen_type: 'sKiN', - tumour_normal_designation: 'Normal', - submitter_sample_id: 'MAS123', - sample_type: 'CTdna', - }, - ]); - chai.expect(result.validationErrors.length).to.eq(0); - chai.expect(result.processedRecords[0]).to.deep.eq({ - program_id: 'PACA-AU', - submitter_donor_id: 'OD1234', - gender: 'Female', - submitter_specimen_id: '87813', - specimen_type: 'Skin', - tumour_normal_designation: 'Normal', - submitter_sample_id: 'MAS123', - sample_type: 'ctDNA', - }); - }); - - it('should not validate if unrecognized fields are provided', () => { - const result = schemaService.processRecords(dictionary, 'donor', [ - { - program_id: 'PACA-AU', - submitter_donor_id: 'ICGC_0002', - gender: 'Other', - ethnicity: 'asian', - vital_status: 'deceased', - cause_of_death: 'died of cancer', - survival_time: '5', - hackField: 'muchHack', - }, - ]); - chai.expect(result.validationErrors.length).to.eq(1); - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.UNRECOGNIZED_FIELD, - message: `hackField is not an allowed field for this schema.`, - fieldName: 'hackField', - index: 0, - info: {}, - }); - }); - - it('should validate number/integer array with field defined ranges', () => { - const result = schemaService.processRecords(dictionary, 'favorite_things', [ - { - id: 'TH-ING', - fraction: ['0.2', '2', '3'], - integers: ['-100', '-2'], - }, - ]); - - chai.expect(result.validationErrors.length).to.eq(2); - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.INVALID_BY_RANGE, - message: 'Value is out of permissible range, it must be > 0 and <= 1.', - index: 0, - fieldName: 'fraction', - info: { value: [2, 3], max: 1, exclusiveMin: 0 }, - }); - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.INVALID_BY_RANGE, - message: 'Value is out of permissible range, it must be >= -10 and <= 10.', - index: 0, - fieldName: 'integers', - info: { value: [-100], max: 10, min: -10 }, - }); - }); - - it('should validate string array with field defined codelist', () => { - const result = schemaService.processRecords(dictionary, 'favorite_things', [ - { - id: 'TH-ING', - fruit: ['Mango', '2'], - }, - ]); - chai.expect(result.validationErrors.length).to.eq(1); - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.INVALID_ENUM_VALUE, - message: 'The value is not permissible for this field.', - fieldName: 'fruit', - index: 0, - info: { value: ['2'] }, - }); - }); - - it('should validate string with field defined codelist', () => { - const result = schemaService.processRecords(dictionary, 'favorite_things', [ - { - id: 'TH-ING', - fruit_single_value: 'Banana', - }, - ]); - chai.expect(result.validationErrors.length).to.eq(1); - chai.expect(result.validationErrors).to.deep.include({ - errorType: SchemaValidationErrorTypes.INVALID_ENUM_VALUE, - message: 'The value is not permissible for this field.', - fieldName: 'fruit_single_value', - index: 0, - info: { value: ['Banana'] }, - }); - }); - - it('should validate string array with field defined regex', () => { - const result = schemaService.processRecords(dictionary, 'favorite_things', [ - { - id: 'TH-ING', - qWords: ['que', 'not_q'], - }, - ]); - chai.expect(result.validationErrors.length).to.eq(1); - chai.expect(result.validationErrors[0]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_REGEX, - message: 'The value is not a permissible for this field, it must meet the regular expression: "^q.*$".', - fieldName: 'qWords', - index: 0, - info: { value: ['not_q'], regex: '^q.*$', examples: undefined }, - }); - }); - - it('should validate string with field defined regex', () => { - const result = schemaService.processRecords(dictionary, 'favorite_things', [ - { - id: 'TH-ING', - qWord: 'not_q', - }, - ]); - chai.expect(result.validationErrors.length).to.eq(1); - chai.expect(result.validationErrors[0]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_REGEX, - message: 'The value is not a permissible for this field, it must meet the regular expression: "^q.*$".', - fieldName: 'qWord', - index: 0, - info: { value: ['not_q'], regex: '^q.*$', examples: undefined }, - }); - }); - - it('should pass unique restriction validation when only null values exists', () => { - const result = schemaService.processRecords(dictionary, 'favorite_things', [ - { - id: 'TH-ING', - unique_value: '', - }, - ]); - chai.expect(result.validationErrors.length).to.eq(0); - }); - - it('should pass unique restriction validation when only one record exists', () => { - const result = schemaService.processRecords(dictionary, 'favorite_things', [ - { - id: 'TH-ING', - unique_value: 'unique_value_1', - }, - ]); - chai.expect(result.validationErrors.length).to.eq(0); - }); - - it('should fail unique restriction validation when duplicate values exist (scalar)', () => { - const result = schemaService.processRecords(dictionary, 'favorite_things', [ - { - id: 'ID-1', - unique_value: 'unique_value_1', - }, - { - id: 'ID-2', - unique_value: 'unique_value_1', - }, - ]); - - chai.expect(result.validationErrors.length).to.eq(2); - chai.expect(result.validationErrors[0]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE, - message: 'Values for column "unique_value" must be unique.', - fieldName: 'unique_value', - index: 0, - info: { value: ['unique_value_1'] }, - }); - chai.expect(result.validationErrors[1]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE, - message: 'Values for column "unique_value" must be unique.', - fieldName: 'unique_value', - index: 1, - info: { value: ['unique_value_1'] }, - }); - }); - it('should fail unique restriction validation when duplicate values exist (array)', () => { - const result = schemaService.processRecords(dictionary, 'favorite_things', [ - { - id: 'ID-1', - unique_value: ['unique_value_1', 'unique_value_2'], - }, - { - id: 'ID-2', - unique_value: ['unique_value_1', 'unique_value_2'], - }, - ]); - - chai.expect(result.validationErrors.length).to.eq(2); - chai.expect(result.validationErrors[0]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE, - message: 'Values for column "unique_value" must be unique.', - fieldName: 'unique_value', - index: 0, - info: { value: ['unique_value_1', 'unique_value_2'] }, - }); - chai.expect(result.validationErrors[1]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE, - message: 'Values for column "unique_value" must be unique.', - fieldName: 'unique_value', - index: 1, - info: { value: ['unique_value_1', 'unique_value_2'] }, - }); - }); - - it('should pass foreignKey restriction validation when values exist in foreign schema', () => { - const parent_schema_1_data = [ - { - id: 'parent_schema_1_id_1', - name: 'parent_schema_1_name_1', - }, - { - id: 'parent_schema_1_id_2', - name: 'parent_schema_1_name_2', - }, - ]; - - const child_schema_simple_fk_data = [ - { - id: '1', - parent_schema_1_id: 'parent_schema_1_id_1', - }, - { - id: '2', - parent_schema_1_id: 'parent_schema_1_id_2', - }, - ]; - const schemaData = { - parent_schema_1: parent_schema_1_data, - child_schema_simple_fk: child_schema_simple_fk_data, - }; - - const result = schemaService.processSchemas(dictionary, schemaData); - - chai.expect(result['parent_schema_1'].validationErrors.length).to.eq(0); - chai.expect(result['child_schema_simple_fk'].validationErrors.length).to.eq(0); - }); - - it('should pass foreignKey restriction validation when local schema has null values', () => { - const parent_schema_1_data = [ - { - id: 'parent_schema_1_id_1', - name: 'parent_schema_1_name_1', - }, - { - id: 'parent_schema_1_id_2', - name: 'parent_schema_1_name_2', - }, - ]; - - const child_schema_simple_fk_data = [ - { - id: '1', - parent_schema_1_id: 'parent_schema_1_id_1', - }, - { - id: '2', - parent_schema_1_id: '', - }, - ]; - const schemaData = { - parent_schema_1: parent_schema_1_data, - child_schema_simple_fk: child_schema_simple_fk_data, - }; - - const result = schemaService.processSchemas(dictionary, schemaData); - - chai.expect(result['parent_schema_1'].validationErrors.length).to.eq(0); - chai.expect(result['child_schema_simple_fk'].validationErrors.length).to.eq(0); - }); - - it('should pass foreignKey restriction validation when values exist in foreign schema (composite fk)', () => { - const parent_schema_1_data = [ - { - id: 'parent_schema_1_id_1', - external_id: 'parent_schema_1_external_id_1', - name: 'parent_schema_1_name_1', - }, - { - id: 'parent_schema_1_id_2', - external_id: 'parent_schema_1_external_id_2', - name: 'parent_schema_1_name_2', - }, - ]; - - const child_schema_composite_fk_data = [ - { - id: '1', - parent_schema_1_id: 'parent_schema_1_id_1', - parent_schema_1_external_id: 'parent_schema_1_external_id_1', - }, - { - id: '2', - parent_schema_1_id: 'parent_schema_1_id_2', - parent_schema_1_external_id: 'parent_schema_1_external_id_2', - }, - ]; - const schemaData = { - parent_schema_1: parent_schema_1_data, - child_schema_composite_fk: child_schema_composite_fk_data, - }; - - const result = schemaService.processSchemas(dictionary, schemaData); - - chai.expect(result['parent_schema_1'].validationErrors.length).to.eq(0); - chai.expect(result['child_schema_composite_fk'].validationErrors.length).to.eq(0); - }); - - it('should fail foreignKey restriction validation when value does not exist in foreign schema', () => { - const parent_schema_1_data = [ - { - id: 'parent_schema_1_id_1', - name: 'parent_schema_1_name_1', - }, - { - id: 'parent_schema_1_id_2', - name: 'parent_schema_1_name_2', - }, - ]; - - const child_schema_simple_fk_data = [ - { - id: '1', - parent_schema_1_id: 'parent_schema_1_id_1', - }, - { - id: '2', - parent_schema_1_id: 'non_existing_value_in_foreign_schema', - }, - ]; - const schemaData = { - parent_schema_1: parent_schema_1_data, - child_schema_simple_fk: child_schema_simple_fk_data, - }; - - const result = schemaService.processSchemas(dictionary, schemaData); - const childSchemaErrors = result['child_schema_simple_fk'].validationErrors; - - chai.expect(childSchemaErrors.length).to.eq(1); - chai.expect(childSchemaErrors[0]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_FOREIGN_KEY, - message: - 'Record violates foreign key restriction defined for field(s) parent_schema_1_id. Key parent_schema_1_id: non_existing_value_in_foreign_schema is not present in schema parent_schema_1.', - fieldName: 'parent_schema_1_id', - index: 1, - info: { foreignSchema: 'parent_schema_1', value: { parent_schema_1_id: 'non_existing_value_in_foreign_schema' } }, - }); - }); - - it('should fail foreignKey restriction validation when values do not exist in foreign schema (composite fk)', () => { - const parent_schema_1_data = [ - { - id: 'parent_schema_1_id_1', - external_id: 'parent_schema_1_external_id_1', - name: 'parent_schema_1_name_1', - }, - { - id: 'parent_schema_1_id_2', - external_id: 'parent_schema_1_external_id_2', - name: 'parent_schema_1_name_2', - }, - ]; - - const child_schema_composite_fk_data = [ - { - id: '1', - parent_schema_1_id: 'parent_schema_1_id_1', - parent_schema_1_external_id: 'parent_schema_1_external_id_1', - }, - { - id: '2', - parent_schema_1_id: 'parent_schema_1_id_2', - parent_schema_1_external_id: 'non_existing_value_in_foreign_schema', - }, - ]; - const schemaData = { - parent_schema_1: parent_schema_1_data, - child_schema_composite_fk: child_schema_composite_fk_data, - }; - - const result = schemaService.processSchemas(dictionary, schemaData); - const childSchemaErrors = result['child_schema_composite_fk'].validationErrors; - - chai.expect(childSchemaErrors.length).to.eq(1); - chai.expect(childSchemaErrors[0]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_FOREIGN_KEY, - message: - 'Record violates foreign key restriction defined for field(s) parent_schema_1_id, parent_schema_1_external_id. Key parent_schema_1_id: parent_schema_1_id_2, parent_schema_1_external_id: non_existing_value_in_foreign_schema is not present in schema parent_schema_1.', - fieldName: 'parent_schema_1_id, parent_schema_1_external_id', - index: 1, - info: { - foreignSchema: 'parent_schema_1', - value: { - parent_schema_1_external_id: 'non_existing_value_in_foreign_schema', - parent_schema_1_id: 'parent_schema_1_id_2', - }, - }, - }); - }); - - it('should fail foreignKey restriction validation when values (array) do not match in foreign schema (composite fk)', () => { - const parent_schema_2_data = [ - { - id1: ['id1_1', 'id1_2'], - id2: ['id2_1'], - }, - ]; - - const child_schema_composite_array_values_fk_data = [ - { - id: '1', - parent_schema_2_id1: ['id1_1'], - parent_schema_2_id12: ['id1_2', 'id2_1'], - }, - ]; - const schemaData = { - parent_schema_2: parent_schema_2_data, - child_schema_composite_array_values_fk: child_schema_composite_array_values_fk_data, - }; - - const result = schemaService.processSchemas(dictionary, schemaData); - const childSchemaErrors = result['child_schema_composite_array_values_fk'].validationErrors; - - chai.expect(childSchemaErrors.length).to.eq(1); - chai.expect(childSchemaErrors[0]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_FOREIGN_KEY, - message: - 'Record violates foreign key restriction defined for field(s) parent_schema_2_id1, parent_schema_2_id12. Key parent_schema_2_id1: [id1_1], parent_schema_2_id12: [id1_2, id2_1] is not present in schema parent_schema_2.', - fieldName: 'parent_schema_2_id1, parent_schema_2_id12', - index: 0, - info: { - foreignSchema: 'parent_schema_2', - value: { - parent_schema_2_id1: ['id1_1'], - parent_schema_2_id12: ['id1_2', 'id2_1'], - }, - }, - }); - }); - - it('should pass uniqueKey restriction validation when only a record exists', () => { - const result = schemaService.processRecords(dictionary, 'unique_key_schema', [ - { - numeric_id_1: '1', - string_id_2: 'string_value', - array_string_id_3: ['array_element_1', 'array_element_2'], - }, - ]); - - chai.expect(result.validationErrors.length).to.eq(0); - }); - - it('should pass uniqueKey restriction validation when values are unique', () => { - const result = schemaService.processRecords(dictionary, 'unique_key_schema', [ - { - numeric_id_1: '1', - string_id_2: 'string_value', - array_string_id_3: ['array_element_1', 'array_element_2'], - }, - { - numeric_id_1: '1', - string_id_2: 'string_value', - array_string_id_3: ['array_element_1', 'array_element_x'], - }, - ]); - - chai.expect(result.validationErrors.length).to.eq(0); - }); - - it('should fail uniqueKey restriction validation when missing values are part of the key and they are not unique', () => { - const result = schemaService.processRecords(dictionary, 'unique_key_schema', [ - { - numeric_id_1: '', - string_id_2: '', - array_string_id_3: [], - }, - { - numeric_id_1: '', - string_id_2: '', - array_string_id_3: [], - }, - ]); - - chai.expect(result.validationErrors.length).to.eq(2); - chai.expect(result.validationErrors[0]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, - message: - 'UniqueKey field values "numeric_id_1: null, string_id_2: null, array_string_id_3: null" must be unique.', - fieldName: 'numeric_id_1, string_id_2, array_string_id_3', - index: 0, - info: { - uniqueKeyFields: ['numeric_id_1', 'string_id_2', 'array_string_id_3'], - value: { - numeric_id_1: '', - string_id_2: '', - array_string_id_3: '', - }, - }, - }); - chai.expect(result.validationErrors[1]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, - message: - 'UniqueKey field values "numeric_id_1: null, string_id_2: null, array_string_id_3: null" must be unique.', - fieldName: 'numeric_id_1, string_id_2, array_string_id_3', - index: 1, - info: { - uniqueKeyFields: ['numeric_id_1', 'string_id_2', 'array_string_id_3'], - value: { - numeric_id_1: '', - string_id_2: '', - array_string_id_3: '', - }, - }, - }); - }); - - it('should fail uniqueKey restriction validation when values are not unique', () => { - const result = schemaService.processRecords(dictionary, 'unique_key_schema', [ - { - numeric_id_1: '1', - string_id_2: 'string_value', - array_string_id_3: ['array_element_1', 'array_element_2'], - }, - { - numeric_id_1: '1', - string_id_2: 'string_value', - array_string_id_3: ['array_element_1', 'array_element_2'], - }, - ]); - - chai.expect(result.validationErrors.length).to.eq(2); - chai.expect(result.validationErrors[0]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, - message: - 'UniqueKey field values "numeric_id_1: 1, string_id_2: string_value, array_string_id_3: [array_element_1, array_element_2]" must be unique.', - fieldName: 'numeric_id_1, string_id_2, array_string_id_3', - index: 0, - info: { - uniqueKeyFields: ['numeric_id_1', 'string_id_2', 'array_string_id_3'], - value: { - numeric_id_1: 1, - string_id_2: 'string_value', - array_string_id_3: ['array_element_1', 'array_element_2'], - }, - }, - }); - chai.expect(result.validationErrors[1]).to.deep.eq({ - errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, - message: - 'UniqueKey field values "numeric_id_1: 1, string_id_2: string_value, array_string_id_3: [array_element_1, array_element_2]" must be unique.', - fieldName: 'numeric_id_1, string_id_2, array_string_id_3', - index: 1, - info: { - uniqueKeyFields: ['numeric_id_1', 'string_id_2', 'array_string_id_3'], - value: { - numeric_id_1: 1, - string_id_2: 'string_value', - array_string_id_3: ['array_element_1', 'array_element_2'], - }, - }, - }); - }); -}); diff --git a/packages/client/tsconfig.build.json b/packages/client/tsconfig.build.json new file mode 100644 index 0000000..df0d74a --- /dev/null +++ b/packages/client/tsconfig.build.json @@ -0,0 +1,4 @@ +{ + "extends": "./tsconfig.json", + "exclude": ["test/**/*.ts"] +} diff --git a/packages/client/tsconfig.json b/packages/client/tsconfig.json index 651ab3a..2875267 100644 --- a/packages/client/tsconfig.json +++ b/packages/client/tsconfig.json @@ -14,8 +14,9 @@ "sourceMap": true, "declaration": true, "outDir": "dist", - "skipLibCheck": true + "skipLibCheck": true, + "noUncheckedIndexedAccess": true }, - "include": ["./src/**/*.ts"], + "include": ["./src/**/*.ts", "./test/**/*.ts"], "exclude": ["node_modules/"] } diff --git a/packages/common/src/types/defined.ts b/packages/common/src/types/defined.ts new file mode 100644 index 0000000..09cd56e --- /dev/null +++ b/packages/common/src/types/defined.ts @@ -0,0 +1,8 @@ +/** + * Remove `undefined` from a type union. + * + * @example + * type MaybeUndefined = string | number | undefined; + * type NeverUndefined = Defined; // type is `string | number` + */ +export type Defined = T extends undefined ? never : T; diff --git a/packages/common/src/types/index.ts b/packages/common/src/types/index.ts index fcd0305..bf51180 100644 --- a/packages/common/src/types/index.ts +++ b/packages/common/src/types/index.ts @@ -17,6 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +export * from './defined'; export * from './generics'; export * from './result'; export * from './singular'; diff --git a/packages/common/src/types/result.ts b/packages/common/src/types/result.ts index ea229fd..f346d2a 100644 --- a/packages/common/src/types/result.ts +++ b/packages/common/src/types/result.ts @@ -21,18 +21,13 @@ export type Success = { success: true; data: T }; export type Failure = { success: false; message: string; data: T }; -/** - * Represents a response that on success will include data of type A, - * and on failure will return data of type B - */ -export type Either = Success | Failure; - /** * Represents a response that on success will include data of type T, * otherwise a message will be returned in place of the data explaining the failure. - * The failure object has data type of void. + * + * Optionally, a data type can be provided for the failure case. */ -export type Result = Success | Failure; +export type Result = Success | Failure; /* ******************* * Convenience Methods @@ -57,12 +52,12 @@ export const failure = (message: string): Failure => ({ }); /** - * Create a Fallback response for the Either type which includes the fallback data + * Create a failure response with data. If the Result expects failure data, use this function instead of the default `failure` * @param {T} data * @returns {Failure} `{success: false, message, data}` */ -export const alternate = (data: T, message: string): Failure => ({ +export const failWith = (message: string, data: T): Failure => ({ success: false, - data, message, + data, }); diff --git a/packages/common/src/types/singular.ts b/packages/common/src/types/singular.ts index 06c3f1c..1a7d33b 100644 --- a/packages/common/src/types/singular.ts +++ b/packages/common/src/types/singular.ts @@ -9,8 +9,9 @@ * type TupleContents = Singular<[number, string, boolean]> // `TupleContents` is `number | string | boolean` * type ArrayOfTuplesContents = Singular<[number, string, boolean]> // `ArrayOfTuplesContents` is `number | string | boolean` */ -export type Singular = T extends Array - ? Singular - : T extends ReadonlyArray - ? Singular - : T; +export type Singular = + T extends Array + ? Singular + : T extends ReadonlyArray + ? Singular + : T; diff --git a/packages/common/src/utils/typeUtils.ts b/packages/common/src/utils/typeUtils.ts index 00bd24e..7f404d0 100644 --- a/packages/common/src/utils/typeUtils.ts +++ b/packages/common/src/utils/typeUtils.ts @@ -25,3 +25,16 @@ * @return an array */ export const asArray = (val: T | T[]): T[] => (Array.isArray(val) ? val : [val]); + +/** + * Checks that the input does not equal undefined (and lets the type checker know). + * + * Useful for filtering undefined values out of lists. + * + * (input) => input !== undefined + * + * @example + * const combinedArray: Array = ['hello', undefined, 'world']; + * const stringArray = combinedArray.filter(isDefined); // type is: Array + */ +export const isDefined = (input: T | undefined) => input !== undefined; diff --git a/packages/dictionary/src/constants.ts b/packages/dictionary/src/constants.ts new file mode 100644 index 0000000..6795b1c --- /dev/null +++ b/packages/dictionary/src/constants.ts @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2023 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +export const DEFAULT_DELIMITER = ','; diff --git a/packages/dictionary/src/index.ts b/packages/dictionary/src/index.ts index dfb2122..27be2f7 100644 --- a/packages/dictionary/src/index.ts +++ b/packages/dictionary/src/index.ts @@ -1,3 +1,4 @@ +export * from './constants'; export * as DiffUtils from './diff'; export * from './references'; export * from './types'; diff --git a/packages/dictionary/src/types/commonTypes.ts b/packages/dictionary/src/types/commonTypes.ts new file mode 100644 index 0000000..bfaab91 --- /dev/null +++ b/packages/dictionary/src/types/commonTypes.ts @@ -0,0 +1,18 @@ +import { z as zod } from 'zod'; + +export const Integer = zod.number().int(); + +/** + * String rules for all name fields used in dictionary, including Dictionary, Schema, and Fields. + * This validates the format of the string since names are not allowed to have `.` characters. + * + * Example Values: + * - `donors` + * - `primary-site` + * - `maximumVelocity` + */ +export const NameString = zod + .string() + .min(1, 'Name fields cannot be empty.') + .regex(/^[^.]+$/, 'Name fields cannot have `.` characters.'); +export type NameString = zod.infer; diff --git a/packages/dictionary/src/types/dataTypes.ts b/packages/dictionary/src/types/dataTypes.ts index 8ce6254..917f447 100644 --- a/packages/dictionary/src/types/dataTypes.ts +++ b/packages/dictionary/src/types/dataTypes.ts @@ -17,17 +17,26 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +import type { Singular } from 'common'; + /** * Represents a data record as taken from an input file. All values are the original strings and have not been validated into * numbers/bools or split into arrays. */ -export type UnprocessedDataRecord = Record; +export type UnprocessedDataRecord = Record; /** * The available data types for a field in a Lectern Schema. */ export type DataRecordValue = string | string[] | number | number[] | boolean | boolean[] | undefined; +export type SingleDataValue = Singular; +export type ArrayDataValue = DataRecordValue extends infer T + ? T extends undefined + ? never + : Array> + : never; + /** * Represents a data record after processing, with the data checked to be a valid type for a Lectern schema. * The type of data should match the expected type for the given field. diff --git a/packages/dictionary/src/types/dbTypes.ts b/packages/dictionary/src/types/dbTypes.ts index f198aa2..71d38ad 100644 --- a/packages/dictionary/src/types/dbTypes.ts +++ b/packages/dictionary/src/types/dbTypes.ts @@ -1,3 +1,22 @@ +/* + * Copyright (c) 2023 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + import { z as zod } from 'zod'; import { DictionaryBase } from './dictionaryTypes'; diff --git a/packages/dictionary/src/types/dictionaryTypes.ts b/packages/dictionary/src/types/dictionaryTypes.ts index 6d0cfca..42885d1 100644 --- a/packages/dictionary/src/types/dictionaryTypes.ts +++ b/packages/dictionary/src/types/dictionaryTypes.ts @@ -19,24 +19,17 @@ import { z as zod } from 'zod'; import allUnique from '../utils/allUnique'; +import { NameString } from './commonTypes'; import { ReferenceTag, References } from './referenceTypes'; - -/** - * String rules for all name fields used in dictionary, including Dictionary, Schema, and Fields. - * This validates the format of the string since names are not allowed to have `.` characters. - * - * Example Values: - * - `donors` - * - `primary-site` - * - `maximumVelocity` - */ -export const NameString = zod - .string() - .min(1, 'Name fields cannot be empty.') - .regex(/^[^.]+$/, 'Name fields cannot have `.` characters.'); -export type NameString = zod.infer; - -export const Integer = zod.number().int(); +import { + RestrictionCodeListInteger, + RestrictionCodeListNumber, + RestrictionCodeListString, + RestrictionIntegerRange, + RestrictionNumberRange, + RestrictionRegex, + RestrictionScript, +} from './restrictionsTypes'; // Meta accepts as values only strings, numbers, booleans, arrays of numbers or arrays of strings // Another Meta object can be nested inside a Meta property @@ -56,84 +49,12 @@ export const DictionaryMeta: zod.ZodType = zod.record( export const SchemaFieldValueType = zod.enum(['string', 'integer', 'number', 'boolean']); export type SchemaFieldValueType = zod.infer; -/* ************ * - * Restrictions * - * ************ */ -export const RestrictionScript = zod.array(zod.string().or(ReferenceTag)).min(1); //TODO: script formatting validation -export type RestrictionScript = zod.infer; - -export const RestrictionNumberRange = zod - .object({ - exclusiveMax: zod.number().optional(), - exclusiveMin: zod.number().optional(), - max: zod.number().optional(), - min: zod.number().optional(), - }) - .refine( - (data) => - data.exclusiveMax !== undefined || - data.max !== undefined || - data.exclusiveMin !== undefined || - data.min !== undefined, - 'Range restriction requires one of `exclusiveMax`, `exclusiveMin`, `max` or `min`.', - ) - .refine( - (data) => !(data.exclusiveMin !== undefined && data.min !== undefined), - 'Range restriction cannot have both `exclusiveMin` and `min`.', - ) - .refine( - (data) => !(data.exclusiveMax !== undefined && data.max !== undefined), - 'Range restriction cannot have both `exclusiveMax` and `max`.', - ); - -export const RestrictionIntegerRange = zod - .object({ - exclusiveMax: Integer.optional(), - exclusiveMin: Integer.optional(), - max: Integer.optional(), - min: Integer.optional(), - }) - .refine( - (data) => - data.exclusiveMax !== undefined || - data.max !== undefined || - data.exclusiveMin !== undefined || - data.min !== undefined, - 'Range restriction requires one of `exclusiveMax`, `exclusiveMin`, `max` or `min`.', - ) - .refine( - (data) => !(data.exclusiveMin !== undefined && data.min !== undefined), - 'Range restriction cannot have both `exclusiveMin` and `min`.', - ) - .refine( - (data) => !(data.exclusiveMax !== undefined && data.max !== undefined), - 'Range restriction cannot have both `exclusiveMax` and `max`.', - ); - -export const RestrictionRange = RestrictionNumberRange; -export type RestrictionRange = zod.infer; - -export const RestrictionRegex = zod.string().superRefine((data, context) => { - try { - // Attempt to build regexp from the value - RegExp(data); - } catch (e) { - // Thrown error creating regex, so we add validation issue. - const errorMessage = e instanceof Error ? e.message : `${e}`; - context.addIssue({ - code: zod.ZodIssueCode.custom, - message: `Error converting expression to Regex: ${errorMessage}`, - }); - } -}); -export type RestrictionRegex = zod.infer; - /* ****************************** * * Field Type Restriction Objects * * ****************************** */ export const StringFieldRestrictions = zod .object({ - codeList: zod.union([zod.string(), ReferenceTag]).array().min(1).or(ReferenceTag), + codeList: RestrictionCodeListString.or(ReferenceTag), required: zod.boolean(), script: RestrictionScript.or(ReferenceTag), regex: RestrictionRegex.or(ReferenceTag), @@ -144,7 +65,7 @@ export type StringFieldRestrictions = zod.infer; export const NumberFieldRestrictions = zod .object({ - codeList: zod.array(zod.number()).min(1).or(ReferenceTag), + codeList: RestrictionCodeListNumber.or(ReferenceTag), required: zod.boolean(), script: RestrictionScript.or(ReferenceTag), range: RestrictionNumberRange, @@ -155,7 +76,7 @@ export type NumberFieldRestrictions = zod.infer; export const IntegerFieldRestrictions = zod .object({ - codeList: zod.array(Integer).min(1).or(ReferenceTag), + codeList: RestrictionCodeListInteger.or(ReferenceTag), required: zod.boolean(), script: RestrictionScript.or(ReferenceTag), range: RestrictionIntegerRange, @@ -227,7 +148,7 @@ export type SchemaRestrictions = SchemaField['restrictions']; /* ****** * * Schema * * ****** */ -export const ForeignKeyRestrictionMapping = zod.object({ +export const ForeignKeyRestriction = zod.object({ schema: NameString, mappings: zod.array( zod.object({ @@ -236,6 +157,7 @@ export const ForeignKeyRestrictionMapping = zod.object({ }), ), }); +export type ForeignKeyRestriction = zod.infer; export const Schema = zod .object({ @@ -245,7 +167,7 @@ export const Schema = zod meta: DictionaryMeta.optional(), restrictions: zod .object({ - foreignKey: zod.array(ForeignKeyRestrictionMapping).min(1), + foreignKey: zod.array(ForeignKeyRestriction).min(1), uniqueKey: zod.array(NameString).min(1), }) .partial() diff --git a/packages/dictionary/src/types/index.ts b/packages/dictionary/src/types/index.ts index a946640..456f247 100644 --- a/packages/dictionary/src/types/index.ts +++ b/packages/dictionary/src/types/index.ts @@ -1,5 +1,7 @@ +export * from './commonTypes'; export * from './dataTypes'; export * from './dbTypes'; export * from './dictionaryTypes'; export * from './diffTypes'; export * from './referenceTypes'; +export * from './restrictionsTypes'; diff --git a/packages/dictionary/src/types/restrictionsTypes.ts b/packages/dictionary/src/types/restrictionsTypes.ts new file mode 100644 index 0000000..e01d0a0 --- /dev/null +++ b/packages/dictionary/src/types/restrictionsTypes.ts @@ -0,0 +1,94 @@ +import { z as zod } from 'zod'; +import { ReferenceTag } from './referenceTypes'; +import { Integer } from './commonTypes'; +import type { Values } from 'common'; + +export const FieldRestrictionTypes = { + codeList: 'codeList', + range: 'range', + required: 'required', + regex: 'regex', + script: 'script', + unique: 'unique', +} as const; +export type FieldRestrictionType = Values; + +export const RestrictionScript = zod.array(zod.string().or(ReferenceTag)).min(1); //TODO: script formatting validation +export type RestrictionScript = zod.infer; + +export const RestrictionCodeListString = zod.union([zod.string(), ReferenceTag]).array().min(1); +export type RestrictionCodeListString = zod.infer; + +export const RestrictionCodeListNumber = zod.number().array().min(1); +export type RestrictionCodeListNumber = zod.infer; + +export const RestrictionCodeListInteger = Integer.array().min(1); +export type RestrictionCodeListInteger = zod.infer; + +export type RestrictionCodeList = RestrictionCodeListString | RestrictionCodeListNumber | RestrictionCodeListInteger; + +export const RestrictionNumberRange = zod + .object({ + exclusiveMax: zod.number().optional(), + exclusiveMin: zod.number().optional(), + max: zod.number().optional(), + min: zod.number().optional(), + }) + .refine( + (data) => + data.exclusiveMax !== undefined || + data.max !== undefined || + data.exclusiveMin !== undefined || + data.min !== undefined, + 'Range restriction requires one of `exclusiveMax`, `exclusiveMin`, `max` or `min`.', + ) + .refine( + (data) => !(data.exclusiveMin !== undefined && data.min !== undefined), + 'Range restriction cannot have both `exclusiveMin` and `min`.', + ) + .refine( + (data) => !(data.exclusiveMax !== undefined && data.max !== undefined), + 'Range restriction cannot have both `exclusiveMax` and `max`.', + ); + +export const RestrictionIntegerRange = zod + .object({ + exclusiveMax: Integer.optional(), + exclusiveMin: Integer.optional(), + max: Integer.optional(), + min: Integer.optional(), + }) + .refine( + (data) => + data.exclusiveMax !== undefined || + data.max !== undefined || + data.exclusiveMin !== undefined || + data.min !== undefined, + 'Range restriction requires one of `exclusiveMax`, `exclusiveMin`, `max` or `min`.', + ) + .refine( + (data) => !(data.exclusiveMin !== undefined && data.min !== undefined), + 'Range restriction cannot have both `exclusiveMin` and `min`.', + ) + .refine( + (data) => !(data.exclusiveMax !== undefined && data.max !== undefined), + 'Range restriction cannot have both `exclusiveMax` and `max`.', + ); + +export const RestrictionRange = RestrictionNumberRange; +export type RestrictionRange = zod.infer; + +export const RestrictionRegex = zod.string().superRefine((data, context) => { + try { + // Attempt to build regexp from the value + RegExp(data); + } catch (e) { + // Thrown error creating regex, so we add validation issue. + const errorMessage = e instanceof Error ? e.message : `${e}`; + context.addIssue({ + code: zod.ZodIssueCode.custom, + message: `Error converting expression to Regex: ${errorMessage}`, + }); + } +}); +export type RestrictionRegex = zod.infer; diff --git a/packages/validation/package.json b/packages/validation/package.json index 42a12de..2ad15e3 100644 --- a/packages/validation/package.json +++ b/packages/validation/package.json @@ -4,7 +4,7 @@ "description": "Logic for validating data using a Lectern dictionary", "main": "dist/index.js", "scripts": { - "build": "pnpm build:clean && tsc", + "build": "pnpm build:clean && tsc -p ./tsconfig.build.json", "build:clean": "rimraf dist/ && mkdir dist", "test": "nyc mocha" }, diff --git a/packages/validation/src/dataRecordValidation/valueTypeValidation.ts b/packages/validation/src/dataRecordValidation/valueTypeValidation.ts deleted file mode 100644 index 12d73e7..0000000 --- a/packages/validation/src/dataRecordValidation/valueTypeValidation.ts +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved - * - * This program and the accompanying materials are made available under the terms of - * the GNU Affero General Public License v3.0. You should have received a copy of the - * GNU Affero General Public License along with this program. - * If not, see . - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT - * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER - * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -import { asArray } from 'common'; -import { REGEXP_BOOLEAN_VALUE, SchemaFieldValueType } from 'dictionary'; -import { - INVALID_VALUE_ERROR_MESSAGE, - SchemaValidationErrorTypes, - type BaseSchemaValidationError, - type ValueTypeValidationError, -} from '../types/validationErrorTypes'; -import type { UnprocessedRecordValidationFunction } from '../types/validationFunctionTypes'; -import { isEmptyString } from '../utils/isEmptyString'; -import { isDefined } from '../utils/typeUtils'; - -/** - * Test the values provided to every field in a DataRecord to find any non-array fields that - * have array values. - * @param record Data Record with original string values provided in data file - * @param index - * @param schemaFields - * @returns - */ -export const validateNonArrayFields: UnprocessedRecordValidationFunction = ( - record, - index, - schemaFields, -): ValueTypeValidationError[] => { - return schemaFields - .map((field) => { - const value = record[field.name]; - if (!field.isArray && Array.isArray(value)) { - return buildFieldValueTypeError({ fieldName: field.name, index }, { value }); - } - return undefined; - }) - .filter(isDefined); -}; - -/** - * Test the values provided to every field in a DataRecord to find any values that cannot be - * converted to the required type defined in the field schema - * @param record Data Record with original string values provided in data file - * @param index - * @param schemaFields - * @returns - */ -export const validateValueTypes: UnprocessedRecordValidationFunction = ( - record, - index, - schemaFields, -): ValueTypeValidationError[] => { - return schemaFields - .map((field) => { - const invalidValues = asArray(record[field.name]).filter( - (value) => value !== undefined && !isValidFieldType(field.valueType, value), - ); - const info = { value: invalidValues }; - - if (invalidValues.length !== 0) { - return buildFieldValueTypeError({ fieldName: field.name, index }, info); - } - return undefined; - }) - .filter(isDefined); -}; - -/** - * Check a value is valid for a given schema value type. - * @param valueType - * @param value - * @returns - */ -const isValidFieldType = (valueType: SchemaFieldValueType, value: string): boolean => { - if (isEmptyString(value)) { - return true; - } - switch (valueType) { - case SchemaFieldValueType.Values.boolean: - return isValidBoolean(value); - case SchemaFieldValueType.Values.integer: - return isValidInteger(value); - case SchemaFieldValueType.Values.number: - return isValidNumber(value); - case SchemaFieldValueType.Values.string: - // input values all start as strings so this is valid in all cases - return true; - } -}; - -const isValidInteger = (value: string): boolean => Number.isSafeInteger(Number(value)); -const isValidNumber = (value: string): boolean => isFinite(Number(value)); -const isValidBoolean = (value: string): boolean => value.trim().match(REGEXP_BOOLEAN_VALUE) !== null; - -const buildFieldValueTypeError = ( - errorData: BaseSchemaValidationError, - info: ValueTypeValidationError['info'], -): ValueTypeValidationError => { - const message = INVALID_VALUE_ERROR_MESSAGE; - return { - ...errorData, - errorType: SchemaValidationErrorTypes.INVALID_FIELD_VALUE_TYPE, - info, - message, - }; -}; diff --git a/packages/validation/src/fieldRestrictions/codeListValidation.ts b/packages/validation/src/fieldRestrictions/codeListValidation.ts deleted file mode 100644 index feec10e..0000000 --- a/packages/validation/src/fieldRestrictions/codeListValidation.ts +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved - * - * This program and the accompanying materials are made available under the terms of - * the GNU Affero General Public License v3.0. You should have received a copy of the - * GNU Affero General Public License along with this program. - * If not, see . - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT - * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER - * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -import { asArray } from 'common'; -import { - BaseSchemaValidationError, - EnumValueValidationError, - INVALID_VALUE_ERROR_MESSAGE, - SchemaValidationErrorTypes, -} from '../types/validationErrorTypes'; -import { ValidationFunction } from '../types/validationFunctionTypes'; -import { isEmptyString } from '../utils/isEmptyString'; -import { isDefined } from '../utils/typeUtils'; - -/** - * Check all values of a DataRecord pass codeList restrictions in their schema. - * @param rec - * @param index - * @param fields - * @returns - */ -export const validateCodeList: ValidationFunction = (rec, index, fields): EnumValueValidationError[] => { - return fields - .map((field) => { - if (field.restrictions && 'codeList' in field.restrictions && field.restrictions.codeList !== undefined) { - const codeList = field.restrictions.codeList; - if (!Array.isArray(codeList)) { - // codeList restriction is a string, not array. This happens when the references have not been replaced. - // We cannot proceed without the final array so we will return undefined. - return undefined; - } - - // put all values into array to standardize validation for array and non array fields - const recordFieldValues = asArray(rec[field.name]); - const invalidValues = recordFieldValues.filter((val) => !isValidEnumValue(codeList, val)); - - if (invalidValues.length !== 0) { - return buildCodeListError({ fieldName: field.name, index }, { value: invalidValues }); - } - } - return undefined; - }) - .filter(isDefined); -}; - -const buildCodeListError = ( - errorData: BaseSchemaValidationError, - info: EnumValueValidationError['info'], -): EnumValueValidationError => { - const message = INVALID_VALUE_ERROR_MESSAGE; - - return { - ...errorData, - errorType: SchemaValidationErrorTypes.INVALID_ENUM_VALUE, - info, - message, - }; -}; - -/** - * If value exists, confirm that it matches an option in the provided code list - * @param codeList - * @param value - * @returns - */ -const isValidEnumValue = (codeList: string[] | number[], value: string | boolean | number | undefined) => { - // do not run validation on empty values - if (value === undefined || (typeof value === 'string' && isEmptyString(value))) { - return true; - } - - return codeList.some((allowedValue) => allowedValue === value); -}; diff --git a/packages/validation/src/fieldRestrictions/rangeValidation.ts b/packages/validation/src/fieldRestrictions/rangeValidation.ts deleted file mode 100644 index 9545a65..0000000 --- a/packages/validation/src/fieldRestrictions/rangeValidation.ts +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved - * - * This program and the accompanying materials are made available under the terms of - * the GNU Affero General Public License v3.0. You should have received a copy of the - * GNU Affero General Public License along with this program. - * If not, see . - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT - * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER - * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -import { asArray } from 'common'; -import { RestrictionRange } from 'dictionary'; -import { - BaseSchemaValidationError, - RangeValidationError, - SchemaValidationErrorTypes, -} from '../types/validationErrorTypes'; -import { ValidationFunction } from '../types/validationFunctionTypes'; -import { isDefined, isNumberArray } from '../utils/typeUtils'; -import { rangeToSymbol } from '../utils/rangeToSymbol'; - -/** - * Check all values of a DataRecord pass range restrictions in their schema. - * @param record - * @param index - * @param schemaFields - * @returns - */ -export const validateRange: ValidationFunction = (record, index, schemaFields): RangeValidationError[] => { - return schemaFields - .map((field) => { - const recordFieldValues = asArray(record[field.name]); - if (!isNumberArray(recordFieldValues)) { - return undefined; - } - - const range = field.restrictions && 'range' in field.restrictions ? field.restrictions.range : undefined; - if (range === undefined) { - return undefined; - } - - const invalidValues = recordFieldValues.filter((value) => isOutOfRange(range, value)); - if (invalidValues.length !== 0) { - const info = { value: invalidValues, ...range }; - return buildRangeError({ fieldName: field.name, index }, info); - } - return undefined; - }) - .filter(isDefined); -}; - -const buildRangeError = ( - errorData: BaseSchemaValidationError, - info: RangeValidationError['info'], -): RangeValidationError => { - const message = `Value is out of permissible range, it must be ${rangeToSymbol(info)}.`; - - return { - ...errorData, - errorType: SchemaValidationErrorTypes.INVALID_BY_RANGE, - info, - message, - }; -}; - -const isOutOfRange = (range: RestrictionRange, value: number | undefined) => { - if (value === undefined) return false; - const invalidRange = - // less than the min if defined ? - (range.min !== undefined && value < range.min) || - (range.exclusiveMin !== undefined && value <= range.exclusiveMin) || - // bigger than max if defined ? - (range.max !== undefined && value > range.max) || - (range.exclusiveMax !== undefined && value >= range.exclusiveMax); - return invalidRange; -}; diff --git a/packages/validation/src/fieldRestrictions/regexValidation.ts b/packages/validation/src/fieldRestrictions/regexValidation.ts deleted file mode 100644 index 430803e..0000000 --- a/packages/validation/src/fieldRestrictions/regexValidation.ts +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved - * - * This program and the accompanying materials are made available under the terms of - * the GNU Affero General Public License v3.0. You should have received a copy of the - * GNU Affero General Public License along with this program. - * If not, see . - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT - * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER - * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -import { SchemaFieldValueType } from 'dictionary'; -import { - BaseSchemaValidationError, - RegexValidationError, - SchemaValidationErrorTypes, -} from '../types/validationErrorTypes'; -import { ValidationFunction } from '../types/validationFunctionTypes'; -import { asArray } from 'common'; -import { isDefined, isStringArray } from '../utils/typeUtils'; -import { isEmptyString } from '../utils/isEmptyString'; - -/** - * Check all values of a DataRecord pass regex restrictions in their schema. - * @param record - * @param index - * @param fields - * @returns - */ -export const validateRegex: ValidationFunction = (record, index, fields): RegexValidationError[] => { - return fields - .map((field) => { - if (field.valueType === SchemaFieldValueType.Values.string && field.restrictions && field.restrictions.regex) { - const regex = field.restrictions.regex; - const recordFieldValues = asArray(record[field.name]); - if (!isStringArray(recordFieldValues)) { - // This field value should be string or string array, we will skip validation if the type is wrong. - return undefined; - } - - const invalidValues = recordFieldValues.filter((v) => isInvalidRegexValue(regex, v)); - if (invalidValues.length !== 0) { - const examples = typeof field.meta?.examples === 'string' ? field.meta.examples : undefined; - - return buildRegexError({ fieldName: field.name, index }, { value: invalidValues, regex, examples }); - } - } - - // Field does not have regex validation - return undefined; - }) - .filter(isDefined); -}; - -const isInvalidRegexValue = (regex: string, value: string) => { - // optional field if the value is absent at this point - if (isEmptyString(value)) { - return false; - } - const regexPattern = new RegExp(regex); - return !regexPattern.test(value); -}; - -const buildRegexError = ( - errorData: BaseSchemaValidationError, - info: RegexValidationError['info'], -): RegexValidationError => { - const examplesMessage = info.examples ? ` Examples: ${info.examples}` : ''; - const message = `The value is not a permissible for this field, it must meet the regular expression: "${info.regex}".${examplesMessage}`; - - return { - ...errorData, - errorType: SchemaValidationErrorTypes.INVALID_BY_REGEX, - info, - message, - }; -}; diff --git a/packages/validation/src/fieldRestrictions/requiredValidation.ts b/packages/validation/src/fieldRestrictions/requiredValidation.ts deleted file mode 100644 index 540e7d4..0000000 --- a/packages/validation/src/fieldRestrictions/requiredValidation.ts +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved - * - * This program and the accompanying materials are made available under the terms of - * the GNU Affero General Public License v3.0. You should have received a copy of the - * GNU Affero General Public License along with this program. - * If not, see . - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT - * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER - * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -import { DataRecord, SchemaField } from 'dictionary'; -import { - BaseSchemaValidationError, - MissingRequiredFieldValidationError, - SchemaValidationErrorTypes, -} from '../types/validationErrorTypes'; -import { ValidationFunction } from '../types/validationFunctionTypes'; -import { isDefined } from '../utils/typeUtils'; -import { asArray } from 'common'; -import { isEmptyString } from '../utils/isEmptyString'; - -/** - * Check all values of a DataRecord pass required restrictions in their schema. - * @param record - * @param index - * @param fields - * @returns - */ -export const validateRequiredFields: ValidationFunction = ( - record, - index, - fields, -): MissingRequiredFieldValidationError[] => { - return fields - .map((field) => { - if (isRequiredMissing(field, record)) { - return buildRequiredError({ fieldName: field.name, index }, {}); - } - return undefined; - }) - .filter(isDefined); -}; - -const buildRequiredError = ( - errorData: BaseSchemaValidationError, - info: MissingRequiredFieldValidationError['info'], -): MissingRequiredFieldValidationError => { - const message = `${errorData.fieldName} is a required field.`; - - return { - ...errorData, - errorType: SchemaValidationErrorTypes.MISSING_REQUIRED_FIELD, - info, - message, - }; -}; - -const isRequiredMissing = (field: SchemaField, record: DataRecord) => { - const isRequired = field.restrictions && field.restrictions.required; - if (!isRequired) { - return false; - } - - // a required field is missing if there are is no value provided for this field (or if an array, all array values are empty) - return asArray(record[field.name]).every( - (item) => item === undefined || (typeof item === 'string' && isEmptyString(item)), - ); -}; diff --git a/packages/validation/src/fieldRestrictions/scriptValidation.ts b/packages/validation/src/fieldRestrictions/scriptValidation.ts deleted file mode 100644 index d002d84..0000000 --- a/packages/validation/src/fieldRestrictions/scriptValidation.ts +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved - * - * This program and the accompanying materials are made available under the terms of - * the GNU Affero General Public License v3.0. You should have received a copy of the - * GNU Affero General Public License along with this program. - * If not, see . - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT - * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER - * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -import { DataRecord, SchemaField } from 'dictionary'; -import vm from 'vm'; -import { - BaseSchemaValidationError, - SchemaValidationErrorTypes, - ScriptValidationError, -} from '../types/validationErrorTypes'; -import { ValidationFunction } from '../types/validationFunctionTypes'; -import { isDefined } from '../utils/typeUtils'; -import { asArray } from 'common'; - -const ctx = vm.createContext(); - -/** - * Check all values of a DataRecord pass all script restrictions in their schema. - * This will run all script restrictions from the provided inside a Node VM context. - * - * Running code in teh VM context will protect the global Node data context from interactions - * with the schema script, either being read or written. - * - * @param record - * @param index - * @param fields - * @returns - */ -export const validateScript: ValidationFunction = (record, index, fields) => { - return fields - .map((field) => { - if (field.restrictions && field.restrictions.script) { - const scriptResult = validateWithScript(field, record); - if (!scriptResult.valid) { - return buildScriptError( - { fieldName: field.name, index }, - { - message: scriptResult.message, - value: record[field.name], - }, - ); - } - } - return undefined; - }) - .filter(isDefined); -}; - -const buildScriptError = ( - errorData: BaseSchemaValidationError, - info: ScriptValidationError['info'], -): ScriptValidationError => { - const message = info.message || `${errorData.fieldName} was invalid based on a script restriction.`; - - return { - ...errorData, - errorType: SchemaValidationErrorTypes.INVALID_BY_SCRIPT, - info, - message, - }; -}; - -const getScript = (scriptString: string) => { - const script = new vm.Script(scriptString); - return script; -}; - -const validateWithScript = ( - field: SchemaField, - record: DataRecord, -): { - valid: boolean; - message: string; -} => { - try { - const args = { - $row: record, - $field: record[field.name], - $name: field.name, - }; - - if (!field.restrictions || !field.restrictions.script) { - throw new Error('called validation by script without script provided'); - } - - // scripts should already be strings inside arrays, but ensure that they are to help transition between lectern versions - // checking for this can be removed in future versions of lectern (feb 2020) - const scripts = asArray(field.restrictions.script); - - let result: { - valid: boolean; - message: string; - } = { - valid: false, - message: '', - }; - - for (const scriptString of scripts) { - const script = getScript(scriptString); - const valFunc = script.runInContext(ctx); - if (!valFunc) throw new Error('Invalid script'); - result = valFunc(args); - /* Return the first script that's invalid. Otherwise result will be valid with message: 'ok'*/ - if (!result.valid) break; - } - - return result; - } catch (err) { - console.error( - `failed running validation script ${field.name} for record: ${JSON.stringify(record)}. Error message: ${err}`, - ); - return { - valid: false, - message: 'failed to run script validation, check script and the input', - }; - } -}; diff --git a/packages/validation/src/index.ts b/packages/validation/src/index.ts index 27d8aec..6a15b40 100644 --- a/packages/validation/src/index.ts +++ b/packages/validation/src/index.ts @@ -17,14 +17,9 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -export * from './dataRecordValidation/fieldNamesValidation'; -export * from './dataRecordValidation/valueTypeValidation'; -export * from './fieldRestrictions/codeListValidation'; -export * from './fieldRestrictions/rangeValidation'; -export * from './fieldRestrictions/regexValidation'; -export * from './fieldRestrictions/requiredValidation'; -export * from './fieldRestrictions/scriptValidation'; -export * from './schemaRestrictions/foreignKeysValidation'; -export * from './schemaRestrictions/uniqueKeyValidation'; -export * from './schemaRestrictions/uniqueValidation'; +export * from './parseValues'; +export * from './validateDictionary'; +export * from './validateField'; +export * from './validateRecord'; +export * from './validateSchema'; export * from './types'; diff --git a/packages/validation/src/parseValues/ParseValuesResult.ts b/packages/validation/src/parseValues/ParseValuesResult.ts new file mode 100644 index 0000000..c8b561a --- /dev/null +++ b/packages/validation/src/parseValues/ParseValuesResult.ts @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import type { DataRecord } from 'dictionary'; +import type { RecordValidationErrorInvalidValue, RecordValidationErrorUnrecognizedField } from '../validateRecord'; +import type { SchemaRecordError } from '../validateSchema'; +import type { Result } from 'common'; + +export type ParseFieldError = RecordValidationErrorInvalidValue | RecordValidationErrorUnrecognizedField; + +export type ParseRecordFailureData = { record: DataRecord; errors: ParseFieldError[] }; +export type ParseRecordResult = Result<{ record: DataRecord }, ParseRecordFailureData>; + +export type ParseSchemaError = SchemaRecordError; +export type ParseSchemaFailureData = { records: DataRecord[]; errors: ParseSchemaError[] }; +export type ParseSchemaResult = Result<{ records: DataRecord[] }, ParseSchemaFailureData>; + +export type ParseDictionaryErrorUnrecognizedSchema = { reason: 'UNRECOGNIZED_SCHEMA' }; +export type ParseDictionaryErrorInvalidRecords = { + reason: 'INVALID_RECORDS'; + errors: ParseSchemaError[]; +}; + +export type ParseDictionaryFailure = { records: DataRecord[] } & ( + | ParseDictionaryErrorUnrecognizedSchema + | ParseDictionaryErrorInvalidRecords +); + +export type ParseDictionaryData = Record>; + +export type ParseDictionaryResult = Result; diff --git a/packages/validation/src/parseValues/index.ts b/packages/validation/src/parseValues/index.ts new file mode 100644 index 0000000..56c8e3e --- /dev/null +++ b/packages/validation/src/parseValues/index.ts @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +export * from './ParseValuesResult'; +export * from './parseValues'; diff --git a/packages/validation/src/parseValues/matchCodeListFormatting.ts b/packages/validation/src/parseValues/matchCodeListFormatting.ts new file mode 100644 index 0000000..8d0d082 --- /dev/null +++ b/packages/validation/src/parseValues/matchCodeListFormatting.ts @@ -0,0 +1,42 @@ +import type { SchemaField } from 'dictionary'; + +/** + * Given a string value, look for any matching values in code list restrictions and return that + * value. This is used by the convertValue functions to ensure the value returned matches the letter + * cases of the corresponding value in the code list. + * + * @example + * // Given a field `fieldWithCodeList` that has a code list restriction `["Apple", "Banana"]` + * const originalValue = 'banana'; + * const matchingValue = matchCodeListFormatting(originalValue, fieldWithCodeList); + * + * // matchingValue will equal `Banana`; + * + * @param value + * @param fieldDefinition + * @returns + */ +export function matchCodeListFormatting(value: string, fieldDefinition: SchemaField): string { + const { valueType, restrictions } = fieldDefinition; + + if (valueType === 'string') { + const codeList = restrictions?.codeList; + if (Array.isArray(codeList)) { + // We have found a code list to compare to! + + // prepare the value for comparison by making it lower case + const candidate = value.toLowerCase(); + + // look for a match + const match = codeList.find((option) => option.trim().toLowerCase() === candidate); + + if (match !== undefined) { + // we have a match! return it! + return match; + } + } + } + + // no match was found, return original value + return value; +} diff --git a/packages/validation/src/parseValues/parseValues.ts b/packages/validation/src/parseValues/parseValues.ts new file mode 100644 index 0000000..b076dde --- /dev/null +++ b/packages/validation/src/parseValues/parseValues.ts @@ -0,0 +1,283 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { failure, failWith, success, type Result } from 'common'; +import { + DEFAULT_DELIMITER, + type ArrayDataValue, + type DataRecord, + type DataRecordValue, + type Dictionary, + type RestrictionCodeListString, + type Schema, + type SchemaField, + type SchemaFieldValueType, + type UnprocessedDataRecord, +} from 'dictionary'; +import { isInteger, isNumber } from '../utils/typeUtils'; +import type { + ParseFieldError, + ParseDictionaryData, + ParseDictionaryFailure, + ParseDictionaryResult, + ParseRecordResult, + ParseSchemaResult, + ParseSchemaError, +} from './ParseValuesResult'; +import { matchCodeListFormatting } from './matchCodeListFormatting'; + +/* === Type Specific conversion functions === */ +// Note: These are intended to be passed only normalized values that have already passed through the + +const convertString = (value: string): Result => { + if (value === '') { + return failure('Empty value.'); + } + + return success(value); +}; +const convertNumber = (value: string): Result => { + const errorText = 'Not a valid number.'; + if (value === '') { + return failure(errorText); + } + const output = Number(value); + return isNumber(output) ? success(output) : failure(errorText); +}; +const convertInteger = (value: string): Result => { + const errorText = 'Not a valid number.'; + if (value === '') { + return failure(errorText); + } + const output = Number(value); + return isInteger(output) ? success(output) : failure(errorText); +}; +const convertBoolean = (value: string): Result => { + const formatted = value.toLowerCase(); + return formatted === 'true' + ? success(true) + : formatted === 'false' + ? success(false) + : failure('Not a valid boolean.'); +}; +const fieldConverters = { + boolean: convertBoolean, + integer: convertInteger, + number: convertNumber, + string: convertString, +} as const satisfies Record; + +/* === Convert functions for single and array values === */ + +/** + * Clean up input string before converting. Performs the following transformations: + * 1. Trims value + * @param value + * @returns + */ +const normalizeValue = (value: string): string => { + const trimmed = value.trim(); + return trimmed; +}; + +const convertValue = (value: string, fieldDefinition: SchemaField) => { + const normalizedValue = normalizeValue(value); + + // Empty values are treated as `undefined` + if (normalizedValue === '') { + return success(undefined); + } + const result = fieldConverters[fieldDefinition.valueType](normalizedValue); + + if (result.success && fieldDefinition.valueType === 'string') { + // extra step to ensure strings match the formatting of codeList values, if they have one + return success(matchCodeListFormatting(normalizedValue, fieldDefinition)); + } + + return result; +}; + +const convertArrayValue = (value: string, fieldDefinition: SchemaField): Result => { + /** + * Locally scoped function that will use a function from `fieldConverters` to convert every value in the array. + * */ + const convertSplitValues = ( + splitValues: string[], + conversionFunction: (value: string) => Result, + ): Result => { + const results = splitValues.map(normalizeValue).map(conversionFunction); + const output: T[] = []; + for (const result of results) { + if (!result.success) { + return failure('One or more values in array could not be converted.'); + } + output.push(result.data); + } + return success(output); + }; + + /* === Start of convertArrayValue logic === */ + const { valueType } = fieldDefinition; + const delimiter = DEFAULT_DELIMITER; + + const normalizedValue = normalizeValue(value); + if (normalizedValue === '') { + return success([]); + } + + const splitValues = normalizedValue.split(delimiter); + // Behold a usless switch, the whole purpose of which is to convince TS that the dataType has a single, known value, + // and therefore the output string has a single, known value. + switch (valueType) { + case 'boolean': { + return convertSplitValues(splitValues, fieldConverters[valueType]); + } + case 'integer': { + return convertSplitValues(splitValues, fieldConverters[valueType]); + } + case 'number': { + return convertSplitValues(splitValues, fieldConverters[valueType]); + } + case 'string': { + const result = convertSplitValues(splitValues, fieldConverters[valueType]); + if (result.success) { + const formattedValues = result.data.map((value) => matchCodeListFormatting(value, fieldDefinition)); + return success(formattedValues); + } + return result; + } + } +}; + +/** + * Parse the string value for a field and convert it to the type defined in the fieldDefinition. + * + * If the field is an array, this will split the field into separate values and attempt to convert each of those values. + * If any of the values in the array cannot be converted, the entire value parsing process will fail. + */ +export function parseFieldValue(value: string, fieldDefinition: SchemaField): Result { + const { isArray } = fieldDefinition; + return isArray ? convertArrayValue(value, fieldDefinition) : convertValue(value, fieldDefinition); +} + +/** + * Parse string values and convert them to properly typed values for fields from their schema definition. + * + * If there are any type errors found during conversion, this will return a failed `Result` + * with a list of the `ConvertTypeErrors`. + * + * If a field is in the record that is not found in the schema, this will return a faile `Result` + * indicating the unrecognized field. Its value will remain a string without any conversion. + */ +export function parseRecordValues(record: UnprocessedDataRecord, schema: Schema): ParseRecordResult { + const errors: ParseFieldError[] = []; + const output: DataRecord = {}; + + for (const [fieldName, stringValue] of Object.entries(record)) { + const fieldDefinition = schema.fields.find((field) => field.name === fieldName); + if (!fieldDefinition) { + // can't convert this field since no definition provided, we will put it in the output unconverted. + output[fieldName] = stringValue; + errors.push({ + reason: 'UNRECOGNIZED_FIELD', + fieldName, + fieldValue: stringValue, + }); + continue; + } + const convertResult = parseFieldValue(stringValue, fieldDefinition); + + if (convertResult.success) { + output[fieldName] = convertResult.data; + } else { + errors.push({ + reason: 'INVALID_VALUE_TYPE', + fieldName, + fieldValue: stringValue, + isArray: !!fieldDefinition.isArray, + valueType: fieldDefinition.valueType, + }); + // add original string value as value for field in record since we couldnt convert + output[fieldName] = stringValue; + } + } + if (errors.length) { + return failWith(`Errors were found while parsing record data.`, { + record: output, + errors, + }); + } + return success({ record: output }); +} + +/** + * + */ +export function parseSchemaValues(records: UnprocessedDataRecord[], schema: Schema): ParseSchemaResult { + const output: DataRecord[] = []; + const errors: ParseSchemaError[] = []; + + records.forEach((record, recordIndex) => { + const conversionResult = parseRecordValues(record, schema); + + output.push(conversionResult.data.record); + if (!conversionResult.success) { + errors.push({ + recordIndex, + recordErrors: conversionResult.data.errors, + }); + } + }); + + return errors.length + ? failWith(`Errors were found while parsing schema data.`, { records: output, errors }) + : success({ records: output }); +} + +/** + * + */ +export function parseDictionaryValues( + schemaData: Record, + dictionary: Dictionary, +): ParseDictionaryResult { + const output: ParseDictionaryData = {}; + for (const [schemaName, records] of Object.entries(schemaData)) { + const schema = dictionary.schemas.find((schema) => schema.name === schemaName); + + if (schema) { + const result = parseSchemaValues(records, schema); + if (result.success) { + output[schemaName] = result; + } else { + output[schemaName] = failWith(result.message, { + reason: 'INVALID_RECORDS', + ...result.data, + }); + } + } else { + output[schemaName] = failWith(`There is no schema of this name in the dictionary.`, { + reason: 'UNRECOGNIZED_SCHEMA', + records, + }); + } + } + const allSucceeded = Object.values(output).every((result) => result.success); + return allSucceeded ? success(output) : failWith(`Errors were found while parsing dictionary data.`, output); +} diff --git a/packages/validation/src/schemaRestrictions/foreignKeysValidation.ts b/packages/validation/src/schemaRestrictions/foreignKeysValidation.ts deleted file mode 100644 index 8d2a935..0000000 --- a/packages/validation/src/schemaRestrictions/foreignKeysValidation.ts +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved - * - * This program and the accompanying materials are made available under the terms of - * the GNU Affero General Public License v3.0. You should have received a copy of the - * GNU Affero General Public License along with this program. - * If not, see . - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT - * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER - * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -import { differenceWith, isEqual } from 'lodash'; -import { DataRecord } from 'dictionary'; -import { ForeignKeyValidationError, SchemaValidationErrorTypes } from '../types/validationErrorTypes'; -import { CrossSchemaValidationFunction } from '../types/validationFunctionTypes'; -import { selectFieldsFromDataset } from '../utils/datasetUtils'; - -/** - * Validate all foreign key restrictions in a Schema - * @param schema - * @param data - * @returns - */ -export const validateForeignKeys: CrossSchemaValidationFunction = (schema, data): ForeignKeyValidationError[] => { - const errors: Array = []; - const foreignKeyDefinitions = schema?.restrictions?.foreignKey; - if (foreignKeyDefinitions) { - foreignKeyDefinitions.forEach((foreignKeyDefinition) => { - const localSchemaData = data[schema.name] || []; - const foreignSchemaData = data[foreignKeyDefinition.schema] || []; - - // A foreign key can have more than one field, in which case is a composite foreign key. - const localFields = foreignKeyDefinition.mappings.map((x) => x.local); - const foreignFields = foreignKeyDefinition.mappings.map((x) => x.foreign); - - const fieldsMappings = new Map(foreignKeyDefinition.mappings.map((x) => [x.foreign, x.local])); - - // Select the keys of the datasets to compare. The keys are records to support the scenario where the fk is composite. - const localValues: [number, DataRecord][] = selectFieldsFromDataset(localSchemaData, localFields); - const foreignValues: [number, DataRecord][] = selectFieldsFromDataset(foreignSchemaData, foreignFields); - - // This artificial record in foreignValues allows null references in localValues to be valid. - const emptyRow: Record = {}; - foreignFields.forEach((field) => (emptyRow[field] = '')); - foreignValues.push([-1, emptyRow]); - - const missingForeignKeys = findMissingForeignKeys(localValues, foreignValues, fieldsMappings); - - missingForeignKeys.forEach((record) => { - const index = record[0]; - const info: ForeignKeyValidationError['info'] = { - value: record[1], - foreignSchema: foreignKeyDefinition.schema, - }; - const errorFieldName = localFields.join(', '); - errors.push({ - errorType: SchemaValidationErrorTypes.INVALID_BY_FOREIGN_KEY, - fieldName: errorFieldName, - index, - info, - message: getForeignKeyErrorMessage({ - fieldName: errorFieldName, - foreignSchema: foreignKeyDefinition.schema, - value: record[1], - }), - }); - }); - }); - } - return errors; -}; - -function getForeignKeyErrorMessage(errorData: { value: DataRecord; foreignSchema: string; fieldName: string }) { - const valueEntries = Object.entries(errorData.value); - const formattedKeyValues: string[] = valueEntries.map(([key, value]) => { - if (Array.isArray(value)) { - return `${key}: [${value.join(', ')}]`; - } else { - return `${key}: ${value}`; - } - }); - const valuesAsString = formattedKeyValues.join(', '); - const detail = `Key ${valuesAsString} is not present in schema ${errorData.foreignSchema}`; - const msg = `Record violates foreign key restriction defined for field(s) ${errorData.fieldName}. ${detail}.`; - return msg; -} - -/** - * Find missing foreign keys by calculating the difference between 2 dataset keys (similar to a set difference). - * Returns rows in `dataKeysA` which are not present in `dataKeysB`. - * @param datasetKeysA Keys of the dataset A. The returned value of this function is a subset of this array. - * @param datasetKeysB Keys of the dataset B. Elements to be substracted from `datasetKeysA`. - * @param fieldsMapping Mapping of the field names so the keys can be compared correctly. - */ -const findMissingForeignKeys = ( - datasetKeysA: [number, DataRecord][], - datasetKeysB: [number, DataRecord][], - fieldsMapping: Map, -): [number, DataRecord][] => { - const diff = differenceWith(datasetKeysA, datasetKeysB, (a, b) => - isEqual(a[1], renameProperties(b[1], fieldsMapping)), - ); - return diff; -}; - -/** - * Renames properties in a record using a mapping between current and new names. - * @param record The record whose properties should be renamed. - * @param fieldsMapping A mapping of current property names to new property names. - * @returns A new record with the properties' names changed according to the mapping. - */ -const renameProperties = (record: DataRecord, fieldsMapping: Map): DataRecord => { - const renamed: DataRecord = {}; - Object.entries(record).forEach(([propertyName, propertyValue]) => { - const newName = fieldsMapping.get(propertyName) ?? propertyName; - renamed[newName] = propertyValue; - }); - return renamed; -}; diff --git a/packages/validation/src/schemaRestrictions/uniqueKeyValidation.ts b/packages/validation/src/schemaRestrictions/uniqueKeyValidation.ts deleted file mode 100644 index e841425..0000000 --- a/packages/validation/src/schemaRestrictions/uniqueKeyValidation.ts +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved - * - * This program and the accompanying materials are made available under the terms of - * the GNU Affero General Public License v3.0. You should have received a copy of the - * GNU Affero General Public License along with this program. - * If not, see . - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT - * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER - * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -import { - BaseSchemaValidationError, - SchemaValidationErrorTypes, - UniqueKeyValidationError, -} from '../types/validationErrorTypes'; -import { DatasetValidationFunction } from '../types/validationFunctionTypes'; -import { findDuplicateKeys, selectFieldsFromDataset } from '../utils/datasetUtils'; - -export const validateUniqueKey: DatasetValidationFunction = (dataset, schema): UniqueKeyValidationError[] => { - const errors: Array = []; - const uniqueKeyRestriction = schema?.restrictions?.uniqueKey; - if (uniqueKeyRestriction) { - const uniqueKeyFields: string[] = uniqueKeyRestriction; - const keysToValidate = selectFieldsFromDataset(dataset, uniqueKeyFields); - const duplicateKeys = findDuplicateKeys(keysToValidate); - - duplicateKeys.forEach(([index, record]) => { - const info = { uniqueKeyFields: uniqueKeyFields, value: record }; - errors.push(buildUniqueKeyError({ fieldName: uniqueKeyFields.join(', '), index }, info)); - }); - } - return errors; -}; - -const buildUniqueKeyError = ( - errorData: BaseSchemaValidationError, - info: UniqueKeyValidationError['info'], -): UniqueKeyValidationError => { - const uniqueKeyFields = info.uniqueKeyFields; - const record = info.value; - const formattedKeyValues = uniqueKeyFields.map((fieldName) => { - if (fieldName in record) { - const value = record[fieldName]; - if (Array.isArray(value)) { - return `${fieldName}: [${value.join(', ')}]`; - } else { - return `${fieldName}: ${value === '' ? 'null' : value}`; - } - } - }); - const valuesAsString = formattedKeyValues.join(', '); - const message = `UniqueKey field values "${valuesAsString}" must be unique.`; - - return { - ...errorData, - errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, - info, - message, - }; -}; diff --git a/packages/validation/src/schemaRestrictions/uniqueValidation.ts b/packages/validation/src/schemaRestrictions/uniqueValidation.ts deleted file mode 100644 index 5443d66..0000000 --- a/packages/validation/src/schemaRestrictions/uniqueValidation.ts +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved - * - * This program and the accompanying materials are made available under the terms of - * the GNU Affero General Public License v3.0. You should have received a copy of the - * GNU Affero General Public License along with this program. - * If not, see . - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT - * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER - * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -import { - BaseSchemaValidationError, - SchemaValidationErrorTypes, - UniqueValidationError, -} from '../types/validationErrorTypes'; -import { DatasetValidationFunction } from '../types/validationFunctionTypes'; -import { findDuplicateKeys, selectFieldsFromDataset } from '../utils/datasetUtils'; - -/** - * Validate all unique field restrictions in a schema. This will find all records that have duplicate - * values for fields that are restricted to being unique. - * @param data - * @param schema - * @returns - */ -export const validateUnique: DatasetValidationFunction = (data, schema): UniqueValidationError[] => { - const errors: Array = []; - schema.fields.forEach((field) => { - const unique = field.restrictions?.unique || undefined; - if (!unique) return undefined; - const keysToValidate = selectFieldsFromDataset(data, [field.name]); - const duplicateKeys = findDuplicateKeys(keysToValidate); - - duplicateKeys.forEach(([index, record]) => { - const info = { value: record[field.name] }; - errors.push(buildUniqueError({ fieldName: field.name, index }, info)); - }); - }); - return errors; -}; - -const buildUniqueError = ( - errorData: BaseSchemaValidationError, - info: UniqueValidationError['info'], -): UniqueValidationError => { - const message = `Values for column "${errorData.fieldName}" must be unique.`; - - return { - ...errorData, - errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE, - info, - message, - }; -}; diff --git a/packages/validation/src/types/index.ts b/packages/validation/src/types/index.ts index ba34998..f6f6f48 100644 --- a/packages/validation/src/types/index.ts +++ b/packages/validation/src/types/index.ts @@ -17,5 +17,4 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -export * from './validationErrorTypes'; -export * from './validationFunctionTypes'; +export * from './testResult'; diff --git a/packages/validation/src/types/testResult.ts b/packages/validation/src/types/testResult.ts new file mode 100644 index 0000000..93bc2b8 --- /dev/null +++ b/packages/validation/src/types/testResult.ts @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +export type TestResultValid = { + valid: true; +}; +export const valid = (): TestResultValid => ({ valid: true }); + +export type TestResultInvalid = { + valid: false; + details: T; +}; + +/** + * Convenience method to return an InvalidTest object. + * @param message + * @param info + * @returns + */ +export const invalid = (info: InvalidInfo): TestResultInvalid => ({ + valid: false, + details: info, +}); + +/** + * A TestResult represents the outcome of a test applied to some data. For example, a test + * could be checking if a string value can be converted to the dataType of a given field, or it + * could test if a number value passes all restrictions provided for a field. + * + * If a test is valid, no additional data is added to the result. If it is invalid, then the + * reason (or array of reasons) for why the test failed should be given. To make this type + * reusable, the specific type of the invalid info is left as a generic. + * + * There are convenience methods available to return valid() and invalid(info) objects: + * + * @example + * if(hasFailure) { + * return invalid(reason); + * } + * return valid(); + */ +export type TestResult = TestResultValid | TestResultInvalid; diff --git a/packages/validation/src/types/validationErrorTypes.ts b/packages/validation/src/types/validationErrorTypes.ts deleted file mode 100644 index 016ce2a..0000000 --- a/packages/validation/src/types/validationErrorTypes.ts +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved - * - * This program and the accompanying materials are made available under the terms of - * the GNU Affero General Public License v3.0. You should have received a copy of the - * GNU Affero General Public License along with this program. - * If not, see . - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT - * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER - * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -import type { Values, Singular } from 'common'; -import type { DataRecord, DataRecordValue, RestrictionRange } from 'dictionary'; - -/** - * Represents the common structure of a validation error without the custom content provided by specific error types. - * The `message` property is not included here as this allows the rest of the error content to be passed to a message building function - * that will produce the final typed SchemaValidationError - */ -export type BaseSchemaValidationError = { - index: number; - fieldName: string; -}; - -type GenericSchemaValidationError< - ErrorType extends SchemaValidationErrorType, - Info extends object, -> = BaseSchemaValidationError & { - errorType: ErrorType; - info: Info; - message: string; -}; - -// Common string for invalid value errors -export const INVALID_VALUE_ERROR_MESSAGE = 'The value is not permissible for this field.'; - -export const SchemaValidationErrorTypes = { - INVALID_FIELD_VALUE_TYPE: 'INVALID_FIELD_VALUE_TYPE', - INVALID_BY_REGEX: 'INVALID_BY_REGEX', - INVALID_BY_RANGE: 'INVALID_BY_RANGE', - INVALID_BY_SCRIPT: 'INVALID_BY_SCRIPT', - INVALID_ENUM_VALUE: 'INVALID_ENUM_VALUE', - INVALID_BY_UNIQUE: 'INVALID_BY_UNIQUE', - INVALID_BY_FOREIGN_KEY: 'INVALID_BY_FOREIGN_KEY', - INVALID_BY_UNIQUE_KEY: 'INVALID_BY_UNIQUE_KEY', - MISSING_REQUIRED_FIELD: 'MISSING_REQUIRED_FIELD', - UNRECOGNIZED_FIELD: 'UNRECOGNIZED_FIELD', -} as const; -export type SchemaValidationErrorType = Values; - -export type EnumValueValidationError = GenericSchemaValidationError< - typeof SchemaValidationErrorTypes.INVALID_ENUM_VALUE, - { value: DataRecordValue[] } ->; -export type ForeignKeyValidationError = GenericSchemaValidationError< - typeof SchemaValidationErrorTypes.INVALID_BY_FOREIGN_KEY, - { value: DataRecord; foreignSchema: string } ->; -export type MissingRequiredFieldValidationError = GenericSchemaValidationError< - typeof SchemaValidationErrorTypes.MISSING_REQUIRED_FIELD, - {} ->; -export type RangeValidationError = GenericSchemaValidationError< - typeof SchemaValidationErrorTypes.INVALID_BY_RANGE, - { value: number[] } & RestrictionRange ->; -export type RegexValidationError = GenericSchemaValidationError< - typeof SchemaValidationErrorTypes.INVALID_BY_REGEX, - { value: string[]; regex: string; examples?: string } ->; -export type ScriptValidationError = GenericSchemaValidationError< - typeof SchemaValidationErrorTypes.INVALID_BY_SCRIPT, - { message: string; value: DataRecordValue } ->; -export type UniqueValidationError = GenericSchemaValidationError< - typeof SchemaValidationErrorTypes.INVALID_BY_UNIQUE, - { value: DataRecordValue } ->; -export type UniqueKeyValidationError = GenericSchemaValidationError< - typeof SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, - { uniqueKeyFields: string[]; value: DataRecord } ->; -export type UnrecognizedFieldValidationError = GenericSchemaValidationError< - typeof SchemaValidationErrorTypes.UNRECOGNIZED_FIELD, - {} ->; -export type ValueTypeValidationError = GenericSchemaValidationError< - typeof SchemaValidationErrorTypes.INVALID_FIELD_VALUE_TYPE, - { value: Singular[] } ->; -export type SchemaValidationError = - | EnumValueValidationError - | ValueTypeValidationError - | ForeignKeyValidationError - | MissingRequiredFieldValidationError - | RangeValidationError - | RegexValidationError - | ScriptValidationError - | UniqueValidationError - | UniqueKeyValidationError - | UnrecognizedFieldValidationError; diff --git a/packages/validation/src/utils/hashDataRecord.ts b/packages/validation/src/utils/hashDataRecord.ts new file mode 100644 index 0000000..346027c --- /dev/null +++ b/packages/validation/src/utils/hashDataRecord.ts @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2023 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import type { DataRecord, DataRecordValue } from 'dictionary'; + +/** + * Create a unique string out of an arbitrary array of data record values. This can be used + * as a unique identifier for an object with these properties and values. This type of hash is needed + * to check uniqueKey constraints. + * + * To generate a unique string from an object, the object is converted to an array of [key, value] tuples. + * Properties with an `undefiend` value are filtered out. + * These tuples are sorted by key name to ensure they are compared in the same order. + * Finally, the array is stringified using JSON.stringify(). + * + * This can be used to compare uniqueKeys across many data records by building a map of the hash of + * the uniqueKey values from each record. + * @param values any DataRecord or subset of fields from a DataRecord + */ +export const hashDataRecord = (values: DataRecord): string => { + const normalizedValues = Object.entries(values) + .filter(([_key, value]) => value !== undefined) + .sort(([keyA], [keyB]) => (keyA < keyB ? -1 : 1)); + return JSON.stringify(normalizedValues); +}; diff --git a/packages/validation/src/utils/isValidValueType.ts b/packages/validation/src/utils/isValidValueType.ts new file mode 100644 index 0000000..5f2639a --- /dev/null +++ b/packages/validation/src/utils/isValidValueType.ts @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import type { DataRecordValue, SchemaField } from 'dictionary'; +import { isBooleanArray, isInteger, isIntegerArray, isNumber, isNumberArray, isStringArray } from './typeUtils'; + +export const isValidValueType = (value: DataRecordValue, fieldDefinition: SchemaField): boolean => { + switch (fieldDefinition.valueType) { + case 'boolean': { + return fieldDefinition.isArray ? isBooleanArray(value) : typeof value === 'boolean'; + } + case 'integer': { + return fieldDefinition.isArray ? isIntegerArray(value) : isInteger(value); + } + case 'number': { + return fieldDefinition.isArray ? isNumberArray(value) : isNumber(value); + } + case 'string': { + return fieldDefinition.isArray ? isStringArray(value) : typeof value === 'string'; + } + } +}; diff --git a/packages/validation/src/utils/isWithinRange.ts b/packages/validation/src/utils/isWithinRange.ts new file mode 100644 index 0000000..67e4947 --- /dev/null +++ b/packages/validation/src/utils/isWithinRange.ts @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import type { RestrictionRange } from 'dictionary'; + +/** + * Test if a number is within a range. Returns true when the number is within the range. + * @param range + * @param value + * @returns + */ +export const isWithinRange = (range: RestrictionRange, value: number): boolean => + // less than the min if defined ? + !( + (range.min !== undefined && value < range.min) || + (range.exclusiveMin !== undefined && value <= range.exclusiveMin) || + // bigger than max if defined ? + (range.max !== undefined && value > range.max) || + (range.exclusiveMax !== undefined && value >= range.exclusiveMax) + ); diff --git a/packages/validation/src/utils/rangeToSymbol.ts b/packages/validation/src/utils/rangeToText.ts similarity index 85% rename from packages/validation/src/utils/rangeToSymbol.ts rename to packages/validation/src/utils/rangeToText.ts index 1616ccd..57b8d67 100644 --- a/packages/validation/src/utils/rangeToSymbol.ts +++ b/packages/validation/src/utils/rangeToText.ts @@ -19,7 +19,7 @@ import { RestrictionRange } from 'dictionary'; -export const rangeToSymbol = (range: RestrictionRange): string => { +export const rangeToText = (range: RestrictionRange): string => { let minString = ''; let maxString = ''; @@ -27,21 +27,24 @@ export const rangeToSymbol = (range: RestrictionRange): string => { (range.min !== undefined || range.exclusiveMin !== undefined) && (range.max !== undefined || range.exclusiveMax !== undefined); - if (range.min !== undefined) { - minString = `>= ${range.min}`; - } - + // The order here is intentionally putting exclusiveMin/exclusiveMax before the simple min/max. + // If a RestrictionRange is created with both min and exclusiveMin (or max and exclusiveMax), + // the generated text will use the more restrictive rule if (range.exclusiveMin !== undefined) { minString = `> ${range.exclusiveMin}`; } - if (range.max !== undefined) { - maxString = `<= ${range.max}`; + if (range.min !== undefined) { + minString = `>= ${range.min}`; } if (range.exclusiveMax !== undefined) { maxString = `< ${range.exclusiveMax}`; } + if (range.max !== undefined) { + maxString = `<= ${range.max}`; + } + return hasBothRange ? `${minString} and ${maxString}` : `${minString}${maxString}`; }; diff --git a/packages/validation/src/utils/typeUtils.ts b/packages/validation/src/utils/typeUtils.ts index a209e68..0720e8d 100644 --- a/packages/validation/src/utils/typeUtils.ts +++ b/packages/validation/src/utils/typeUtils.ts @@ -18,31 +18,78 @@ */ /** - * Checks that the input does not equal undefined (and lets the type checker know). + * Given a predicate function that checks for type `T`, this will create a new predicate funcion that + * will check if a value is of type `T[]`. * - * Useful for filtering undefined values out of lists. + * @example + * // Create type and predicate for `Person`: + * type Person = { name: string; age: number }; + * const isPerson = (value: unknown): value is Person => + * !!value && + * typeof value === 'object' && + * 'name' in value && + * typeof value.name === 'string' && + * 'age' in value && + * typeof value.age === 'number'; * - * (input) => input !== undefined + * // Use `isArrayOf` and the new predicate to create `isPersonArray`: + * const isPersonArray = isArrayOf(isPerson); * - * @example - * const combinedArray: Array = ['hello', undefined, 'world']; - * const stringArray = combinedArray.filter(isDefined); // type is: Array + * // Usage of `isPersonArray`: + * isPersonArray([{name:'Lisa', age: 8}, {name: 'Bart', age: 10}]); // true + * isPersonArray(['not a person']); // false + * isPersonArray('not an array'); // false + * isPersonArray([{name:'Lisa', age: 8}, {not: 'a person'}]); // false + * @param predicate + * @returns + */ +export const isArrayOf = + (predicate: (value: unknown) => value is T) => + (value: unknown) => + Array.isArray(value) && value.every(predicate); + +/** + * Determines if a variable is of type `boolean[]`. + * @param value + * @returns */ -export const isDefined = (input: T | undefined): input is T => input !== undefined; +export const isBooleanArray = isArrayOf((value: unknown) => typeof value === 'boolean'); /** - * Determines if a variable is of type number[] + * Determines if a variable is a number, with added restriction that it is Finite. + * This eliminates the values `NaN` and `Infinity`. + * + * Note: this is just a wrapper on `Number.isFinite` which is used by Lectern for identifying numbers in data value type checks. + * @param value + * @returns + */ +export const isNumber = (value: unknown): value is number => Number.isFinite(value); +/** + * Determines if variable is of type number[], with added restriction that every element is Finite. * @param value * @returns */ -export const isNumberArray = (value: unknown): value is number[] => - Array.isArray(value) && value.every((item) => typeof item === 'number'); +export const isNumberArray = isArrayOf(isNumber); /** * Determines if a variable is of type string[] * @param value * @returns */ -export const isStringArray = (value: unknown): value is string[] => { - return Array.isArray(value) && value.every((item) => typeof item === 'string'); -}; +export const isStringArray = isArrayOf((value: unknown) => typeof value === 'string'); + +/** + * Determines if a variable is a number, with added restriction that it is an Integer. + * + * Note: This is a wrapper over `Number.isInteger` which is used by Lectern for identifying integers in data value type checks. + * @param value + * @returns + */ +export const isInteger = (value: unknown): value is number => Number.isInteger(value); + +/** + * Determines if a variables is of type number[], with add restriction that every element is an Integer. + * @param value + * @returns + */ +export const isIntegerArray = isArrayOf(isInteger); diff --git a/packages/validation/src/types/validationFunctionTypes.ts b/packages/validation/src/validateDictionary/DictionaryValidationError.ts similarity index 52% rename from packages/validation/src/types/validationFunctionTypes.ts rename to packages/validation/src/validateDictionary/DictionaryValidationError.ts index bba1c30..d321b63 100644 --- a/packages/validation/src/types/validationFunctionTypes.ts +++ b/packages/validation/src/validateDictionary/DictionaryValidationError.ts @@ -17,28 +17,34 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { DataRecord, UnprocessedDataRecord, Schema } from 'dictionary'; -import { SchemaValidationError } from './validationErrorTypes'; +import type { FieldDetails } from '../validateRecord'; +import type { SchemaRecordError, SchemaValidationRecordErrorDetails } from '../validateSchema'; -// these validation functions run AFTER the record has been converted to the correct types from raw strings -export type UnprocessedRecordValidationFunction = ( - record: UnprocessedDataRecord, - index: number, - schemaFields: Schema['fields'], -) => Array; +export type DictionaryValidationErrorBase = { + schemaName: string; +}; -// these validation functions run BEFORE the record has been converted to the correct types from raw strings -export type ValidationFunction = ( - record: DataRecord, - index: number, - schemaFields: Schema['fields'], -) => Array; +export type DictionaryValidationErrorUnrecognizedSchema = DictionaryValidationErrorBase & { + reason: 'UNRECOGNIZED_SCHEMA'; +}; -// these validation functions run AFTER the records has been converted to the correct types from raw strings, and apply to a dataset instead of -// individual records -export type DatasetValidationFunction = (data: Array, schema: Schema) => Array; +export type DictionaryValidationErrorRecordForeignKey = FieldDetails & { + reason: 'INVALID_BY_FOREIGNKEY'; + foreignSchema: { + schemaName: string; + fieldName: string; + }; +}; -export type CrossSchemaValidationFunction = ( - schema: Schema, - data: Record, -) => Array; +export type DictionaryValidationRecordErrorDetails = + | SchemaValidationRecordErrorDetails + | DictionaryValidationErrorRecordForeignKey; + +export type DictionaryValidationErrorInvalidRecords = DictionaryValidationErrorBase & { + reason: 'INVALID_RECORDS'; + invalidRecords: SchemaRecordError[]; +}; + +export type DictionaryValidationError = + | DictionaryValidationErrorUnrecognizedSchema + | DictionaryValidationErrorInvalidRecords; diff --git a/packages/validation/src/validateDictionary/collectSchemaReferenceData.ts b/packages/validation/src/validateDictionary/collectSchemaReferenceData.ts new file mode 100644 index 0000000..d786560 --- /dev/null +++ b/packages/validation/src/validateDictionary/collectSchemaReferenceData.ts @@ -0,0 +1,110 @@ +import { isDefined } from 'common'; +import type { DataRecord, DataRecordValue, Dictionary, ForeignKeyRestriction, Schema } from 'dictionary'; + +/** + * This type alias is for a structure used to collect then lookup values from schemas. It is helpful + * to check if a value exists in a schema's records. Each set is a collection of all values from the + * data set for a field from the schema. + */ +export type SchemaDataReference = Map>; + +/** + * Collect all the values for each field from a list of records into a Set. Return those sets + * in a Map with the field name as the key + * + * undefined values will be skipped. + */ +const collectFieldValues = (records: DataRecord[], fieldNames: Set | string[]): SchemaDataReference => + records.reduce((schemaDataReference, record) => { + fieldNames.forEach((fieldName) => { + let set = schemaDataReference.get(fieldName); + if (!set) { + // Ensure each has a set created. This will happen exactly once. + set = new Set(); + schemaDataReference.set(fieldName, set); + } + + const value = record[fieldName]; + if (value !== undefined) { + set.add(value); + + // Only update the set in the data reference if we have added a value. + schemaDataReference.set(fieldName, set); + } + }); + + return schemaDataReference; + }, new Map()); + +/** + * Create the reference data needed to efficiently check foreign key restrictions. + * + * For every field that is referenced by a foreign key restriction, this creates a set of all values for that + * field from the records provided for that schema. This will be used when testing foreign key restrictions as + * as quick way to look up if the local field's value is present in the foreign schema's field (since lookups + * in a Set are much quicker than iterating through all items in an array). Additionally, this reference allows + * us to perform the foreign key test without providing all record data to the test, instead passing only this + * reference data. + * + * The output of this function is a Map of schema names to a Map of Field names, to a Set of field values. + * It may be convenient to think of the output like this: + * + * ``` + * { + * "schemaA": { + * "fieldName1": [... set of all values ...], + * "fieldName2": [... set of all values ...] + * } + * "schemaB": { + * "fieldName3": [... set of all values ...] + * } + * } + * ``` + * @param data Record that maps schema names to the list of records to be validated for that schema + * @param dictionary + * @returns + */ +export const collectSchemaReferenceData = ( + data: Record, + dictionary: Dictionary, +): Map => { + const schemasWithForeignKeyRestrictions: Array<{ + schema: Schema; + foreignKeyRestriction: ForeignKeyRestriction[]; + }> = dictionary.schemas + .map((schema) => + schema.restrictions?.foreignKey ? { schema, foreignKeyRestriction: schema.restrictions?.foreignKey } : undefined, + ) + .filter(isDefined); + + const foreignKeyMappings = schemasWithForeignKeyRestrictions.flatMap( + ({ foreignKeyRestriction: foreignKeyMappings }) => foreignKeyMappings, + ); + + // Map of schema names to Set of field names + const foreignSchemaFieldNamesMap = foreignKeyMappings.reduce>>( + (schemaMap, foreignKeyMapping) => { + let fieldSet = schemaMap.get(foreignKeyMapping.schema); + if (!fieldSet) { + fieldSet = new Set(); + schemaMap.set(foreignKeyMapping.schema, fieldSet); + } + + const fieldNames = foreignKeyMapping.mappings.map((mapping) => mapping.foreign); + fieldNames.forEach((fieldName) => fieldSet.add(fieldName)); + + return schemaMap; + }, + new Map(), + ); + + // Map of schema names to SchemaDataReference objects containing all the unique values for the fields that are needed for foreign key restrictions + const foreignSchemaDataReferenceMap = new Map(); + for (const [schemaName, fieldSet] of foreignSchemaFieldNamesMap.entries()) { + const records = data[schemaName] || []; + const referenceMap = collectFieldValues(records, fieldSet); + foreignSchemaDataReferenceMap.set(schemaName, referenceMap); + } + + return foreignSchemaDataReferenceMap; +}; diff --git a/packages/validation/src/validateDictionary/index.ts b/packages/validation/src/validateDictionary/index.ts new file mode 100644 index 0000000..d068b9f --- /dev/null +++ b/packages/validation/src/validateDictionary/index.ts @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +export * from './DictionaryValidationError'; +export * from './validateDictionary'; diff --git a/packages/validation/src/validateDictionary/testForeignKeyRestriction.ts b/packages/validation/src/validateDictionary/testForeignKeyRestriction.ts new file mode 100644 index 0000000..f746dd1 --- /dev/null +++ b/packages/validation/src/validateDictionary/testForeignKeyRestriction.ts @@ -0,0 +1,63 @@ +import type { DataRecord, ForeignKeyRestriction } from 'dictionary'; +import type { SchemaDataReference } from './collectSchemaReferenceData'; +import type { DictionaryValidationErrorRecordForeignKey } from './DictionaryValidationError'; +import { invalid, valid, type TestResult } from '../types'; +import { isDefined } from 'common'; + +/** + * Test foreignKey restrictions on a single DataRecord, checked against a pre-calculated `foreignSchemaRefereceData` with the values + * of relevant fields from foreign schemas. The `foreignSchemaRefereceData` can be created using `collectSchemaReferenceData`. + * + * Since foreignKey restrictions contain nested arrays, this test is a double loop over: + * 1. Foreign Schema name + * 2. Field name in foreign schema + * + * This is because each foreign key rule is an array of arrays, representing multiple foreign schemas with multiple foreign fields that + * must be matched to a local field. + */ +export const testForeignKeyRestriction = ( + record: DataRecord, + foreignKeyRestrictions: ForeignKeyRestriction[], + foreignSchemaReferenceData: Map, +): TestResult => { + if (foreignKeyRestrictions.length === 0) { + return valid(); + } + + // Loop over all foreign key rules - + // These are nested since we could have multiple foreign schema mappings and for each foreign schema we could have multiple fields. + // The result would be an array of arrays (array of schema results which are each an array of field mapping results) so we flatten the result. + const recordErrors = foreignKeyRestrictions.flatMap((restriction) => { + const foreignKeyErrors: DictionaryValidationErrorRecordForeignKey[] = restriction.mappings + .map((foreignKeyMapping) => { + // This is the actual test, taking the local value and looking up if that value exists in + const localValue = record[foreignKeyMapping.local]; + if (localValue === undefined) { + // Only apply foreignKey test to records that have a value in the local field + return undefined; + } + + const hasForeignReference = foreignSchemaReferenceData + .get(restriction.schema) + ?.get(foreignKeyMapping.foreign) + ?.has(localValue); + + return hasForeignReference + ? undefined + : { + reason: 'INVALID_BY_FOREIGNKEY', + fieldName: foreignKeyMapping.local, + foreignSchema: { fieldName: foreignKeyMapping.foreign, schemaName: restriction.schema }, + fieldValue: localValue, + }; + }) + .filter(isDefined); + return foreignKeyErrors; + }); + + if (recordErrors.length) { + return invalid(recordErrors); + } + + return valid(); +}; diff --git a/packages/validation/src/validateDictionary/testUnrecognizedSchema.ts b/packages/validation/src/validateDictionary/testUnrecognizedSchema.ts new file mode 100644 index 0000000..bd02362 --- /dev/null +++ b/packages/validation/src/validateDictionary/testUnrecognizedSchema.ts @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import type { Dictionary } from 'dictionary'; +import { invalid, valid, type TestResult } from '../types'; +import type { DictionaryValidationErrorUnrecognizedSchema } from './DictionaryValidationError'; + +export const testUnrecognizedSchema = ( + schemaName: string, + dictionary: Dictionary, +): TestResult => { + if (dictionary.schemas.some((schema) => schema.name === schemaName)) { + return valid(); + } + return invalid({ + reason: 'UNRECOGNIZED_SCHEMA', + schemaName, + }); +}; diff --git a/packages/validation/src/validateDictionary/validateDictionary.ts b/packages/validation/src/validateDictionary/validateDictionary.ts new file mode 100644 index 0000000..81d2831 --- /dev/null +++ b/packages/validation/src/validateDictionary/validateDictionary.ts @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { isDefined } from 'common'; +import { DataRecord, Dictionary } from 'dictionary'; +import { invalid, valid, type TestResult } from '../types'; +import { validateSchema, type SchemaRecordError } from '../validateSchema'; +import { collectSchemaReferenceData } from './collectSchemaReferenceData'; +import type { + DictionaryValidationError, + DictionaryValidationRecordErrorDetails, + DictionaryValidationErrorRecordForeignKey, +} from './DictionaryValidationError'; +import { testForeignKeyRestriction } from './testForeignKeyRestriction'; +import { testUnrecognizedSchema } from './testUnrecognizedSchema'; + +const mergeSchemaRecordValidationErrors = ( + first: Array>, + second: Array>, +): Array> => { + const output: Array> = [...first]; + for (const error of second) { + const matchedError = output.find((outputError) => outputError.recordIndex === error.recordIndex); + if (matchedError) { + matchedError.recordErrors.push(...error.recordErrors); + } else { + output.push(error); + } + } + return output; +}; + +/** + * + * @param data + * @param dictionary + * @returns + */ +export const validateDictionary = ( + data: Record, + dictionary: Dictionary, +): TestResult => { + // check all schemas are recognized + // this map.filter.map loop would have performance concerns except that the expected use case will always have small list of schemas + // If needed, we can collect this into a function that performs a single reduce. + const unrecognizedSchemaErrors = Object.keys(data) + .map((schemaName) => testUnrecognizedSchema(schemaName, dictionary)) + .filter((result) => !result.valid) + .map((result) => result.details); + + const foreignSchemaReferenceData = collectSchemaReferenceData(data, dictionary); + + const recognizedSchemaErrors: DictionaryValidationError[] = dictionary.schemas + .map((schema) => { + const records = data[schema.name] || []; + const schemaValidationResult = validateSchema(records, schema); + + const foreignKeyRestriction = schema.restrictions?.foreignKey; + const foreignKeyErrors: SchemaRecordError[] = foreignKeyRestriction + ? records + .map((record, recordIndex) => { + const foreignKeyTestResult = testForeignKeyRestriction( + record, + foreignKeyRestriction, + foreignSchemaReferenceData, + ); + if (foreignKeyTestResult.valid) { + return undefined; + } + return { + recordIndex, + recordErrors: foreignKeyTestResult.details, + }; + }) + .filter(isDefined) + : []; + const combinedErrors = mergeSchemaRecordValidationErrors( + schemaValidationResult.valid ? [] : schemaValidationResult.details, + foreignKeyErrors, + ); + + return combinedErrors.length + ? { reason: 'INVALID_RECORDS', schemaName: schema.name, invalidRecords: combinedErrors } + : undefined; + }) + .filter(isDefined); + + const collectedResults: DictionaryValidationError[] = [...unrecognizedSchemaErrors, ...recognizedSchemaErrors]; + + if (collectedResults.length) { + return invalid(collectedResults); + } + + return valid(); +}; diff --git a/packages/validation/src/validateField/FieldRestrictionRule.ts b/packages/validation/src/validateField/FieldRestrictionRule.ts new file mode 100644 index 0000000..31b1acc --- /dev/null +++ b/packages/validation/src/validateField/FieldRestrictionRule.ts @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import type { + FieldRestrictionTypes, + RestrictionCodeList, + RestrictionRange, + RestrictionRegex, + RestrictionScript, +} from 'dictionary'; + +export type FieldRestrictionRuleCodeList = { + type: typeof FieldRestrictionTypes.codeList; + rule: RestrictionCodeList; +}; + +export type FieldRestrictionRuleRange = { + type: typeof FieldRestrictionTypes.range; + rule: RestrictionRange; +}; + +export type FieldRestrictionRuleRequired = { + type: typeof FieldRestrictionTypes.required; + rule: boolean; +}; + +export type FieldRestrictionRuleRegex = { + type: typeof FieldRestrictionTypes.regex; + rule: RestrictionRegex; +}; + +// export type FieldRestrictionRuleScript = { +// type: typeof FieldRestrictionTypes.script; +// rule: RestrictionScript; +// }; + +// export type FieldRestrictionRuleUnique = { +// type: typeof FieldRestrictionTypes.unique; +// rule: boolean; +// }; + +export type FieldRestrictionRule = + | FieldRestrictionRuleCodeList + | FieldRestrictionRuleRange + | FieldRestrictionRuleRequired + | FieldRestrictionRuleRegex; diff --git a/packages/validation/src/dataRecordValidation/fieldNamesValidation.ts b/packages/validation/src/validateField/FieldRestrictionTest.ts similarity index 55% rename from packages/validation/src/dataRecordValidation/fieldNamesValidation.ts rename to packages/validation/src/validateField/FieldRestrictionTest.ts index 2df6bbb..48a4bc4 100644 --- a/packages/validation/src/dataRecordValidation/fieldNamesValidation.ts +++ b/packages/validation/src/validateField/FieldRestrictionTest.ts @@ -17,32 +17,28 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { - BaseSchemaValidationError, - SchemaValidationErrorTypes, - UnrecognizedFieldValidationError, -} from '../types/validationErrorTypes'; -import { UnprocessedRecordValidationFunction } from '../types/validationFunctionTypes'; +import type { ArrayDataValue, DataRecordValue, SingleDataValue } from 'dictionary'; +import type { TestResult } from '../types'; -export const validateFieldNames: UnprocessedRecordValidationFunction = ( - record, - index, - fields, -): UnrecognizedFieldValidationError[] => { - const expectedFields = new Set(fields.map((field) => field.name)); - return Object.keys(record) - .filter((fieldName) => !expectedFields.has(fieldName)) - .map((fieldName) => buildUnrecognizedFieldError({ fieldName, index })); +export type RestrictionTestInvalidArrayItem = { + position: number; + value: SingleDataValue; }; -export const buildUnrecognizedFieldError = (errorData: BaseSchemaValidationError): UnrecognizedFieldValidationError => { - const message = `${errorData.fieldName} is not an allowed field for this schema.`; - const info = {}; - - return { - ...errorData, - errorType: SchemaValidationErrorTypes.UNRECOGNIZED_FIELD, - info, - message, - }; +export type RestrictionTestInvalidInfo = { + message: string; + invalidItems?: Array; }; + +export type FieldRestrictionSingleValueTestFunction = ( + rule: Rule, + value: SingleDataValue, +) => TestResult; +export type FieldRestrictionArrayTestFunction = ( + rule: Rule, + values: ArrayDataValue, +) => TestResult; +export type FieldRestrictionTestFunction = ( + rule: Rule, + value: DataRecordValue, +) => TestResult; diff --git a/packages/validation/src/validateField/FieldValidationError.ts b/packages/validation/src/validateField/FieldValidationError.ts new file mode 100644 index 0000000..c1c8328 --- /dev/null +++ b/packages/validation/src/validateField/FieldValidationError.ts @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import type { SchemaFieldValueType } from 'dictionary'; +import type { FieldRestrictionRule } from '../validateField/FieldRestrictionRule'; +import type { RestrictionTestInvalidInfo } from '../validateField/FieldRestrictionTest'; + +export type FieldValidationErrorRestrictionInfo = RestrictionTestInvalidInfo & { + restriction: FieldRestrictionRule; +}; + +export type FieldValidationErrorRestrictions = { + reason: 'INVALID_BY_RESTRICTION'; + errors: Array; +}; + +/** + * This is the result when the value does not match the value type defined in the field. The properties + * `valueType` and `isArray` are the expected type values as defined in the field definition. + */ +export type FieldValidationErrorValueType = { + reason: 'INVALID_VALUE_TYPE'; + valueType: SchemaFieldValueType; + isArray: boolean; +}; + +export type FieldValidationError = FieldValidationErrorRestrictions | FieldValidationErrorValueType; diff --git a/packages/validation/src/validateField/index.ts b/packages/validation/src/validateField/index.ts new file mode 100644 index 0000000..53056c4 --- /dev/null +++ b/packages/validation/src/validateField/index.ts @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +export * from './restrictions'; +export * from './FieldRestrictionRule'; +export * from './FieldRestrictionTest'; +export * from './FieldValidationError'; +export * from './validateField'; diff --git a/packages/validation/src/validateField/resolveFieldRestrictions.ts b/packages/validation/src/validateField/resolveFieldRestrictions.ts new file mode 100644 index 0000000..b8b4a54 --- /dev/null +++ b/packages/validation/src/validateField/resolveFieldRestrictions.ts @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import type { DataRecord, DataRecordValue, SchemaField } from 'dictionary'; +import type { FieldRestrictionRule } from './FieldRestrictionRule'; + +/** + * Convert the restrictions found in a SchemaField definition into a list of rules that apply for this specific value + * and DataRecord. + */ +export const resolveFieldRestrictions = ( + _value: DataRecordValue, + _record: DataRecord, + field: SchemaField, +): FieldRestrictionRule[] => { + // TODO: This function requires value and record parameters so that conditional restrictions can be resolved. + // The original implementation with a static set of available restrictions does not need these parameters. + if (!field.restrictions) { + return []; + } + + switch (field.valueType) { + case 'boolean': { + const output: FieldRestrictionRule[] = []; + if (field.restrictions.required) { + output.push({ type: 'required', rule: field.restrictions.required }); + } + // if (field.restrictions.script) { + // output.push({ type: 'script', rule: asArray(field.restrictions.script) }); + // } + return output; + } + case 'integer': { + const output: FieldRestrictionRule[] = []; + if (Array.isArray(field.restrictions.codeList)) { + output.push({ type: 'codeList', rule: field.restrictions.codeList }); + } + if (field.restrictions.range) { + output.push({ type: 'range', rule: field.restrictions.range }); + } + if (field.restrictions.required) { + output.push({ type: 'required', rule: field.restrictions.required }); + } + // if (field.restrictions.script) { + // output.push({ type: 'script', rule: asArray(field.restrictions.script) }); + // } + return output; + } + case 'number': { + const output: FieldRestrictionRule[] = []; + if (Array.isArray(field.restrictions.codeList)) { + output.push({ type: 'codeList', rule: field.restrictions.codeList }); + } + if (field.restrictions.range) { + output.push({ type: 'range', rule: field.restrictions.range }); + } + if (field.restrictions.required) { + output.push({ type: 'required', rule: field.restrictions.required }); + } + // if (field.restrictions.script) { + // output.push({ type: 'script', rule: asArray(field.restrictions.script) }); + // } + return output; + } + case 'string': { + const output: FieldRestrictionRule[] = []; + if (Array.isArray(field.restrictions.codeList)) { + output.push({ type: 'codeList', rule: field.restrictions.codeList }); + } + if (field.restrictions.regex) { + output.push({ type: 'regex', rule: field.restrictions.regex }); + } + if (field.restrictions.required) { + output.push({ type: 'required', rule: field.restrictions.required }); + } + // if (field.restrictions.script) { + // output.push({ type: 'script', rule: asArray(field.restrictions.script) }); + // } + return output; + } + } +}; diff --git a/packages/validation/src/validateField/restrictions/createFieldRestrictionTestForArrays.ts b/packages/validation/src/validateField/restrictions/createFieldRestrictionTestForArrays.ts new file mode 100644 index 0000000..a2a7e87 --- /dev/null +++ b/packages/validation/src/validateField/restrictions/createFieldRestrictionTestForArrays.ts @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { isDefined } from 'common'; +import { invalid, valid } from '../../types'; +import type { + FieldRestrictionArrayTestFunction, + FieldRestrictionSingleValueTestFunction, +} from '../FieldRestrictionTest'; + +/** + * Given a function that will apply a fieldRestriction validation test to a single value, this function will + * create a new function that will apply that test to every element of an array. The new function returns an + * object that includes a list of invalid items and which position they hold in the array. + * @param test + * @param errorMessage Message to display when there are one or more invalid items in the array. + * This can be a string, or a function generates a message from the details of the specific FieldRestrictionRule. + * @returns + */ +export const createFieldRestrictionTestForArrays = + ( + test: FieldRestrictionSingleValueTestFunction, + errorMessage: string | ((rule: Rule) => string), + ): FieldRestrictionArrayTestFunction => + (rule, values) => { + const invalidItems = values + .map((value, position) => (test(rule, value).valid ? undefined : { position, value })) + .filter(isDefined); + + if (invalidItems.length) { + const message = typeof errorMessage === 'function' ? errorMessage(rule) : errorMessage; + return invalid({ message, invalidItems }); + } + return valid(); + }; diff --git a/packages/validation/src/validateField/restrictions/index.ts b/packages/validation/src/validateField/restrictions/index.ts new file mode 100644 index 0000000..682deae --- /dev/null +++ b/packages/validation/src/validateField/restrictions/index.ts @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +export * from './testCodeList'; +export * from './testRange'; +export * from './testRegex'; +export * from './testRequired'; diff --git a/packages/validation/src/validateField/restrictions/testCodeList.ts b/packages/validation/src/validateField/restrictions/testCodeList.ts new file mode 100644 index 0000000..d789bba --- /dev/null +++ b/packages/validation/src/validateField/restrictions/testCodeList.ts @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import type { RestrictionCodeList } from 'dictionary'; +import { invalid, valid } from '../../types/testResult'; +import type { FieldRestrictionSingleValueTestFunction, FieldRestrictionTestFunction } from '../FieldRestrictionTest'; +import { createFieldRestrictionTestForArrays } from './createFieldRestrictionTestForArrays'; + +const testCodeListSingleValue: FieldRestrictionSingleValueTestFunction = (rule, value) => { + // TODO: this can be sped up by using a set for the rule instead of an array. finding an element in the set is faster. + if (!(typeof value === 'string' || typeof value === 'number')) { + // only apply code list check to strings and numbers + return valid(); + } + + // We want to compare strings after removing whitespace and converting both the option and the value to lowercase + const testValue = typeof value === 'string' ? value.trim().toLowerCase() : value; + + for (const option of rule) { + const testOption = typeof option === 'string' ? option.trim().toLowerCase() : option; + if (testOption === testValue) { + return valid(); + } + } + return invalid({ message: `The value for this field must match an option from the list.` }); +}; + +const testCodeListArray = createFieldRestrictionTestForArrays( + testCodeListSingleValue, + `All values in this field must match an option from the list.`, +); + +export const testCodeList: FieldRestrictionTestFunction = (rule, value) => + Array.isArray(value) ? testCodeListArray(rule, value) : testCodeListSingleValue(rule, value); diff --git a/packages/validation/src/validateField/restrictions/testRange.ts b/packages/validation/src/validateField/restrictions/testRange.ts new file mode 100644 index 0000000..1a7eaf6 --- /dev/null +++ b/packages/validation/src/validateField/restrictions/testRange.ts @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { RestrictionRange } from 'dictionary'; +import { invalid, valid } from '../../types/testResult'; +import { isWithinRange } from '../../utils/isWithinRange'; +import type { FieldRestrictionSingleValueTestFunction, FieldRestrictionTestFunction } from '../FieldRestrictionTest'; +import { createFieldRestrictionTestForArrays } from './createFieldRestrictionTestForArrays'; + +const testRangeSingleValue: FieldRestrictionSingleValueTestFunction = (rule, value) => { + if (typeof value !== 'number') { + // only apply range tests to numbers + return valid(); + } + + if (isWithinRange(rule, value)) { + return valid(); + } + return invalid({ message: `The value must be within the range.`, rule }); +}; + +const testRangeArray = createFieldRestrictionTestForArrays( + testRangeSingleValue, + `All values in the array must be within the range.`, +); + +export const testRange: FieldRestrictionTestFunction = (rule, value) => + Array.isArray(value) ? testRangeArray(rule, value) : testRangeSingleValue(rule, value); diff --git a/packages/validation/src/validateField/restrictions/testRegex.ts b/packages/validation/src/validateField/restrictions/testRegex.ts new file mode 100644 index 0000000..20e3f05 --- /dev/null +++ b/packages/validation/src/validateField/restrictions/testRegex.ts @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { type RestrictionRegex } from 'dictionary'; +import { invalid, valid } from '../../types/testResult'; +import type { FieldRestrictionSingleValueTestFunction, FieldRestrictionTestFunction } from '../FieldRestrictionTest'; +import { createFieldRestrictionTestForArrays } from './createFieldRestrictionTestForArrays'; + +/** + * regex tests are only performed on strings. All other values will be true. + * @param rule + * @param value + * @returns + */ +const testRegexSingleValue: FieldRestrictionSingleValueTestFunction = (rule, value) => { + // Regex tests are only applied to strings + if (typeof value !== 'string') { + return valid(); + } + + const regexPattern = new RegExp(rule); + + if (regexPattern.test(value)) { + return valid(); + } + return invalid({ message: `The value must match the regular expression.` }); +}; + +const testRegexArray = createFieldRestrictionTestForArrays( + testRegexSingleValue, + (_rule) => `All values in the array must match the regular expression.`, +); + +export const testRegex: FieldRestrictionTestFunction = (rule, value) => + Array.isArray(value) ? testRegexArray(rule, value) : testRegexSingleValue(rule, value); diff --git a/packages/validation/src/validateField/restrictions/testRequired.ts b/packages/validation/src/validateField/restrictions/testRequired.ts new file mode 100644 index 0000000..b4556c5 --- /dev/null +++ b/packages/validation/src/validateField/restrictions/testRequired.ts @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { type ArrayDataValue } from 'dictionary'; +import { invalid, valid, type TestResult } from '../../types/testResult'; +import type { + FieldRestrictionSingleValueTestFunction, + FieldRestrictionTestFunction, + RestrictionTestInvalidInfo, +} from '../FieldRestrictionTest'; +import { createFieldRestrictionTestForArrays } from './createFieldRestrictionTestForArrays'; + +const testRequiredSingleValue: FieldRestrictionSingleValueTestFunction = (rule, value) => { + if (rule === false) { + return valid(); + } + if (value === undefined || value === '') { + return invalid({ message: `A value is required for this field.` }); + } + + return valid(); +}; + +/** + * This function is the common pattern for applying a fieldRestriction value test to an array. + * For the required restriction, we wanted to perform a couple additional checks to modify how + * this works, so this is used inside testRequiredArray after those additional checks are complete. + */ +const internalTestRequiredArray = createFieldRestrictionTestForArrays( + testRequiredSingleValue, + `This field requires all items to have a defined value.`, +); + +/** + * Test for required value on an array field. Before using the common pattern of applying the value test to + * each item in the array, we first check: + * - if the rule is `false` then the value is always valid + * - if the length of the array is 0 then the value is invalid + * @param rule + * @param values + * @returns + */ +const testRequiredArray = (rule: boolean, values: ArrayDataValue): TestResult => { + // Note: This doesn't apply the + if (rule === false) { + return valid(); + } + if (values.length === 0) { + return invalid({ message: 'A value is required for this field.' }); + } + return internalTestRequiredArray(rule, values); +}; + +/** + * Validate if a value is valid based on the required restriction value. + * + * When a field has the restriction `required: true`, it cannot be `undefined` or be an empty string. If + * the field is an array it cannot be an empty array (length 0). + * + * When a field has the restriction `required: false` then this test will always return as `valid: true` + * @param rule + * @param value + * @returns + */ +export const testRequired: FieldRestrictionTestFunction = (rule, value) => + Array.isArray(value) ? testRequiredArray(rule, value) : testRequiredSingleValue(rule, value); diff --git a/packages/validation/src/validateField/validateField.ts b/packages/validation/src/validateField/validateField.ts new file mode 100644 index 0000000..3929de8 --- /dev/null +++ b/packages/validation/src/validateField/validateField.ts @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import type { DataRecord, DataRecordValue, SchemaField } from 'dictionary'; +import { invalid, valid, type TestResult } from '../types'; +import { isValidValueType } from '../utils/isValidValueType'; +import type { FieldRestrictionRule } from './FieldRestrictionRule'; +import { resolveFieldRestrictions } from './resolveFieldRestrictions'; +import { testCodeList } from './restrictions/testCodeList'; +import { testRange } from './restrictions/testRange'; +import { testRegex } from './restrictions/testRegex'; +import { testRequired } from './restrictions/testRequired'; +import type { FieldValidationError, FieldValidationErrorRestrictionInfo } from './FieldValidationError'; + +const testRestriction = (value: DataRecordValue, restriction: FieldRestrictionRule) => { + switch (restriction.type) { + case 'codeList': { + return testCodeList(restriction.rule, value); + } + case 'range': { + return testRange(restriction.rule, value); + } + case 'regex': { + return testRegex(restriction.rule, value); + } + case 'required': { + return testRequired(restriction.rule, value); + } + // case 'unique': { + // return testRequired(restriction.rule, value); + // } + // case 'script': { + // return valid(); + // } + } +}; + +/** + * Apply a list of field restriction tests to a data record value. + * + * Note: It is possible to provide a value that is a type a restriction cannot process. As an example, + * a string could be provided along with a range restriction. When these mismatches occur the restriction + * will not be checked and the test will return as valid. Validating that the value provided matches the + * schema defined value type should be performed seperately. + * @param value + * @param restrictions + */ +const applyFieldRestrictionTests = ( + value: DataRecordValue, + restrictions: FieldRestrictionRule[], +): FieldValidationErrorRestrictionInfo[] => { + const errors = restrictions.reduce((output, restriction) => { + const result = testRestriction(value, restriction); + if (!result.valid) { + output.push({ + restriction, + ...result.details, + }); + } + return output; + }, []); + + return errors; +}; + +/** + * Confirm that a value is valid for a field definition. + * + * This validation expects values with correct types matching the Field definition, not raw string inputs from a TSV. + * @param value + * @param record + * @param fieldDefinition + * @returns + */ +export const validateField = ( + value: DataRecordValue, + record: DataRecord, + fieldDefinition: SchemaField, +): TestResult => { + // Awkward nested if - this makes it slightly easier to reason with and comment. + // If we have a value, we want to make sure it is the correct type, otherwise we are just going to return as invalid immediately. + if (value !== undefined) { + // We now know the value is defined, so we will confirm that the value is the correct type based on the field definition + if (!isValidValueType(value, fieldDefinition)) { + // The value is the wrong type! return invalid immediately! + return invalid({ + reason: 'INVALID_VALUE_TYPE', + valueType: fieldDefinition.valueType, + isArray: !!fieldDefinition.isArray, + }); + } + } + + // Now we know we don't have the wrong value type, we transform the field restrictions into a list of FieldRestrictionRules + // and then apply each of these rules + const restrictions = resolveFieldRestrictions(value, record, fieldDefinition); + + const errors = applyFieldRestrictionTests(value, restrictions); + + if (errors.length) { + return invalid({ reason: 'INVALID_BY_RESTRICTION', errors }); + } + + return valid(); +}; diff --git a/packages/validation/src/validateRecord/RecordValidationError.ts b/packages/validation/src/validateRecord/RecordValidationError.ts new file mode 100644 index 0000000..ff6d409 --- /dev/null +++ b/packages/validation/src/validateRecord/RecordValidationError.ts @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import type { DataRecordValue } from 'dictionary'; +import type { + FieldValidationErrorRestrictions, + FieldValidationErrorValueType, +} from '../validateField/FieldValidationError'; + +export type FieldDetails = { + fieldName: string; + fieldValue: DataRecordValue; +}; + +export type RecordValidationErrorInvalidValue = FieldDetails & FieldValidationErrorValueType; +export type RecordValidationErrorRestrictions = FieldDetails & FieldValidationErrorRestrictions; +export type RecordValidationErrorUnrecognizedField = FieldDetails & { + reason: 'UNRECOGNIZED_FIELD'; +}; + +export type RecordValidationError = + | RecordValidationErrorInvalidValue + | RecordValidationErrorRestrictions + | RecordValidationErrorUnrecognizedField; diff --git a/packages/validation/src/validateRecord/index.ts b/packages/validation/src/validateRecord/index.ts new file mode 100644 index 0000000..ba8310d --- /dev/null +++ b/packages/validation/src/validateRecord/index.ts @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +export * from './RecordValidationError'; +export * from './validateRecord'; diff --git a/packages/validation/src/validateRecord/validateRecord.ts b/packages/validation/src/validateRecord/validateRecord.ts new file mode 100644 index 0000000..cd29d87 --- /dev/null +++ b/packages/validation/src/validateRecord/validateRecord.ts @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import type { DataRecord, Schema } from 'dictionary'; +import { validateField } from '../validateField/validateField'; +import { invalid, valid, type TestResult } from '../types'; +import type { + FieldDetails, + RecordValidationError, + RecordValidationErrorInvalidValue, + RecordValidationErrorRestrictions, + RecordValidationErrorUnrecognizedField, +} from './RecordValidationError'; +import type { FieldValidationError } from '../validateField'; + +/** + * Validate a DataRecord using the fields in a Schema. Will confirm that there are no unrecognized fields, all required fields have a + * value, and that all values in the record are valid as defined by the Schema. + * + * This validation expects values with correct types matching the fields in the Schema, not raw string inputs from as TSV. + * @param record + * @param schema + * @returns + */ +export const validateRecord = (record: DataRecord, schema: Schema): TestResult => { + // We first check if the data record has any fields that are not in our schema definition + const unrecognizedFieldErrors = Object.entries(record).reduce( + (output, [fieldName, value]) => { + if (!schema.fields.some((field) => field.name === fieldName)) { + output.push({ + reason: 'UNRECOGNIZED_FIELD', + fieldName, + fieldValue: value, + }); + } + return output; + }, + [], + ); + + // Now we can apply the validation rules for each field in the schema. + // If a field is missing in the record then the value will be `undefined`. This will fail a Required restriction but pass all others. + const fieldValidationErrors = schema.fields.reduce<(FieldDetails & FieldValidationError)[]>((output, field) => { + const fieldName = field.name; + const value = record[fieldName]; + + const fieldValidationResult = validateField(value, record, field); + if (!fieldValidationResult.valid) { + fieldValidationResult.details; + output.push({ + fieldName, + fieldValue: value, + ...fieldValidationResult.details, + }); + } + return output; + }, []); + + const errors = [...unrecognizedFieldErrors, ...fieldValidationErrors]; + if (errors.length) { + return invalid(errors); + } + return valid(); +}; diff --git a/packages/validation/src/validateSchema/SchemaValidationError.ts b/packages/validation/src/validateSchema/SchemaValidationError.ts new file mode 100644 index 0000000..fa556df --- /dev/null +++ b/packages/validation/src/validateSchema/SchemaValidationError.ts @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import type { DataRecord } from 'dictionary'; +import type { RecordValidationError, FieldDetails } from '../validateRecord'; + +export type SchemaValidationRecordErrorUniqueKey = { + reason: 'INVALID_BY_UNIQUE_KEY'; + uniqueKey: DataRecord; + matchingRecords: number[]; +}; + +export type SchemaValidationRecordErrorUnique = FieldDetails & { + reason: 'INVALID_BY_UNIQUE'; + matchingRecords: number[]; +}; + +export type SchemaValidationRecordErrorDetails = + | RecordValidationError + | SchemaValidationRecordErrorUnique + | SchemaValidationRecordErrorUniqueKey; + +export type SchemaRecordError = { + recordIndex: number; + recordErrors: ErrorDetails[]; +}; + +export type SchemaValidationError = SchemaRecordError; diff --git a/packages/validation/src/validateSchema/index.ts b/packages/validation/src/validateSchema/index.ts new file mode 100644 index 0000000..52c7de7 --- /dev/null +++ b/packages/validation/src/validateSchema/index.ts @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2023 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +export * from './SchemaValidationError'; +export * from './validateSchema'; diff --git a/packages/validation/src/validateSchema/restrictions/generateDataSetHashMap.ts b/packages/validation/src/validateSchema/restrictions/generateDataSetHashMap.ts new file mode 100644 index 0000000..990da9f --- /dev/null +++ b/packages/validation/src/validateSchema/restrictions/generateDataSetHashMap.ts @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2023 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import type { DataRecord } from 'dictionary'; +import { hashDataRecord } from '../../utils/hashDataRecord'; +import { getUniqueKeyValues } from './uniqueKey/getUniqueKeyValues'; + +/** + * The key in this Map is a unique hash made of select fields from a record. This is generated using `hashDataRecord()` + * The value in this Map is an array of numbers, where each number represents the position of a record in a data set (data sets will be `DataRecord[]` with all records from the same schema). + * + * This is a named alias over a Map; + */ +export type DataSetHashMap = Map; + +/** + * Provides a DataSetHashMap for a provided data set. For every record in the data set this will generate a string + * based on the values of specific fields, then an entry will added to the Map using this string as the key, with the + * value an array of numbers representing the array index position of all records that share the same values for the + * specified fields. + * + * This means that for a data set where each record generates a unique hash, every entry in the Map will be an array + * with a single number (the index of the record with that hash). When two or more records have the same hash, the + * Map's value for that hash will be an array with the index of all records with that hash. + * @param records + * @param fieldsToHash + * @returns + */ +export const generateDataSetHashMap = (records: DataRecord[], fieldsToHash: string[]): DataSetHashMap => { + const output = new Map(); + + records.forEach((record, index) => { + // generate hash for this record + const uniqueKeyValues = getUniqueKeyValues(record, fieldsToHash); + + const hash = hashDataRecord(uniqueKeyValues); + + // check if this hash has an existing value, otherwise create a new array + const indexList = output.get(hash) || []; + const updatedIndexList = indexList.concat(index); + + output.set(hash, updatedIndexList); + }); + + return output; +}; diff --git a/packages/validation/src/validateSchema/restrictions/uniqueField/testUniqueFieldRestriction.ts b/packages/validation/src/validateSchema/restrictions/uniqueField/testUniqueFieldRestriction.ts new file mode 100644 index 0000000..32189b1 --- /dev/null +++ b/packages/validation/src/validateSchema/restrictions/uniqueField/testUniqueFieldRestriction.ts @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2023 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import type { DataRecordValue } from 'dictionary'; +import { invalid, valid, type TestResult } from '../../../types'; +import { hashDataRecord } from '../../../utils/hashDataRecord'; +import type { SchemaValidationRecordErrorUnique } from '../../SchemaValidationError'; + +/** + * Check if this record has a unique value for the given field. + * + * This requires a `uniqueKeyMap` to be provided - such a map can be generated using `generateDataSetHashMap(records, [fieldName])`. + * @param record + * @param uniqueKeyRule + * @param uniqueKeyMap + * @returns + */ +export const testUniqueFieldRestriction = ( + fieldValue: DataRecordValue, + fieldName: string, + uniqueKeyMap: Map, +): TestResult => { + // Only apply unique field restriction when a value is provided + if (fieldValue === undefined || (Array.isArray(fieldValue) && fieldValue.length === 0)) { + return valid(); + } + // Build unique key for this record, based on the fields listed in the rule + const hash = hashDataRecord({ [fieldName]: fieldValue }); + + // Lookup the key in the provided map. Report errors when the map has more than one record index found for this hash + const matchingRecords = uniqueKeyMap.get(hash); + if (matchingRecords && matchingRecords.length > 1) { + return invalid({ + reason: 'INVALID_BY_UNIQUE', + fieldName, + fieldValue, + matchingRecords, + }); + } + return valid(); +}; diff --git a/packages/validation/src/validateSchema/restrictions/uniqueKey/getUniqueKeyValues.ts b/packages/validation/src/validateSchema/restrictions/uniqueKey/getUniqueKeyValues.ts new file mode 100644 index 0000000..1d16dea --- /dev/null +++ b/packages/validation/src/validateSchema/restrictions/uniqueKey/getUniqueKeyValues.ts @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2023 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import type { DataRecord } from 'dictionary'; +/** + * Extract from a data record an object with only the unique key field values. + * @param record + * @param uniqueKeyRule + * @returns + */ +export const getUniqueKeyValues = (record: DataRecord, uniqueKeyRule: string[]): DataRecord => + uniqueKeyRule.reduce((acc, fieldName) => { + acc[fieldName] = record[fieldName]; + return acc; + }, {}); diff --git a/packages/validation/src/validateSchema/restrictions/uniqueKey/testUniqueKey.ts b/packages/validation/src/validateSchema/restrictions/uniqueKey/testUniqueKey.ts new file mode 100644 index 0000000..87c467a --- /dev/null +++ b/packages/validation/src/validateSchema/restrictions/uniqueKey/testUniqueKey.ts @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2023 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import type { DataRecord, Schema, SchemaField } from 'dictionary'; +import type { SchemaValidationRecordErrorUniqueKey } from '../../SchemaValidationError'; +import { invalid, valid, type TestResult } from '../../../types'; +import { hashDataRecord } from '../../../utils/hashDataRecord'; +import { getUniqueKeyValues } from './getUniqueKeyValues'; + +/** + * Check if this record is unique in a data set based on the fields defined in the Schema's uniqueKey restriction. + * + * A uniqueKey restriction is like a compound primary-key constraint. It is defined by a list of fieldNames. For + * each record in a data set, we find the uniqueKey value for each record by combining the record's values for the + * fields listed in the uniqueKey restriction and generate a hash based on these fields and values. Each record in the + * data set must have a unique value for this uniqueKey hash. + * + * To perform this check, the function needs to be provided a map object where the keys are the uniqueKey hash strings, + * and the values are arrays of numbers where each number represents the indicies of records in the data set with that + * uniqueKey hash. If there are no uniqueKey errors, then every entry in this map will have exactly one item in it. + * + * This function returns a SchemaValidationRecordErrorUniqueKey which is the SchemaValidationRecordError with information + * about the uniqueKey error. + * @param record + * @param uniqueKeyRule + * @param uniqueKeyMap + * @returns + */ +export const testUniqueKey = ( + record: DataRecord, + uniqueKeyRule: string[], + uniqueKeyMap: Map, +): TestResult => { + // Build unique key for this record, based on the fields listed in the rule + const uniqueKeyValues = getUniqueKeyValues(record, uniqueKeyRule); + const uniqueKeyHash = hashDataRecord(uniqueKeyValues); + + // Lookup the key in the provided map. We report errors when the map has more than one record index found for this hash + const matchingRecords = uniqueKeyMap.get(uniqueKeyHash); + if (matchingRecords && matchingRecords.length > 1) { + return invalid({ + reason: 'INVALID_BY_UNIQUE_KEY', + matchingRecords, + uniqueKey: uniqueKeyValues, + }); + } + return valid(); +}; diff --git a/packages/validation/src/validateSchema/validateSchema.ts b/packages/validation/src/validateSchema/validateSchema.ts new file mode 100644 index 0000000..594b3e2 --- /dev/null +++ b/packages/validation/src/validateSchema/validateSchema.ts @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import type { DataRecord, Schema } from 'dictionary'; +import { invalid, valid, type TestResult } from '../types'; +import type { SchemaValidationError, SchemaValidationRecordErrorDetails } from './SchemaValidationError'; +import { validateRecord } from '../validateRecord'; +import { generateDataSetHashMap } from './restrictions/generateDataSetHashMap'; +import { testUniqueKey } from './restrictions/uniqueKey/testUniqueKey'; +import { isDefined } from 'common'; +import { testUniqueFieldRestriction } from './restrictions/uniqueField/testUniqueFieldRestriction'; + +/** + * Validate a data set using a Lectern Schema. The data to validate is an array of DataRecords that contains all + * records for the given schema. Each record of the data set will be validated individually, plus schema level + * validation tests will be applied. + * + * Validation tests specific to Schema validation are: + * - unique fields: for any fields marked as unique, this will check if there are multiple records with the same value + * - uniqueKey: if the schema has a uniqueKey defined, this will check if there are multiple records with the same + * uniqueKey value + * + * @param records + * @param schema + * @returns + */ +export const validateSchema = (records: Array, schema: Schema): TestResult => { + // Setup to improve performance of Schema validations that compare a record to every record in the data set. + // We build maps of uniqueKey and unique field values so that they can be used while testing these restrictions for each record + const uniqueKeyRule = schema.restrictions?.uniqueKey; + const uniqueKeyMap = + uniqueKeyRule && uniqueKeyRule.length > 0 ? generateDataSetHashMap(records, uniqueKeyRule) : undefined; + + const uniqueFieldMaps = new Map>(); + schema.fields.forEach((field) => { + if (field.restrictions?.unique) { + uniqueFieldMaps.set(field.name, generateDataSetHashMap(records, [field.name])); + } + }); + + // Test each record, apply the schema restrictions to + const schemaValidationErrors = records + .map((record, recordIndex) => { + // recordErrors is output collection of errors for this record + const recordErrors: SchemaValidationRecordErrorDetails[] = []; + + // UniqueKey Test + const uniqueKeyResult = + uniqueKeyMap && uniqueKeyRule ? testUniqueKey(record, uniqueKeyRule, uniqueKeyMap) : valid(); + if (!uniqueKeyResult.valid) { + recordErrors.push(uniqueKeyResult.details); + } + + // Unique Field Restriction Tests + uniqueFieldMaps.forEach((hashMap, fieldName) => { + const uniqueFieldResult = testUniqueFieldRestriction(record[fieldName], fieldName, hashMap); + if (!uniqueFieldResult.valid) { + recordErrors.push(uniqueFieldResult.details); + } + }); + + // Data Record validation + const recordValidationResult = validateRecord(record, schema); + if (!recordValidationResult.valid) { + recordErrors.push(...recordValidationResult.details); + } + return recordErrors.length ? { recordIndex, recordErrors } : undefined; + }) + .filter(isDefined); + return schemaValidationErrors.length ? invalid(schemaValidationErrors) : valid(); +}; diff --git a/packages/validation/test/fixtures/dictionaries/dictionaryFourSchemas.ts b/packages/validation/test/fixtures/dictionaries/dictionaryFourSchemas.ts new file mode 100644 index 0000000..991c715 --- /dev/null +++ b/packages/validation/test/fixtures/dictionaries/dictionaryFourSchemas.ts @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { Dictionary } from 'dictionary'; +import { validateFixture } from '../../testUtils/validateFixture'; +import { schemaSingleString } from '../schema/schemaSingleString'; +import { schemaAllDataTypes } from '../schema/schemaAllDataTypes'; +import { schemaAllDataTypesRequired } from '../schema/schemaAllDataTypesRequired'; +import { schemaUniqueKey } from '../schema/schemaUniqueKey'; + +export const dictionaryFourSchemas = { + name: 'dictionary-four-restrictions', + schemas: [schemaSingleString, schemaAllDataTypes, schemaAllDataTypesRequired, schemaUniqueKey], + version: '1.0', +} as const satisfies Dictionary; + +validateFixture(dictionaryFourSchemas, Dictionary, 'dictionaryFourSchemas is not a valid Dictionary'); diff --git a/packages/validation/test/fixtures/dictionaries/dictionaryMultipleSchemasNoRestrictions.ts b/packages/validation/test/fixtures/dictionaries/dictionaryMultipleSchemasNoRestrictions.ts new file mode 100644 index 0000000..662e1e0 --- /dev/null +++ b/packages/validation/test/fixtures/dictionaries/dictionaryMultipleSchemasNoRestrictions.ts @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { Dictionary } from 'dictionary'; +import { validateFixture } from '../../testUtils/validateFixture'; +import { schemaSingleString } from '../schema/schemaSingleString'; +import { schemaAllDataTypes } from '../schema/schemaAllDataTypes'; + +export const dictionaryMultipleSchemasNoRestrictions = { + name: 'dictionary-multiple-no-restrictions', + schemas: [schemaSingleString, schemaAllDataTypes], + version: '1.0', +} as const satisfies Dictionary; + +validateFixture( + dictionaryMultipleSchemasNoRestrictions, + Dictionary, + 'dictionaryMultipleSchemasNoRestrictions is not a valid Dictionary', +); diff --git a/packages/validation/test/fixtures/dictionaries/dictionarySingleSchemaNoRestrictions.ts b/packages/validation/test/fixtures/dictionaries/dictionarySingleSchemaNoRestrictions.ts new file mode 100644 index 0000000..b933953 --- /dev/null +++ b/packages/validation/test/fixtures/dictionaries/dictionarySingleSchemaNoRestrictions.ts @@ -0,0 +1,15 @@ +import { Dictionary } from 'dictionary'; +import { validateFixture } from '../../testUtils/validateFixture'; +import { schemaSingleString } from '../schema/schemaSingleString'; + +export const dictionarySingleSchemaNoRestrictions = { + name: 'dictionary-single-string', + schemas: [schemaSingleString], + version: '1.0', +} as const satisfies Dictionary; + +validateFixture( + dictionarySingleSchemaNoRestrictions, + Dictionary, + 'dictionarySingleSchemaNoRestrictions is not a valid Dictionary', +); diff --git a/packages/validation/test/fixtures/dictionaries/dictionarySingleSchemaRequiredRestrictions.ts b/packages/validation/test/fixtures/dictionaries/dictionarySingleSchemaRequiredRestrictions.ts new file mode 100644 index 0000000..9b25ddc --- /dev/null +++ b/packages/validation/test/fixtures/dictionaries/dictionarySingleSchemaRequiredRestrictions.ts @@ -0,0 +1,15 @@ +import { Dictionary } from 'dictionary'; +import { validateFixture } from '../../testUtils/validateFixture'; +import { schemaAllDataTypesRequired } from '../schema/schemaAllDataTypesRequired'; + +export const dictionarySingleSchemaRequiredRestrictions = { + name: 'dictionary-all-fields-required', + schemas: [schemaAllDataTypesRequired], + version: '1.0', +} as const satisfies Dictionary; + +validateFixture( + dictionarySingleSchemaRequiredRestrictions, + Dictionary, + 'dictionarySingleSchemaRequiredRestrictions is not a valid Dictionary', +); diff --git a/packages/validation/test/fixtures/dictionaries/dictionarySingleSchemaUniqueKeyRestriction.ts b/packages/validation/test/fixtures/dictionaries/dictionarySingleSchemaUniqueKeyRestriction.ts new file mode 100644 index 0000000..094ee02 --- /dev/null +++ b/packages/validation/test/fixtures/dictionaries/dictionarySingleSchemaUniqueKeyRestriction.ts @@ -0,0 +1,15 @@ +import { Dictionary } from 'dictionary'; +import { validateFixture } from '../../testUtils/validateFixture'; +import { schemaUniqueKey } from '../schema/schemaUniqueKey'; + +export const dictionarySingleSchemaUniqueKeyRestriction = { + name: 'dictionary-single-schema-unique-key', + schemas: [schemaUniqueKey], + version: '1.0', +} as const satisfies Dictionary; + +validateFixture( + dictionarySingleSchemaUniqueKeyRestriction, + Dictionary, + 'dictionarySingleSchemaUniqueKeyRestriction is not a valid Dictionary', +); diff --git a/packages/validation/test/fixtures/dictionaries/foreignKey/dictionaryForeignKeyMultiple.ts b/packages/validation/test/fixtures/dictionaries/foreignKey/dictionaryForeignKeyMultiple.ts new file mode 100644 index 0000000..7ecdd04 --- /dev/null +++ b/packages/validation/test/fixtures/dictionaries/foreignKey/dictionaryForeignKeyMultiple.ts @@ -0,0 +1,40 @@ +import { Dictionary, Schema } from 'dictionary'; +import { validateFixture } from '../../../testUtils/validateFixture'; +import { schemaAllDataTypes } from '../../schema/schemaAllDataTypes'; +import { schemaSingleString } from '../../schema/schemaSingleString'; + +const schemaWithMultipleForeignKeys = { + name: 'multiple-foreign-keys', + fields: [ + { + name: 'string-field', + valueType: 'string', + }, + { + name: 'number-field', + valueType: 'number', + }, + ], + restrictions: { + foreignKey: [ + { + schema: schemaAllDataTypes.name, + mappings: [ + { foreign: 'any-string', local: 'string-field' }, + { foreign: 'any-number', local: 'number-field' }, + ], + }, + { schema: schemaSingleString.name, mappings: [{ foreign: 'any-string', local: 'string-field' }] }, + ], + }, +} as const satisfies Schema; + +validateFixture(schemaWithMultipleForeignKeys, Schema, 'schemaWithMultipleForeignKeys is not a valid Schema'); + +export const dictionaryForeignKeyMultiple = { + name: 'dictionary-multiple-foreign-key', + schemas: [schemaAllDataTypes, schemaSingleString, schemaWithMultipleForeignKeys], + version: '1.0', +} as const satisfies Dictionary; + +validateFixture(dictionaryForeignKeyMultiple, Dictionary, 'dictionaryForeignKeyMultiple is not a valid Dictionary'); diff --git a/packages/validation/test/fixtures/dictionaries/foreignKey/dictionaryForeignKeySimple.ts b/packages/validation/test/fixtures/dictionaries/foreignKey/dictionaryForeignKeySimple.ts new file mode 100644 index 0000000..558e9e5 --- /dev/null +++ b/packages/validation/test/fixtures/dictionaries/foreignKey/dictionaryForeignKeySimple.ts @@ -0,0 +1,28 @@ +import { Dictionary, Schema } from 'dictionary'; +import { validateFixture } from '../../../testUtils/validateFixture'; +import { schemaAllDataTypes } from '../../schema/schemaAllDataTypes'; + +const schemaWithForeignKey = { + name: 'string-matching-foreign-string', + fields: [ + { + name: 'string-with-foreign-key', + valueType: 'string', + }, + ], + restrictions: { + foreignKey: [ + { schema: schemaAllDataTypes.name, mappings: [{ foreign: 'any-string', local: 'string-with-foreign-key' }] }, + ], + }, +} as const satisfies Schema; + +validateFixture(schemaWithForeignKey, Schema, 'schemaWithForeignKey is not a valid Schema'); + +export const dictionaryForeignKeySimple = { + name: 'dictionary-single-foreign-key', + schemas: [schemaAllDataTypes, schemaWithForeignKey], + version: '1.0', +} as const satisfies Dictionary; + +validateFixture(dictionaryForeignKeySimple, Dictionary, 'dictionaryForeignKeySimple is not a valid Dictionary'); diff --git a/packages/validation/test/fixtures/fields/multipleRestrictions/fieldStringManyRestrictions.ts b/packages/validation/test/fixtures/fields/multipleRestrictions/fieldStringManyRestrictions.ts new file mode 100644 index 0000000..f057da8 --- /dev/null +++ b/packages/validation/test/fixtures/fields/multipleRestrictions/fieldStringManyRestrictions.ts @@ -0,0 +1,17 @@ +import type { SchemaStringField } from 'dictionary'; +import { regexYearMonthDay } from '../../restrictions/regexFixtures'; + +export const fieldStringManyRestrictions = { + name: 'complicated-multi-restriction-rules', + valueType: 'string', + description: + 'Designed to test scenario where a field has multiple restrictions. There may not be a practical use case for a field with this combination of restrictions, in particular having codeList and regex combined, but it is still important to test that such a combination does work.', + meta: { + examples: ['2001-01-01', '2002-02-02', '2003-03-03'], + }, + restrictions: { + codeList: ['2001-01-01', '2002-02-02', '2003-03-03', 'April 4, 2004', 'May-5-2005', '06/06/2006'], + regex: regexYearMonthDay, + required: true, + }, +} as const satisfies SchemaStringField; diff --git a/packages/validation/test/fixtures/fields/noRestrictions/fieldBooleanNoRestriction.ts b/packages/validation/test/fixtures/fields/noRestrictions/fieldBooleanNoRestriction.ts new file mode 100644 index 0000000..825a75a --- /dev/null +++ b/packages/validation/test/fixtures/fields/noRestrictions/fieldBooleanNoRestriction.ts @@ -0,0 +1,7 @@ +import type { SchemaBooleanField } from 'dictionary'; + +export const fieldBooleanNoRestriction = { + name: 'any-boolean', + description: 'Valid values are any boolean (true or false).', + valueType: 'boolean', +} as const satisfies SchemaBooleanField; diff --git a/packages/validation/test/fixtures/fields/noRestrictions/fieldIntegerNoRestriction.ts b/packages/validation/test/fixtures/fields/noRestrictions/fieldIntegerNoRestriction.ts new file mode 100644 index 0000000..6c4a66b --- /dev/null +++ b/packages/validation/test/fixtures/fields/noRestrictions/fieldIntegerNoRestriction.ts @@ -0,0 +1,6 @@ +import type { SchemaIntegerField } from 'dictionary'; + +export const fieldIntegerNoRestriction = { + name: 'any-integer', + valueType: 'integer', +} as const satisfies SchemaIntegerField; diff --git a/packages/validation/test/fixtures/fields/noRestrictions/fieldNumberNoRestriction.ts b/packages/validation/test/fixtures/fields/noRestrictions/fieldNumberNoRestriction.ts new file mode 100644 index 0000000..824e585 --- /dev/null +++ b/packages/validation/test/fixtures/fields/noRestrictions/fieldNumberNoRestriction.ts @@ -0,0 +1,6 @@ +import type { SchemaNumberField } from 'dictionary'; + +export const fieldNumberNoRestriction = { + name: 'any-number', + valueType: 'number', +} as const satisfies SchemaNumberField; diff --git a/packages/validation/test/fixtures/fields/noRestrictions/fieldStringArrayNoRestriction.ts b/packages/validation/test/fixtures/fields/noRestrictions/fieldStringArrayNoRestriction.ts new file mode 100644 index 0000000..1d7098e --- /dev/null +++ b/packages/validation/test/fixtures/fields/noRestrictions/fieldStringArrayNoRestriction.ts @@ -0,0 +1,7 @@ +import type { SchemaStringField } from 'dictionary'; + +export const fieldStringArrayNoRestriction = { + name: 'any-string-array', + valueType: 'string', + isArray: true, +} as const satisfies SchemaStringField; diff --git a/packages/validation/test/fixtures/fields/noRestrictions/fieldStringNoRestriction.ts b/packages/validation/test/fixtures/fields/noRestrictions/fieldStringNoRestriction.ts new file mode 100644 index 0000000..96cc975 --- /dev/null +++ b/packages/validation/test/fixtures/fields/noRestrictions/fieldStringNoRestriction.ts @@ -0,0 +1,6 @@ +import type { SchemaStringField } from 'dictionary'; + +export const fieldStringNoRestriction = { + name: 'any-string', + valueType: 'string', +} as const satisfies SchemaStringField; diff --git a/packages/validation/test/fixtures/fields/schemaRestrictions/fieldStringUnique.ts b/packages/validation/test/fixtures/fields/schemaRestrictions/fieldStringUnique.ts new file mode 100644 index 0000000..b8c6d82 --- /dev/null +++ b/packages/validation/test/fixtures/fields/schemaRestrictions/fieldStringUnique.ts @@ -0,0 +1,7 @@ +import type { SchemaStringField } from 'dictionary'; + +export const fieldStringUnique = { + name: 'unique-string', + valueType: 'string', + restrictions: { unique: true }, +} as const satisfies SchemaStringField; diff --git a/packages/validation/test/fixtures/fields/schemaRestrictions/fieldStringUniqueArray.ts b/packages/validation/test/fixtures/fields/schemaRestrictions/fieldStringUniqueArray.ts new file mode 100644 index 0000000..c652d82 --- /dev/null +++ b/packages/validation/test/fixtures/fields/schemaRestrictions/fieldStringUniqueArray.ts @@ -0,0 +1,8 @@ +import type { SchemaStringField } from 'dictionary'; + +export const fieldStringUniqueArray = { + name: 'unique-string-array', + valueType: 'string', + isArray: true, + restrictions: { unique: true }, +} as const satisfies SchemaStringField; diff --git a/packages/validation/test/fixtures/fields/simpleRestrictions/boolean/fieldBooleanArrayRequired.ts b/packages/validation/test/fixtures/fields/simpleRestrictions/boolean/fieldBooleanArrayRequired.ts new file mode 100644 index 0000000..b85e03e --- /dev/null +++ b/packages/validation/test/fixtures/fields/simpleRestrictions/boolean/fieldBooleanArrayRequired.ts @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import type { SchemaBooleanField } from 'dictionary'; + +export const fieldBooleanArrayRequired = { + name: 'boolean-array-required', + valueType: 'boolean', + isArray: true, + restrictions: { + required: true, + }, +} as const satisfies SchemaBooleanField; diff --git a/packages/validation/test/fixtures/fields/simpleRestrictions/boolean/fieldBooleanRequired.ts b/packages/validation/test/fixtures/fields/simpleRestrictions/boolean/fieldBooleanRequired.ts new file mode 100644 index 0000000..88f8792 --- /dev/null +++ b/packages/validation/test/fixtures/fields/simpleRestrictions/boolean/fieldBooleanRequired.ts @@ -0,0 +1,9 @@ +import type { SchemaBooleanField } from 'dictionary'; + +export const fieldBooleanRequired = { + name: 'boolean-required', + valueType: 'boolean', + restrictions: { + required: true, + }, +} as const satisfies SchemaBooleanField; diff --git a/packages/validation/test/fixtures/fields/simpleRestrictions/integer/fieldIntegerArrayRequired.ts b/packages/validation/test/fixtures/fields/simpleRestrictions/integer/fieldIntegerArrayRequired.ts new file mode 100644 index 0000000..c08c82f --- /dev/null +++ b/packages/validation/test/fixtures/fields/simpleRestrictions/integer/fieldIntegerArrayRequired.ts @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import type { SchemaIntegerField } from 'dictionary'; + +export const fieldIntegerArrayRequired = { + name: 'integer-array-required', + valueType: 'integer', + isArray: true, + restrictions: { + required: true, + }, +} as const satisfies SchemaIntegerField; diff --git a/packages/validation/test/fixtures/fields/simpleRestrictions/integer/fieldIntegerRequired.ts b/packages/validation/test/fixtures/fields/simpleRestrictions/integer/fieldIntegerRequired.ts new file mode 100644 index 0000000..8459948 --- /dev/null +++ b/packages/validation/test/fixtures/fields/simpleRestrictions/integer/fieldIntegerRequired.ts @@ -0,0 +1,9 @@ +import type { SchemaIntegerField } from 'dictionary'; + +export const fieldIntegerRequired = { + name: 'integer-required', + valueType: 'integer', + restrictions: { + required: true, + }, +} as const satisfies SchemaIntegerField; diff --git a/packages/validation/test/fixtures/fields/simpleRestrictions/number/fieldNumberArrayCodeList.ts b/packages/validation/test/fixtures/fields/simpleRestrictions/number/fieldNumberArrayCodeList.ts new file mode 100644 index 0000000..a1edbdc --- /dev/null +++ b/packages/validation/test/fixtures/fields/simpleRestrictions/number/fieldNumberArrayCodeList.ts @@ -0,0 +1,12 @@ +import type { SchemaNumberField } from 'dictionary'; +import { codeListNumber } from '../../../restrictions/codeListsFixtures'; + +export const fieldNumberArrayCodeList = { + name: 'number-code-list', + isArray: true, + valueType: 'number', + description: 'Optional field. Values must be from the numeric code list of the first 4 metallic ratios.', + restrictions: { + codeList: codeListNumber, + }, +} as const satisfies SchemaNumberField; diff --git a/packages/validation/test/fixtures/fields/simpleRestrictions/number/fieldNumberRange.ts b/packages/validation/test/fixtures/fields/simpleRestrictions/number/fieldNumberRange.ts new file mode 100644 index 0000000..a5e65cd --- /dev/null +++ b/packages/validation/test/fixtures/fields/simpleRestrictions/number/fieldNumberRange.ts @@ -0,0 +1,11 @@ +import type { SchemaNumberField } from 'dictionary'; +import { rangePercent } from '../../../restrictions/rangeFixtures'; + +export const fieldNumberRange = { + name: 'percentage', + valueType: 'number', + description: 'Optional field. If a value is given it must be a percentage from 0-100', + restrictions: { + range: rangePercent, + }, +} as const satisfies SchemaNumberField; diff --git a/packages/validation/test/fixtures/fields/simpleRestrictions/number/fieldNumberRequired.ts b/packages/validation/test/fixtures/fields/simpleRestrictions/number/fieldNumberRequired.ts new file mode 100644 index 0000000..ea4b052 --- /dev/null +++ b/packages/validation/test/fixtures/fields/simpleRestrictions/number/fieldNumberRequired.ts @@ -0,0 +1,9 @@ +import type { SchemaNumberField } from 'dictionary'; + +export const fieldNumberRequired = { + name: 'number-required', + valueType: 'number', + restrictions: { + required: true, + }, +} as const satisfies SchemaNumberField; diff --git a/packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringArrayCodeList.ts b/packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringArrayCodeList.ts new file mode 100644 index 0000000..48b04c7 --- /dev/null +++ b/packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringArrayCodeList.ts @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import type { SchemaStringField } from 'dictionary'; +import { codeListString } from '../../../restrictions/codeListsFixtures'; + +export const fieldStringArrayCodeList = { + name: 'favorite-foods', + valueType: 'string', + isArray: true, + description: 'Optional field. Values must be from the food code list (apple, banana, carrot, donut).', + restrictions: { + codeList: codeListString, + }, +} as const satisfies SchemaStringField; diff --git a/packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringArrayRequired.ts b/packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringArrayRequired.ts new file mode 100644 index 0000000..9873a58 --- /dev/null +++ b/packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringArrayRequired.ts @@ -0,0 +1,11 @@ +import type { SchemaStringField } from 'dictionary'; + +export const fieldStringArrayRequired = { + name: 'string-array-required', + isArray: true, + valueType: 'string', + description: 'Required field. An array with at least one string.', + restrictions: { + required: true, + }, +} as const satisfies SchemaStringField; diff --git a/packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringCodeList.ts b/packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringCodeList.ts new file mode 100644 index 0000000..0839f9c --- /dev/null +++ b/packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringCodeList.ts @@ -0,0 +1,11 @@ +import type { SchemaStringField } from 'dictionary'; +import { codeListString } from '../../../restrictions/codeListsFixtures'; + +export const fieldStringCodeList = { + name: 'favorite-food', + valueType: 'string', + description: 'Optional field. Values must be from the food code list (apple, banana, carrot, donut).', + restrictions: { + codeList: codeListString, + }, +} as const satisfies SchemaStringField; diff --git a/packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringRegex.ts b/packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringRegex.ts new file mode 100644 index 0000000..52cdc90 --- /dev/null +++ b/packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringRegex.ts @@ -0,0 +1,14 @@ +import type { SchemaStringField } from 'dictionary'; +import { regexYearMonthDay } from '../../../restrictions/regexFixtures'; + +export const fieldStringRegex = { + name: 'notable-date', + valueType: 'string', + description: 'Optional field. Values must be a date in the form YYYY-MM-DD.', + meta: { + examples: ['1867-07-01', '1969-07-16', '1989-12-17'], + }, + restrictions: { + regex: regexYearMonthDay, + }, +} as const satisfies SchemaStringField; diff --git a/packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringRequired.ts b/packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringRequired.ts new file mode 100644 index 0000000..9b2d1ca --- /dev/null +++ b/packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringRequired.ts @@ -0,0 +1,10 @@ +import type { SchemaStringField } from 'dictionary'; + +export const fieldStringRequired = { + name: 'string-required', + valueType: 'string', + description: 'Required field. Any string value.', + restrictions: { + required: true, + }, +} as const satisfies SchemaStringField; diff --git a/packages/validation/test/fixtures/restrictions/codeListsFixtures.ts b/packages/validation/test/fixtures/restrictions/codeListsFixtures.ts new file mode 100644 index 0000000..36cd8ed --- /dev/null +++ b/packages/validation/test/fixtures/restrictions/codeListsFixtures.ts @@ -0,0 +1,5 @@ +import type { RestrictionCodeListInteger, RestrictionCodeListNumber, RestrictionCodeListString } from 'dictionary'; + +export const codeListString: RestrictionCodeListString = ['Apple ', ' Banana', ' Carrot ', 'Donut']; // Food, extra whitespace on items in order to test that matching is being trimmed. +export const codeListInteger: RestrictionCodeListInteger = [1, 1, 2, 3, 5, 8, 13, 24]; // Fibonacci +export const codeListNumber: RestrictionCodeListNumber = [1.61803, 2.41421, 3.30278, 4.23607]; // Metallic Ratios diff --git a/packages/validation/test/fixtures/restrictions/rangeFixtures.ts b/packages/validation/test/fixtures/restrictions/rangeFixtures.ts new file mode 100644 index 0000000..c90dc14 --- /dev/null +++ b/packages/validation/test/fixtures/restrictions/rangeFixtures.ts @@ -0,0 +1,6 @@ +import type { RestrictionRange } from 'dictionary'; + +export const rangePercent: RestrictionRange = { + max: 100, + min: 0, +}; diff --git a/packages/validation/test/fixtures/restrictions/regexFixtures.ts b/packages/validation/test/fixtures/restrictions/regexFixtures.ts new file mode 100644 index 0000000..c896d34 --- /dev/null +++ b/packages/validation/test/fixtures/restrictions/regexFixtures.ts @@ -0,0 +1,5 @@ +import type { RestrictionRegex } from 'dictionary'; + +export const regexYearMonthDay: RestrictionRegex = '^([12]\\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12]\\d|3[01]))$'; // Example: "1999-01-31" +export const regexMTGMana: RestrictionRegex = + '^({((([WUBRG]|([0-9]|[1-9][0-9]*))(/[WUBRG])?)|(X)|([WUBRG](/[WUBRG])?/[P]))})+$'; diff --git a/packages/validation/test/fixtures/schema/schemaAllDataMixedRestrictions.ts b/packages/validation/test/fixtures/schema/schemaAllDataMixedRestrictions.ts new file mode 100644 index 0000000..4f67646 --- /dev/null +++ b/packages/validation/test/fixtures/schema/schemaAllDataMixedRestrictions.ts @@ -0,0 +1,17 @@ +import { Schema } from 'dictionary'; +import { fieldStringManyRestrictions } from '../fields/multipleRestrictions/fieldStringManyRestrictions'; +import { fieldBooleanNoRestriction } from '../fields/noRestrictions/fieldBooleanNoRestriction'; +import { fieldIntegerRequired } from '../fields/simpleRestrictions/integer/fieldIntegerRequired'; +import { fieldNumberArrayCodeList } from '../fields/simpleRestrictions/number/fieldNumberArrayCodeList'; +import { validateFixture } from '../../testUtils/validateFixture'; + +export const schemaAllDataTypesMixedRestrictions = { + name: 'all-data-types-mixed-restrictions', + fields: [fieldStringManyRestrictions, fieldNumberArrayCodeList, fieldIntegerRequired, fieldBooleanNoRestriction], +} as const satisfies Schema; + +validateFixture( + schemaAllDataTypesMixedRestrictions, + Schema, + 'schemaAllDataTypesMixedRestrictions is not a valid Schema', +); diff --git a/packages/validation/test/fixtures/schema/schemaAllDataTypes.ts b/packages/validation/test/fixtures/schema/schemaAllDataTypes.ts new file mode 100644 index 0000000..98e3e73 --- /dev/null +++ b/packages/validation/test/fixtures/schema/schemaAllDataTypes.ts @@ -0,0 +1,14 @@ +import { Schema } from 'dictionary'; +import { fieldBooleanNoRestriction } from '../fields/noRestrictions/fieldBooleanNoRestriction'; +import { fieldIntegerNoRestriction } from '../fields/noRestrictions/fieldIntegerNoRestriction'; +import { fieldNumberNoRestriction } from '../fields/noRestrictions/fieldNumberNoRestriction'; +import { fieldStringNoRestriction } from '../fields/noRestrictions/fieldStringNoRestriction'; +import { validateFixture } from '../../testUtils/validateFixture'; + +export const schemaAllDataTypes = { + name: 'all-data-types', + description: 'Contains a field of each possible data type. All optional, no restrictions.', + fields: [fieldStringNoRestriction, fieldNumberNoRestriction, fieldIntegerNoRestriction, fieldBooleanNoRestriction], +} as const satisfies Schema; + +validateFixture(schemaAllDataTypes, Schema, 'schemaAllDataTypes is not a valid Schema'); diff --git a/packages/validation/test/fixtures/schema/schemaAllDataTypesRequired.ts b/packages/validation/test/fixtures/schema/schemaAllDataTypesRequired.ts new file mode 100644 index 0000000..b29d08f --- /dev/null +++ b/packages/validation/test/fixtures/schema/schemaAllDataTypesRequired.ts @@ -0,0 +1,14 @@ +import { Schema } from 'dictionary'; +import { fieldBooleanRequired } from '../fields/simpleRestrictions/boolean/fieldBooleanRequired'; +import { fieldIntegerRequired } from '../fields/simpleRestrictions/integer/fieldIntegerRequired'; +import { fieldNumberRequired } from '../fields/simpleRestrictions/number/fieldNumberRequired'; +import { fieldStringRequired } from '../fields/simpleRestrictions/string/fieldStringRequired'; +import { validateFixture } from '../../testUtils/validateFixture'; + +export const schemaAllDataTypesRequired = { + name: 'all-data-types-required', + description: 'Contains a field of each possible data type. All fields are required.', + fields: [fieldStringRequired, fieldNumberRequired, fieldIntegerRequired, fieldBooleanRequired], +} as const satisfies Schema; + +validateFixture(schemaAllDataTypesRequired, Schema, 'schemaAllDataTypesRequired is not a valid Schema'); diff --git a/packages/validation/test/fixtures/schema/schemaSingleString.ts b/packages/validation/test/fixtures/schema/schemaSingleString.ts new file mode 100644 index 0000000..9f5a645 --- /dev/null +++ b/packages/validation/test/fixtures/schema/schemaSingleString.ts @@ -0,0 +1,11 @@ +import { Schema } from 'dictionary'; +import { fieldStringNoRestriction } from '../fields/noRestrictions/fieldStringNoRestriction'; +import { validateFixture } from '../../testUtils/validateFixture'; + +export const schemaSingleString = { + name: 'single-string', + description: 'Contains a single, optional string field.', + fields: [fieldStringNoRestriction], +} as const satisfies Schema; + +validateFixture(schemaSingleString, Schema, 'schemaSingleString is not a valid Schema'); diff --git a/packages/validation/test/fixtures/schema/schemaSingleStringRequired.ts b/packages/validation/test/fixtures/schema/schemaSingleStringRequired.ts new file mode 100644 index 0000000..66ff645 --- /dev/null +++ b/packages/validation/test/fixtures/schema/schemaSingleStringRequired.ts @@ -0,0 +1,11 @@ +import { Schema } from 'dictionary'; +import { fieldStringRequired } from '../fields/simpleRestrictions/string/fieldStringRequired'; +import { validateFixture } from '../../testUtils/validateFixture'; + +export const schemaSingleStringRequired = { + name: 'single-string-required', + description: 'Contains a single, optional string field.', + fields: [fieldStringRequired], +} as const satisfies Schema; + +validateFixture(schemaSingleStringRequired, Schema, 'schemaSingleStringRequired is not a valid Schema'); diff --git a/packages/validation/test/fixtures/schema/schemaUniqueKey.ts b/packages/validation/test/fixtures/schema/schemaUniqueKey.ts new file mode 100644 index 0000000..d9185ad --- /dev/null +++ b/packages/validation/test/fixtures/schema/schemaUniqueKey.ts @@ -0,0 +1,27 @@ +import { Schema } from 'dictionary'; +import { validateFixture } from '../../testUtils/validateFixture'; +import { fieldBooleanNoRestriction } from '../fields/noRestrictions/fieldBooleanNoRestriction'; +import { fieldIntegerNoRestriction } from '../fields/noRestrictions/fieldIntegerNoRestriction'; +import { fieldNumberNoRestriction } from '../fields/noRestrictions/fieldNumberNoRestriction'; +import { fieldStringNoRestriction } from '../fields/noRestrictions/fieldStringNoRestriction'; + +const fields = [ + fieldStringNoRestriction, + fieldNumberNoRestriction, + fieldIntegerNoRestriction, + fieldBooleanNoRestriction, +]; + +const uniqueKey = fields.map((field) => field.name); + +export const schemaUniqueKey = { + name: 'unique-key', + description: + 'Contains a field of each possible data type with a schema restriction that the combined value must be unique. All fields are optional with no other restrictions.', + fields, + restrictions: { + uniqueKey, + }, +} as const satisfies Schema; + +validateFixture(schemaUniqueKey, Schema, 'schemaUniqueKey is not a valid Schema'); diff --git a/packages/validation/test/fixtures/schema/schemaUniqueKeyWithArray.ts b/packages/validation/test/fixtures/schema/schemaUniqueKeyWithArray.ts new file mode 100644 index 0000000..6c84fa9 --- /dev/null +++ b/packages/validation/test/fixtures/schema/schemaUniqueKeyWithArray.ts @@ -0,0 +1,27 @@ +import { Schema } from 'dictionary'; +import { validateFixture } from '../../testUtils/validateFixture'; +import { fieldBooleanNoRestriction } from '../fields/noRestrictions/fieldBooleanNoRestriction'; +import { fieldIntegerNoRestriction } from '../fields/noRestrictions/fieldIntegerNoRestriction'; +import { fieldNumberNoRestriction } from '../fields/noRestrictions/fieldNumberNoRestriction'; +import { fieldStringArrayNoRestriction } from '../fields/noRestrictions/fieldStringArrayNoRestriction'; + +const fields = [ + fieldStringArrayNoRestriction, + fieldNumberNoRestriction, + fieldIntegerNoRestriction, + fieldBooleanNoRestriction, +]; + +const uniqueKey = fields.map((field) => field.name); + +export const schemaUniqueKeyWithArray = { + name: 'unique-key', + description: + 'Contains a field of each possible data type with a schema restriction that the combined value must be unique. All fields are optional with no other restrictions.', + fields, + restrictions: { + uniqueKey, + }, +} as const satisfies Schema; + +validateFixture(schemaUniqueKeyWithArray, Schema, 'schemaUniqueKeyWithArray is not a valid Schema'); diff --git a/packages/validation/test/fixtures/schema/schemaUniqueString.ts b/packages/validation/test/fixtures/schema/schemaUniqueString.ts new file mode 100644 index 0000000..40aa14f --- /dev/null +++ b/packages/validation/test/fixtures/schema/schemaUniqueString.ts @@ -0,0 +1,12 @@ +import { Schema } from 'dictionary'; +import { fieldStringUnique } from '../fields/schemaRestrictions/fieldStringUnique'; +import assert from 'node:assert'; +import { validateFixture } from '../../testUtils/validateFixture'; + +export const schemaUniqueString = { + name: 'single-unique-string', + description: 'Contains a single string field where every value must be unique.', + fields: [fieldStringUnique], +} as const satisfies Schema; + +validateFixture(schemaUniqueString, Schema, 'schemaUniqueString is not a valid Schema'); diff --git a/packages/validation/test/fixtures/schema/schemaUniqueStringArray.ts b/packages/validation/test/fixtures/schema/schemaUniqueStringArray.ts new file mode 100644 index 0000000..002f98d --- /dev/null +++ b/packages/validation/test/fixtures/schema/schemaUniqueStringArray.ts @@ -0,0 +1,13 @@ +import { Schema } from 'dictionary'; +import { fieldStringUnique } from '../fields/schemaRestrictions/fieldStringUnique'; +import assert from 'node:assert'; +import { validateFixture } from '../../testUtils/validateFixture'; +import { fieldStringUniqueArray } from '../fields/schemaRestrictions/fieldStringUniqueArray'; + +export const schemaUniqueStringArray = { + name: 'single-unique-string-array', + description: 'Contains a single string field where every value must be unique.', + fields: [fieldStringUniqueArray], +} as const satisfies Schema; + +validateFixture(schemaUniqueStringArray, Schema, 'schemaUniqueStringArray is not a valid Schema'); diff --git a/packages/validation/test/parseValues/parseDictionary.spec.ts b/packages/validation/test/parseValues/parseDictionary.spec.ts new file mode 100644 index 0000000..30cf521 --- /dev/null +++ b/packages/validation/test/parseValues/parseDictionary.spec.ts @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { expect } from 'chai'; +import assert from 'node:assert'; +import { parseDictionaryValues } from '../../src'; +import { dictionaryFourSchemas } from '../fixtures/dictionaries/dictionaryFourSchemas'; + +describe('Parse Values - parseDictionaryValues', () => { + it('Successfully parses all records for all schemas in the dictionary dataset', () => { + const singleStringRecords = [{ 'any-string': 'whatever' }]; + const allDataTypeRecords = [ + { + 'any-string': 'hello world 1', + 'any-number': '12.34', + 'any-integer': '1234321', + 'any-boolean': 'true', + }, + { + 'any-string': 'hello world 2', + 'any-number': '23.45', + 'any-integer': '2345432', + 'any-boolean': 'TRUE', + }, + { + 'any-string': 'hello world 3', + 'any-number': '34.56', + 'any-integer': '3456543', + 'any-boolean': 'False', + }, + ]; + const allDataRequiredRecords = [ + { + 'string-required': 'super important string', + 'number-required': '1000000000', + 'integer-required': '0', + 'boolean-required': 'false', + }, + ]; + const uniqueKeyRecords = [ + { + 'any-string': 'could be anything', + 'any-number': '1', + 'any-integer': '-1', + 'any-boolean': 'false', + }, + { + 'any-string': 'could be this', + 'any-number': '2.2', + 'any-integer': '-2', + 'any-boolean': 'true', + }, + ]; + + const dataset = { + 'single-string': singleStringRecords, + 'all-data-types': allDataTypeRecords, + 'all-data-types-required': allDataRequiredRecords, + 'unique-key': uniqueKeyRecords, + }; + const result = parseDictionaryValues(dataset, dictionaryFourSchemas); + expect(result.success).true; + assert(result.success === true); + }); + it('Reports unrecognized schema error', () => { + const unknownSchemaRecords = [{ 'any-field': 'whatever' }]; + + const dataset = { + 'unknown-schema': unknownSchemaRecords, + }; + const result = parseDictionaryValues(dataset, dictionaryFourSchemas); + expect(result.success).false; + assert(result.success === false); + + // Get the + const parseConversionResult = result.data['unknown-schema']; + expect(parseConversionResult?.success).false; + assert(parseConversionResult?.success === false); + + expect(parseConversionResult.data.reason).equal('UNRECOGNIZED_SCHEMA'); + }); + it('Returns invalid when there is a mix of correct and incorrect schemas', () => { + const singleStringRecords = [{ 'any-string': 'whatever' }]; // valid + const allDataRequiredRecords = [ + { + 'string-required': 'super important string', + 'number-required': '1000000000', + 'integer-required': '0.123', // needs to be an integer + 'boolean-required': 'false', + }, + ]; + const unrecognizedSchemaRecords = [{ 'any-field': 'whatever' }]; + + const dataset = { + 'single-string': singleStringRecords, + 'all-data-types-required': allDataRequiredRecords, + 'unknown-schema': unrecognizedSchemaRecords, + }; + const result = parseDictionaryValues(dataset, dictionaryFourSchemas); + expect(result.success).false; + assert(result.success === false); + + const singleStringResult = result.data['single-string']; + expect(singleStringResult?.success).true; + + const allFieldsResult = result.data['all-data-types-required']; + expect(allFieldsResult?.success).false; + assert(allFieldsResult?.success === false); + expect(allFieldsResult.data.reason).equal('INVALID_RECORDS'); + + const unrecognizedResult = result.data['unknown-schema']; + expect(unrecognizedResult?.success).false; + assert(unrecognizedResult?.success === false); + expect(unrecognizedResult.data.reason).equal('UNRECOGNIZED_SCHEMA'); + }); +}); diff --git a/packages/validation/test/parseValues/parseField.spec.ts b/packages/validation/test/parseValues/parseField.spec.ts new file mode 100644 index 0000000..74e4fc3 --- /dev/null +++ b/packages/validation/test/parseValues/parseField.spec.ts @@ -0,0 +1,304 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { expect } from 'chai'; +import { parseFieldValue } from '../../src/parseValues'; +import { fieldStringNoRestriction } from '../fixtures/fields/noRestrictions/fieldStringNoRestriction'; +import { fieldNumberNoRestriction } from '../fixtures/fields/noRestrictions/fieldNumberNoRestriction'; +import { fieldIntegerNoRestriction } from '../fixtures/fields/noRestrictions/fieldIntegerNoRestriction'; +import { fieldBooleanNoRestriction } from '../fixtures/fields/noRestrictions/fieldBooleanNoRestriction'; +import { fieldStringArrayRequired } from '../fixtures/fields/simpleRestrictions/string/fieldStringArrayRequired'; +import { fieldNumberArrayCodeList } from '../fixtures/fields/simpleRestrictions/number/fieldNumberArrayCodeList'; +import { fieldIntegerArrayRequired } from '../fixtures/fields/simpleRestrictions/integer/fieldIntegerArrayRequired'; +import { fieldBooleanArrayRequired } from '../fixtures/fields/simpleRestrictions/boolean/fieldBooleanArrayRequired'; +import { fieldStringCodeList } from '../fixtures/fields/simpleRestrictions/string/fieldStringCodeList'; +import { codeListString } from '../fixtures/restrictions/codeListsFixtures'; +import { fieldStringArrayCodeList } from '../fixtures/fields/simpleRestrictions/string/fieldStringArrayCodeList'; + +describe('Parse Values - parseFieldValue', () => { + describe('Single Value Fields', () => { + it('All field types successfully parses strings of only whitespace as `undefined`', () => { + expect(parseFieldValue('', fieldStringNoRestriction).success).true; + expect(parseFieldValue('', fieldStringNoRestriction).data).undefined; + expect(parseFieldValue(' ', fieldStringNoRestriction).success).true; + expect(parseFieldValue(' ', fieldStringNoRestriction).data).undefined; + + expect(parseFieldValue('', fieldNumberNoRestriction).success).true; + expect(parseFieldValue('', fieldNumberNoRestriction).data).undefined; + expect(parseFieldValue(' ', fieldNumberNoRestriction).success).true; + expect(parseFieldValue(' ', fieldNumberNoRestriction).data).undefined; + + expect(parseFieldValue('', fieldIntegerNoRestriction).success).true; + expect(parseFieldValue('', fieldIntegerNoRestriction).data).undefined; + expect(parseFieldValue(' ', fieldIntegerNoRestriction).success).true; + expect(parseFieldValue(' ', fieldIntegerNoRestriction).data).undefined; + + expect(parseFieldValue('', fieldBooleanNoRestriction).success).true; + expect(parseFieldValue('', fieldBooleanNoRestriction).data).undefined; + expect(parseFieldValue(' ', fieldBooleanNoRestriction).success).true; + expect(parseFieldValue(' ', fieldBooleanNoRestriction).data).undefined; + + expect(parseFieldValue(' ', fieldStringNoRestriction).success).true; + expect(parseFieldValue(' ', fieldStringNoRestriction).data).undefined; + }); + + describe('Boolean', () => { + it('Boolean field successfully parses booleans, case insensitive', () => { + const resultTrue = parseFieldValue('true', fieldBooleanNoRestriction); + expect(resultTrue.success).true; + expect(resultTrue.data).equal(true); + + const resultFalse = parseFieldValue('false', fieldBooleanNoRestriction); + expect(resultFalse.success).true; + expect(resultFalse.data).equal(false); + + expect(parseFieldValue('False', fieldBooleanNoRestriction).success).true; + expect(parseFieldValue('False', fieldBooleanNoRestriction).data).equals(false); + expect(parseFieldValue('TRUE', fieldBooleanNoRestriction).success).true; + expect(parseFieldValue('TRUE', fieldBooleanNoRestriction).data).equals(true); + expect(parseFieldValue('tRuE', fieldBooleanNoRestriction).success).true; + expect(parseFieldValue('tRuE', fieldBooleanNoRestriction).data).equals(true); + }); + it('Boolean field successfully parses value with whitespace', () => { + const resultTrue = parseFieldValue(' TRUE', fieldBooleanNoRestriction); + expect(resultTrue.success).true; + expect(resultTrue.data).equal(true); + + const resultFalse = parseFieldValue(' False ', fieldBooleanNoRestriction); + expect(resultFalse.success).true; + expect(resultFalse.data).equal(false); + }); + it('Boolean field rejects non boolean strings', () => { + expect(parseFieldValue('hello world', fieldBooleanNoRestriction).success).false; + expect(parseFieldValue(',', fieldBooleanNoRestriction).success).false; + expect(parseFieldValue('falso', fieldBooleanNoRestriction).success).false; + }); + }); + + describe('Integer', () => { + it('Integer field successfully parses integers', () => { + const value = '12345'; + const result = parseFieldValue(value, fieldIntegerNoRestriction); + expect(result.success).true; + expect(result.data).equal(Number(value)); + + expect(parseFieldValue('100', fieldIntegerNoRestriction).success).true; + expect(parseFieldValue('100', fieldIntegerNoRestriction).data).equals(100); + expect(parseFieldValue('-99999999', fieldIntegerNoRestriction).success).true; + expect(parseFieldValue('-99999999', fieldIntegerNoRestriction).data).equals(-99999999); + expect(parseFieldValue('180.00', fieldIntegerNoRestriction).success).true; + expect(parseFieldValue('180.00', fieldIntegerNoRestriction).data).equals(180); + }); + it('Integer field rejects non integer values', () => { + expect(parseFieldValue('100.01', fieldIntegerNoRestriction).success).false; + expect(parseFieldValue('-99999.999', fieldIntegerNoRestriction).success).false; + }); + it('Integer field rejects non numeric strings', () => { + expect(parseFieldValue('another one', fieldIntegerNoRestriction).success).false; + expect(parseFieldValue(',', fieldIntegerNoRestriction).success).false; + expect(parseFieldValue('12.34.56', fieldIntegerNoRestriction).success).false; + }); + it('Integer field rejects Infinity', () => { + expect(parseFieldValue('Infinity', fieldIntegerNoRestriction).success).false; + expect(parseFieldValue('-Infinity', fieldIntegerNoRestriction).success).false; + }); + it('Intger field rejects NaN', () => { + expect(parseFieldValue('NaN', fieldIntegerNoRestriction).success).false; + }); + }); + + describe('Number', () => { + it('Number field successfully parses numbers', () => { + const value = '123.45'; + const result = parseFieldValue(value, fieldNumberNoRestriction); + expect(result.success).true; + expect(result.data).equal(Number(value)); + + expect(parseFieldValue('100', fieldNumberNoRestriction).success).true; + expect(parseFieldValue('100', fieldNumberNoRestriction).data).equals(100); + expect(parseFieldValue('-99999999', fieldNumberNoRestriction).success).true; + expect(parseFieldValue('-99999999.99', fieldNumberNoRestriction).data).equals(-99999999.99); + expect(parseFieldValue('-0', fieldNumberNoRestriction).success).true; + expect(parseFieldValue('-0', fieldNumberNoRestriction).data).equals(0); + }); + it('Number field rejects non numeric strings', () => { + expect(parseFieldValue('hello world', fieldNumberNoRestriction).success).false; + expect(parseFieldValue(',', fieldNumberNoRestriction).success).false; + expect(parseFieldValue('one', fieldNumberNoRestriction).success).false; + }); + it('Number field rejects Infinity', () => { + expect(parseFieldValue('Infinity', fieldNumberNoRestriction).success).false; + expect(parseFieldValue('-Infinity', fieldNumberNoRestriction).success).false; + }); + it('Number field rejects NaN', () => { + expect(parseFieldValue('NaN', fieldNumberNoRestriction).success).false; + }); + }); + + describe('String', () => { + it('Succesfuly parses strings, returning trimmed value', () => { + const value = 'any random string value!!!'; + const result = parseFieldValue(value, fieldStringNoRestriction); + expect(result.success).true; + expect(result.data).equal(value); + + expect(parseFieldValue(' 123', fieldStringNoRestriction).success).true; + expect(parseFieldValue(' 123', fieldStringNoRestriction).data).equals('123'); + expect(parseFieldValue('false ', fieldStringNoRestriction).success).true; + expect(parseFieldValue('false ', fieldStringNoRestriction).data).equals('false'); + expect(parseFieldValue(' !@#$%^&* ()_+ ', fieldStringNoRestriction).success).true; + expect(parseFieldValue(' !@#$%^&* ()_+ ', fieldStringNoRestriction).data).equals('!@#$%^&* ()_+'); + }); + it('Updates string to match formatting of codeList value', () => { + const value = 'banana'; + const result = parseFieldValue(value, fieldStringCodeList); + expect(result.success).true; + expect(result.data).equal(codeListString[1]); + }); + }); + }); + describe('Arrays', () => { + it('Parses strings of only whitespace to `[]`', () => { + expect(parseFieldValue('', fieldStringArrayRequired).success).true; + expect(parseFieldValue('', fieldStringArrayRequired).data).deep.equals([]); + expect(parseFieldValue(' ', fieldStringArrayRequired).success).true; + expect(parseFieldValue(' ', fieldStringArrayRequired).data).deep.equals([]); + + expect(parseFieldValue('', fieldNumberArrayCodeList).success).true; + expect(parseFieldValue('', fieldNumberArrayCodeList).data).deep.equals([]); + expect(parseFieldValue(' ', fieldNumberArrayCodeList).success).true; + expect(parseFieldValue(' ', fieldNumberArrayCodeList).data).deep.equals([]); + + expect(parseFieldValue('', fieldIntegerArrayRequired).success).true; + expect(parseFieldValue('', fieldIntegerArrayRequired).data).deep.equals([]); + expect(parseFieldValue(' ', fieldIntegerArrayRequired).success).true; + expect(parseFieldValue(' ', fieldIntegerArrayRequired).data).deep.equals([]); + + expect(parseFieldValue('', fieldBooleanArrayRequired).success).true; + expect(parseFieldValue('', fieldBooleanArrayRequired).data).deep.equals([]); + expect(parseFieldValue(' ', fieldBooleanArrayRequired).success).true; + expect(parseFieldValue(' ', fieldBooleanArrayRequired).data).deep.equals([]); + + expect(parseFieldValue(' ', fieldStringArrayRequired).success).true; + expect(parseFieldValue(' ', fieldStringArrayRequired).data).deep.equals([]); + }); + it('Rejects arrays with trailing delimiter', () => { + const stringResult = parseFieldValue('first,second,third,', fieldStringArrayRequired); + expect(stringResult.success).false; + + const numberResult = parseFieldValue('123,0.123,4567.89,', fieldNumberArrayCodeList); + expect(numberResult.success).false; + + const integerResult = parseFieldValue('123,', fieldIntegerArrayRequired); + expect(integerResult.success).false; + + const booleanResult = parseFieldValue('true,true,false,', fieldBooleanArrayRequired); + expect(booleanResult.success).false; + }); + it('Rejects arrays with leading delimiter', () => { + const stringResult = parseFieldValue(',first,second,third', fieldStringArrayRequired); + expect(stringResult.success).false; + + const numberResult = parseFieldValue(',123,0.123,4567.89', fieldNumberArrayCodeList); + expect(numberResult.success).false; + + const integerResult = parseFieldValue(',123', fieldIntegerArrayRequired); + expect(integerResult.success).false; + + const booleanResult = parseFieldValue(',true,true,false', fieldBooleanArrayRequired); + expect(booleanResult.success).false; + }); + it('Rejects value of only delimiter', () => { + expect(parseFieldValue(',', fieldStringArrayRequired).success).false; + + expect(parseFieldValue(',', fieldNumberArrayCodeList).success).false; + + expect(parseFieldValue(',', fieldIntegerArrayRequired).success).false; + + expect(parseFieldValue(',', fieldBooleanArrayRequired).success).false; + }); + describe('String array', () => { + it('Successfully splits string into array', () => { + const result = parseFieldValue('first,second,third,fourth', fieldStringArrayRequired); + expect(result.success).true; + expect(result.data).deep.equal(['first', 'second', 'third', 'fourth']); + }); + it('Successfully trims each value', () => { + const result = parseFieldValue(' first , second , third, fourth ', fieldStringArrayRequired); + expect(result.success).true; + expect(result.data).deep.equal(['first', 'second', 'third', 'fourth']); + }); + it('Rejects array with missing values (two delimiters are adjacent)', () => { + const result = parseFieldValue('first,second, ,,fourth', fieldStringArrayRequired); + expect(result.success).false; + }); + it('For field with code list, updates value formatting to match codeList', () => { + const value = 'apple, banana, carrot, donut, elephant'; + const result = parseFieldValue(value, fieldStringArrayCodeList); + expect(result.success).true; + expect(result.data).deep.equal([...codeListString, 'elephant']); + }); + }); + describe('Integer array', () => { + it('Successfully parses array', () => { + const result = parseFieldValue('12,45,-111111', fieldIntegerArrayRequired); + expect(result.success).true; + expect(result.data).deep.equals([12, 45, -111111]); + }); + it('Successfully parses values with whitespace', () => { + const result = parseFieldValue(' 23 , -556555 , 0 ', fieldIntegerArrayRequired); + expect(result.success).true; + expect(result.data).deep.equals([23, -556555, 0]); + }); + it('Rejects array where value is missing (two delimiters are adjacent)', () => { + const result = parseFieldValue('12, 45,,-111111', fieldIntegerArrayRequired); + expect(result.success).false; + }); + }); + describe('Number array', () => { + it('Number array field, successfully parses array', () => { + const result = parseFieldValue('12,0.45,-111.111', fieldNumberArrayCodeList); + expect(result.success).true; + expect(result.data).deep.equals([12, 0.45, -111.111]); + }); + it('Number array field, successfully parses values with whitespace', () => { + const result = parseFieldValue(' 23 , -556.555 , 0.0 ', fieldNumberArrayCodeList); + expect(result.success).true; + expect(result.data).deep.equals([23, -556.555, 0]); + }); + it('Number array field, rejects array where value is missing (two delimiters are adjacent)', () => { + const result = parseFieldValue('12, 0.45,,-111.111', fieldNumberArrayCodeList); + expect(result.success).false; + }); + }); + describe('Bolean array', () => { + it('Boolean array field, successfully parses array', () => { + const result = parseFieldValue('true,false,TRUE', fieldBooleanArrayRequired); + expect(result.success).true; + expect(result.data).deep.equals([true, false, true]); + }); + it('Boolean array field, rejects array where value is missing (two delimiters are adjacent)', () => { + expect(parseFieldValue('true,false,,TRUE', fieldBooleanArrayRequired).success).false; + expect(parseFieldValue('true,false,TRUE,,', fieldBooleanArrayRequired).success).false; + }); + it('Boolean array field, rejects array where value is missing (two delimiters are adjacent)'); + expect(parseFieldValue(',true,false,TRUE', fieldBooleanArrayRequired).success).false; + }); + }); +}); diff --git a/packages/validation/test/parseValues/parseRecord.spec.ts b/packages/validation/test/parseValues/parseRecord.spec.ts new file mode 100644 index 0000000..a75a12a --- /dev/null +++ b/packages/validation/test/parseValues/parseRecord.spec.ts @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { expect } from 'chai'; +import { parseRecordValues } from '../../src'; +import { schemaAllDataTypes } from '../fixtures/schema/schemaAllDataTypes'; +import assert from 'assert'; + +describe('Parse Values - parseRecordValues', () => { + it('Successfully parses all fields in record', () => { + const record = { + 'any-string': 'hello world', + 'any-number': '12.34', + 'any-integer': '7890', + 'any-boolean': 'true', + }; + const result = parseRecordValues(record, schemaAllDataTypes); + expect(result.success).true; + assert(result.success === true); + expect(result.data.record).deep.equal({ + 'any-string': 'hello world', + 'any-number': 12.34, + 'any-integer': 7890, + 'any-boolean': true, + }); + }); + it('Failed conversion reports an error for each invalid field', () => { + const record = { + 'any-string': 'hello world', + 'any-number': 'NaN', + 'any-integer': '12.34', + 'any-boolean': 'not a boolean', + }; + const result = parseRecordValues(record, schemaAllDataTypes); + expect(result.success).false; + assert(result.success === false); + expect(result.data.errors.length).equal(3); + expect(result.data.record).deep.equal({ + 'any-string': 'hello world', + 'any-number': 'NaN', + 'any-integer': '12.34', + 'any-boolean': 'not a boolean', + }); + + const anyNumberError = result.data.errors.find((error) => error.fieldName === 'any-number'); + expect(anyNumberError).not.undefined; + assert(anyNumberError !== undefined); + expect(anyNumberError.fieldValue).equal('NaN'); + + const anyIntegerError = result.data.errors.find((error) => error.fieldName === 'any-integer'); + expect(anyIntegerError).not.undefined; + assert(anyIntegerError !== undefined); + expect(anyIntegerError.fieldValue).equal('12.34'); + + const anyBooleanError = result.data.errors.find((error) => error.fieldName === 'any-boolean'); + expect(anyBooleanError).not.undefined; + }); + it('Failed conversion result contains record with valid fields parsed', () => { + const record = { + 'any-string': 'hello world', + 'any-number': '12.34', + 'any-integer': '12.34', + 'any-boolean': 'true', + }; + const result = parseRecordValues(record, schemaAllDataTypes); + expect(result.success).false; + assert(result.success === false); + expect(result.data.record).deep.equal({ + 'any-string': 'hello world', + 'any-number': 12.34, // number is parsed + 'any-integer': '12.34', // integer is invalid, not parsed + 'any-boolean': true, // boolean is parsed + }); + }); + it('Failed conversion result when record contains unrecognized fields', () => { + const record = { + 'unrecognized-field': 'true', + }; + const result = parseRecordValues(record, schemaAllDataTypes); + expect(result.success).false; + assert(result.success === false); + expect(result.data.record).deep.equal({ + 'unrecognized-field': 'true', + }); + expect(result.data.errors).length(1); + const unrecognizedFieldError = result.data.errors[0]; + expect(unrecognizedFieldError).exist; + assert(unrecognizedFieldError !== undefined); + expect(unrecognizedFieldError.reason).equal('UNRECOGNIZED_FIELD'); + expect(unrecognizedFieldError.fieldName).equal('unrecognized-field'); + expect(unrecognizedFieldError.fieldValue).equal('true'); + }); +}); diff --git a/packages/validation/test/parseValues/parseSchema.spec.ts b/packages/validation/test/parseValues/parseSchema.spec.ts new file mode 100644 index 0000000..af71e79 --- /dev/null +++ b/packages/validation/test/parseValues/parseSchema.spec.ts @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { expect } from 'chai'; +import { parseSchemaValues } from '../../src'; +import { schemaAllDataTypes } from '../fixtures/schema/schemaAllDataTypes'; +import assert from 'assert'; + +describe('Parse Values - parseSchemaValues', () => { + it('Successfully parses all records in schema dataset', () => { + const records = [ + { + 'any-string': 'hello world 1', + 'any-number': '12.34', + 'any-integer': '1234321', + 'any-boolean': 'true', + }, + { + 'any-string': 'hello world 2', + 'any-number': '23.45', + 'any-integer': '2345432', + 'any-boolean': 'TRUE', + }, + { + 'any-string': 'hello world 3', + 'any-number': '34.56', + 'any-integer': '3456543', + 'any-boolean': 'False', + }, + ]; + const result = parseSchemaValues(records, schemaAllDataTypes); + expect(result.success).true; + assert(result.success === true); + expect(result.data.records[0]).deep.equal({ + 'any-string': 'hello world 1', + 'any-number': 12.34, + 'any-integer': 1234321, + 'any-boolean': true, + }); + expect(result.data.records[1]).deep.equal({ + 'any-string': 'hello world 2', + 'any-number': 23.45, + 'any-integer': 2345432, + 'any-boolean': true, + }); + expect(result.data.records[2]).deep.equal({ + 'any-string': 'hello world 3', + 'any-number': 34.56, + 'any-integer': 3456543, + 'any-boolean': false, + }); + }); + it('Failed conversion reports errors from each record that failed', () => { + const records = [ + { + 'any-string': 'valid record', + 'any-number': '12.34', + 'any-integer': '1234321', + 'any-boolean': 'true', + }, + { + 'any-string': 'invalid 1', + 'any-number': 'not a number', + 'any-integer': '1234321', + 'any-boolean': 'true', + }, + { + 'any-string': 'invalid 2', + 'any-number': '23.45', + 'any-integer': 'not an integer', + 'any-boolean': 'TRUE', + }, + { + 'any-string': 'invalid 3', + 'any-number': '34.56', + 'any-integer': '3456543', + 'any-boolean': 'not a boolean', + }, + ]; + const result = parseSchemaValues(records, schemaAllDataTypes); + expect(result.success).false; + assert(result.success === false); + expect(result.data.errors.length).equal(3); + + const firstError = result.data.errors.find((error) => error.recordIndex === 1); + expect(firstError).not.undefined; + assert(firstError !== undefined); + expect(firstError.recordErrors.length).equal(1); + expect(firstError.recordErrors[0]?.fieldName).equal('any-number'); + + const secondError = result.data.errors.find((error) => error.recordIndex === 2); + expect(secondError).not.undefined; + assert(secondError !== undefined); + expect(secondError.recordErrors.length).equal(1); + expect(secondError.recordErrors[0]?.fieldName).equal('any-integer'); + + const thirdError = result.data.errors.find((error) => error.recordIndex === 3); + expect(thirdError).not.undefined; + assert(thirdError !== undefined); + expect(thirdError.recordErrors.length).equal(1); + expect(thirdError.recordErrors[0]?.fieldName).equal('any-boolean'); + }); +}); diff --git a/packages/validation/test/placeholder.spec.ts b/packages/validation/test/placeholder.spec.ts deleted file mode 100644 index 99c1673..0000000 --- a/packages/validation/test/placeholder.spec.ts +++ /dev/null @@ -1,3 +0,0 @@ -describe('Placeholder', () => { - it('Dummy test', () => {}); -}); diff --git a/packages/validation/test/testUtils/expectAllValuesMatch.ts b/packages/validation/test/testUtils/expectAllValuesMatch.ts new file mode 100644 index 0000000..01c3bf9 --- /dev/null +++ b/packages/validation/test/testUtils/expectAllValuesMatch.ts @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { expect } from 'chai'; +export const expectAllValuesMatch = (values: T[]): void => { + values.forEach((value) => { + expect(value, 'All values in array are expected to match').equal(values[0]); + }); +}; diff --git a/packages/validation/test/testUtils/validateFixture.ts b/packages/validation/test/testUtils/validateFixture.ts new file mode 100644 index 0000000..54e7fef --- /dev/null +++ b/packages/validation/test/testUtils/validateFixture.ts @@ -0,0 +1,34 @@ +import type { ZodSchema } from 'zod'; + +/** + * This function tests that an object is a valid example of a Lectern types, and will throw an error + * if that validation fails. The purpose of this is to ensure that all fixtures defined for Lectern test + * validation pass all the rules as defined in the Lectern dictionary schemas, including the rules that + * are not enforced by the type system. For example, Schemas cannot have multiple fields with the same name, + * and uniqueKey restrictions cannot name fields that are missing in the schema. + * + * By adding this validation to a fixture file, the fixture will throw an error when loaded by the test + * runner, preventing the tests from runnign at all. Additionally, this function will output in the test + * logs the details from the Zod validation explaining why the provided fixture is invalid. + * + * @example + * // schemaExmple is a test fixture of a schema we want to use in a test: + * export const schemaExample = { + * name: 'some-schema', + * description: 'Invalid schema due to repeated field', + * fields: [fieldStringRequired, fieldStringRequired], + * } as const satisfies Schema; + * + * // Ensure that the text fixture + * validateFixture(schemaExample, Schema, 'schemaExample is not a valid schema.'); + * + * @param value + * @param schema + * @param message + */ +export const validateFixture = (value: NoInfer, schema: ZodSchema, message: string): void => { + const validationResult = schema.safeParse(value); + if (!validationResult.success) { + throw new Error(`Fixture failed validation test. ${message}: ${JSON.stringify(validationResult.error.flatten())}`); + } +}; diff --git a/packages/validation/test/utils/hashDataRecord.spec.ts b/packages/validation/test/utils/hashDataRecord.spec.ts new file mode 100644 index 0000000..5ab8b5d --- /dev/null +++ b/packages/validation/test/utils/hashDataRecord.spec.ts @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { expect } from 'chai'; +import { hashDataRecord } from '../../src/utils/hashDataRecord'; +import { expectAllValuesMatch } from '../testUtils/expectAllValuesMatch'; + +describe('Utils - hashDataRecord', () => { + it('Same hash given the same object', () => { + const object = { some: 'value' }; + const hashes = [hashDataRecord(object), hashDataRecord(object)]; + expectAllValuesMatch(hashes); + }); + it('Same hash when properties added in different order', () => { + const hashes = [ + hashDataRecord({ a: 1, b: '2', c: true, d: 2 }), + hashDataRecord({ a: 1, b: '2', d: 2, c: true }), + hashDataRecord({ a: 1, d: 2, b: '2', c: true }), + hashDataRecord({ a: 1, d: 2, c: true, b: '2' }), + hashDataRecord({ d: 2, a: 1, c: true, b: '2' }), + hashDataRecord({ d: 2, a: 1, b: '2', c: true }), + hashDataRecord({ d: 2, b: '2', a: 1, c: true }), + hashDataRecord({ d: 2, c: true, a: 1, b: '2' }), + hashDataRecord({ d: 2, c: true, b: '2', a: 1 }), + hashDataRecord({ d: 2, b: '2', c: true, a: 1 }), + ]; + + expectAllValuesMatch(hashes); + }); + + it('Different hash when a property is missing', () => { + const aAndB = hashDataRecord({ a: 1, b: '2' }); + const aOnly = hashDataRecord({ a: 1 }); + expect(aAndB).not.equal(aOnly); + + const aWithBUndefined = hashDataRecord({ a: 1, b: undefined }); + expect(aAndB).not.equal(aWithBUndefined); + }); + it('Same hash when a property is missing or set to undefined', () => { + const hashes = [ + hashDataRecord({ a: 1 }), + hashDataRecord({ a: 1, b: undefined }), + hashDataRecord({ a: 1, c: undefined }), + hashDataRecord({ a: 1, b: undefined, c: undefined }), + ]; + + expectAllValuesMatch(hashes); + }); + it('Same hash for empty objects', () => { + const hashes = [hashDataRecord({}), hashDataRecord({})]; + + expectAllValuesMatch(hashes); + }); +}); diff --git a/packages/validation/test/utils/rangeTest.spec.ts b/packages/validation/test/utils/rangeTest.spec.ts new file mode 100644 index 0000000..38a7973 --- /dev/null +++ b/packages/validation/test/utils/rangeTest.spec.ts @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import type { RestrictionRange } from 'dictionary'; +import { isWithinRange } from '../../src/utils/isWithinRange'; +import { expect } from 'chai'; + +describe('Utils - rangeTest', () => { + describe('Min only', () => { + it('Returns true when value greater than min', () => { + const range: RestrictionRange = { min: 0 }; + const result = isWithinRange(range, 10); + expect(result).to.be.true; + }); + it('Returns false when value lesser than min', () => { + const range: RestrictionRange = { min: 0 }; + const result = isWithinRange(range, -10); + expect(result).to.be.false; + }); + it('Returns true when value equals min', () => { + const min = 0; + const range: RestrictionRange = { min }; + const result = isWithinRange(range, min); + expect(result).to.be.true; + }); + }); + describe('Max only', () => { + it('Returns true when value lesser than max', () => { + const range: RestrictionRange = { max: 100 }; + const result = isWithinRange(range, 10); + expect(result).to.be.true; + }); + it('Returns false when value greater than max', () => { + const range: RestrictionRange = { max: 100 }; + const result = isWithinRange(range, 110); + expect(result).to.be.false; + }); + it('Returns true when value equals max', () => { + const max = 100; + const range: RestrictionRange = { max }; + const result = isWithinRange(range, max); + expect(result).to.be.true; + }); + }); + describe('Exclusive max', () => { + it('Returns true when value lesser than exclusiveMax', () => { + const range: RestrictionRange = { exclusiveMax: 100 }; + const result = isWithinRange(range, 10); + expect(result).to.be.true; + }); + it('Returns false when value greater than exclusiveMax', () => { + const range: RestrictionRange = { exclusiveMax: 100 }; + const result = isWithinRange(range, 110); + expect(result).to.be.false; + }); + it('Returns false when value equals exclusiveMax', () => { + const exclusiveMax = 100; + const range: RestrictionRange = { exclusiveMax }; + const result = isWithinRange(range, exclusiveMax); + expect(result).to.be.false; + }); + }); + describe('Exclusive min', () => { + it('Returns true when value greater than exclusiveMin', () => { + const range: RestrictionRange = { exclusiveMin: 10 }; + const result = isWithinRange(range, 20); + expect(result).to.be.true; + }); + it('Returns false when value lesser than exclusiveMin', () => { + const range: RestrictionRange = { exclusiveMin: 10 }; + const result = isWithinRange(range, 0); + expect(result).to.be.false; + }); + it('Returns false when value equals exclusiveMin', () => { + const exclusiveMin = 100; + const range: RestrictionRange = { exclusiveMin }; + const result = isWithinRange(range, exclusiveMin); + expect(result).to.be.false; + }); + }); + it('Returns true when value is within range', () => { + expect(isWithinRange({ min: 0, max: 10 }, 5)).to.be.true; + expect(isWithinRange({ min: 0, max: 10 }, 0)).to.be.true; + expect(isWithinRange({ min: 0, max: 10 }, 10)).to.be.true; + expect(isWithinRange({ min: 0, exclusiveMax: 10 }, 5)).to.be.true; + expect(isWithinRange({ min: 0, exclusiveMax: 10 }, 0)).to.be.true; + expect(isWithinRange({ exclusiveMin: 0, max: 10 }, 5)).to.be.true; + expect(isWithinRange({ exclusiveMin: 0, max: 10 }, 10)).to.be.true; + expect(isWithinRange({ exclusiveMin: 0, exclusiveMax: 10 }, 5)).to.be.true; + }); + it('Returns false when value is outside range', () => { + expect(isWithinRange({ min: 0, max: 10 }, -5)).to.be.false; + expect(isWithinRange({ min: 0, max: 10 }, 15)).to.be.false; + expect(isWithinRange({ min: 0, exclusiveMax: 10 }, -1)).to.be.false; + expect(isWithinRange({ min: 0, exclusiveMax: 10 }, 11)).to.be.false; + expect(isWithinRange({ min: 0, exclusiveMax: 10 }, 10)).to.be.false; + expect(isWithinRange({ exclusiveMin: 0, exclusiveMax: 10 }, -1)).to.be.false; + expect(isWithinRange({ exclusiveMin: 0, exclusiveMax: 10 }, 11)).to.be.false; + expect(isWithinRange({ exclusiveMin: 0, max: 10 }, 0)).to.be.false; + expect(isWithinRange({ exclusiveMin: 0, exclusiveMax: 10 }, 0)).to.be.false; + expect(isWithinRange({ exclusiveMin: 0, exclusiveMax: 10 }, 10)).to.be.false; + }); +}); diff --git a/packages/validation/test/validateDictionary/validateDictionary.spec.ts b/packages/validation/test/validateDictionary/validateDictionary.spec.ts new file mode 100644 index 0000000..dca460a --- /dev/null +++ b/packages/validation/test/validateDictionary/validateDictionary.spec.ts @@ -0,0 +1,395 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { expect } from 'chai'; +import { validateDictionary, type DictionaryValidationErrorInvalidRecords } from '../../src/validateDictionary'; +import { dictionarySingleSchemaNoRestrictions } from '../fixtures/dictionaries/dictionarySingleSchemaNoRestrictions'; +import assert from 'node:assert'; +import { dictionarySingleSchemaRequiredRestrictions } from '../fixtures/dictionaries/dictionarySingleSchemaRequiredRestrictions'; +import { dictionarySingleSchemaUniqueKeyRestriction } from '../fixtures/dictionaries/dictionarySingleSchemaUniqueKeyRestriction'; +import { dictionaryMultipleSchemasNoRestrictions } from '../fixtures/dictionaries/dictionaryMultipleSchemasNoRestrictions'; +import { dictionaryForeignKeySimple } from '../fixtures/dictionaries/foreignKey/dictionaryForeignKeySimple'; +import { dictionaryForeignKeyMultiple } from '../fixtures/dictionaries/foreignKey/dictionaryForeignKeyMultiple'; + +describe('Dictionary - validateDictionary', () => { + describe('Single Schema', () => { + it('Valid with no restrictions and empty data set', () => { + const dataSet = {}; + const result = validateDictionary(dataSet, dictionarySingleSchemaNoRestrictions); + expect(result.valid).true; + }); + it('Valid with correct data set, no restrictions', () => { + const dataSet = { + 'single-string': [{ 'any-string': '1234' }, {}, { 'any-string': 'some string' }], + }; + const result = validateDictionary(dataSet, dictionarySingleSchemaNoRestrictions); + expect(result.valid).true; + }); + it('Valid with correct data set, with restrictions', () => { + const schemaName = dictionarySingleSchemaRequiredRestrictions.schemas[0].name; + const stringField = dictionarySingleSchemaRequiredRestrictions.schemas[0].fields[0].name; + const numberField = dictionarySingleSchemaRequiredRestrictions.schemas[0].fields[1].name; + const integerField = dictionarySingleSchemaRequiredRestrictions.schemas[0].fields[2].name; + const booleanField = dictionarySingleSchemaRequiredRestrictions.schemas[0].fields[3].name; + const dataSet = { + [schemaName]: [ + { [stringField]: 'a random string', [numberField]: 98.76, [integerField]: 543, [booleanField]: false }, + { [stringField]: 'another random string', [numberField]: 87.65, [integerField]: 432, [booleanField]: true }, + ], + }; + const result = validateDictionary(dataSet, dictionarySingleSchemaRequiredRestrictions); + expect(result.valid).true; + }); + it('Invalid with recognized schema names', () => { + const wrongSchemaName = 'wrong-schema-name'; + const dataSet = { + [wrongSchemaName]: [], + }; + const result = validateDictionary(dataSet, dictionarySingleSchemaNoRestrictions); + expect(result.valid).false; + assert(result.valid === false); + + expect(result.details.length, 'Only one schema found invalid.').equal(1); + expect(result.details[0]?.reason).equal('UNRECOGNIZED_SCHEMA'); + expect(result.details[0]?.schemaName, 'Correctly names the unrecognized schema.').equal(wrongSchemaName); + }); + it('Invalid with correct data type rules', () => { + const schemaName = dictionarySingleSchemaNoRestrictions.schemas[0].name; + const fieldName = dictionarySingleSchemaNoRestrictions.schemas[0].fields[0].name; + const dataSet = { + [schemaName]: [{ [fieldName]: 1234 }], // value must be string, not number + }; + const result = validateDictionary(dataSet, dictionarySingleSchemaNoRestrictions); + expect(result.valid).false; + assert(result.valid === false); + + expect(result.details.length, 'Only one schema found invalid.').equal(1); + expect(result.details[0]?.schemaName, 'Correctly names schema with error.').equal(schemaName); + expect(result.details[0]?.reason).equal('INVALID_RECORDS'); + expect((result.details[0] as DictionaryValidationErrorInvalidRecords).invalidRecords[0]?.recordIndex).equal(0); + expect( + (result.details[0] as DictionaryValidationErrorInvalidRecords).invalidRecords[0]?.recordErrors.length, + 'Only one error found for field.', + ).equal(1); + expect( + (result.details[0] as DictionaryValidationErrorInvalidRecords).invalidRecords[0]?.recordErrors[0]?.reason, + ).equal('INVALID_VALUE_TYPE'); + }); + it('Invalid with schema uniqueKey restriction', () => { + const schemaName = dictionarySingleSchemaUniqueKeyRestriction.schemas[0].name; + // records 0 and 2 share the same unique key + const dataSet = { + [schemaName]: [ + { 'any-string': 'asdf', 'any-number': 12.34, 'any-integer': 123, 'any-boolean': true }, + { 'any-string': 'qwerty', 'any-number': 12.34, 'any-integer': 123, 'any-boolean': true }, + { 'any-string': 'asdf', 'any-number': 12.34, 'any-integer': 123, 'any-boolean': true }, + ], + }; + const result = validateDictionary(dataSet, dictionarySingleSchemaUniqueKeyRestriction); + expect(result.valid).false; + assert(result.valid === false); + + expect(result.details.length, 'Only one schema found invalid.').equal(1); + expect(result.details[0]?.schemaName, 'Correctly names schema with error.').equal(schemaName); + expect(result.details[0]?.reason).equal('INVALID_RECORDS'); + expect( + (result.details[0] as DictionaryValidationErrorInvalidRecords).invalidRecords.length, + 'Correct number of invalid records reported.', + ).equal(2); + expect( + (result.details[0] as DictionaryValidationErrorInvalidRecords).invalidRecords.some( + (invalidRecord) => invalidRecord.recordIndex === 0, + ), + 'Record 0 should be reported as invalid', + ).true; + expect( + (result.details[0] as DictionaryValidationErrorInvalidRecords).invalidRecords.some( + (invalidRecord) => invalidRecord.recordIndex === 2, + ), + 'Record 2 should be reported as invalid', + ).true; + expect( + (result.details[0] as DictionaryValidationErrorInvalidRecords).invalidRecords.every( + (invalidRecord) => invalidRecord.recordErrors.length === 1, + ), + 'Only one error for each invalid record', + ).true; + expect( + (result.details[0] as DictionaryValidationErrorInvalidRecords).invalidRecords.every( + (invalidRecord) => invalidRecord.recordErrors[0]?.reason === 'INVALID_BY_UNIQUE_KEY', + ), + 'Every invalid record is by unique key', + ).true; + }); + }); + describe('Multiple Schemas, no dictionary restrictions', () => { + it('Valid for empty data set when no restrictions on all schemas', () => { + const dataSet = {}; + const result = validateDictionary(dataSet, dictionaryMultipleSchemasNoRestrictions); + expect(result.valid).true; + }); + it('Valid for correct data with multiple schemas', () => { + const dataSet = { + 'single-string': [{ 'any-string': 'hello' }, { 'any-string': 'hello' }, { 'any-string': 'hello' }], + 'all-data-types': [ + { 'any-string': 'asdf', 'any-number': 12345.6789, 'any-integer': 1234567890, 'any-boolean': false }, + { 'any-string': 'asdf', 'any-number': 12345.6789, 'any-integer': 1234567890, 'any-boolean': false }, + { 'any-string': 'asdf', 'any-number': 12345.6789, 'any-integer': 1234567890, 'any-boolean': false }, + ], + }; + const result = validateDictionary(dataSet, dictionaryMultipleSchemasNoRestrictions); + expect(result.valid).true; + }); + it('Invalid with invalid fields, lists all errors organized by schema', () => { + const dataSet = { + 'single-string': [ + { 'any-string': 123 }, // wrong type + { 'unreocgnized-field': 123 }, + { 'any-string': 'hello', 'extra-unrecognized-field': 123 }, + ], + 'all-data-types': [ + { 'any-string': 123, 'invalid-field-name': 123 }, // invalid type and unrecognized field + { 'any-string': 'asdf', 'any-number': 12345.6789, 'any-integer': 1234567890, 'any-boolean': false }, + { 'any-string': 'asdf', 'any-number': 12345.6789, 'any-integer': 1234567890, 'any-boolean': false }, + ], + }; + const result = validateDictionary(dataSet, dictionaryMultipleSchemasNoRestrictions); + expect(result.valid).false; + assert(result.valid === false); + expect(result.details.length, 'One error for each schema.').equal(2); + + const singleStringSchemaResult = result.details.find((result) => result.schemaName === 'single-string'); + expect(singleStringSchemaResult).not.undefined; + assert(singleStringSchemaResult !== undefined); + expect(singleStringSchemaResult.reason).equal('INVALID_RECORDS'); + assert(singleStringSchemaResult.reason === 'INVALID_RECORDS'); + expect( + singleStringSchemaResult.invalidRecords.length, + 'Should be 3 invalid fields in single-string schema', + ).equal(3); + + const allDataSchemaResult = result.details.find((result) => result.schemaName === 'all-data-types'); + expect(allDataSchemaResult).not.undefined; + assert(allDataSchemaResult !== undefined); + expect(allDataSchemaResult.reason).equal('INVALID_RECORDS'); + assert(allDataSchemaResult.reason === 'INVALID_RECORDS'); + expect(allDataSchemaResult.invalidRecords.length, 'Should be 1 invalid record in all-data-types schema').equal(1); + expect( + allDataSchemaResult.invalidRecords[0]?.recordErrors.length, + 'Should be 2 invalid fields in the record ', + ).equal(2); + expect( + allDataSchemaResult.invalidRecords[0]?.recordErrors.filter( + (recordError) => recordError.reason === 'UNRECOGNIZED_FIELD', + ).length, + 'Should have 1 unrecognized field error', + ).equal(1); + expect( + allDataSchemaResult.invalidRecords[0]?.recordErrors.filter( + (recordError) => recordError.reason === 'INVALID_VALUE_TYPE', + ).length, + 'Should have 1 invalid value type error', + ).equal(1); + }); + it('Invalid with unrecognized schema amongst multiple correct schemas', () => { + const invalidSchemaName = 'invalid-schema'; + const dataSet = { + 'single-string': [], + 'all-data-types': [], + [invalidSchemaName]: [], + }; + const result = validateDictionary(dataSet, dictionaryMultipleSchemasNoRestrictions); + expect(result.valid).false; + assert(result.valid === false); + + expect(result.details[0]?.reason === 'UNRECOGNIZED_SCHEMA'); + assert(result.details[0] !== undefined && result.details[0].reason === 'UNRECOGNIZED_SCHEMA'); + expect(result.details[0].schemaName).equal(invalidSchemaName); + }); + }); + describe('Multiple Schemas, foreignKey restrictions', () => { + it('Valid with correct foreign key values', () => { + const dataSet = { + 'all-data-types': [ + { 'any-string': '1234' }, + {}, + { 'any-string': 'some string' }, + { 'any-string': 'some string' }, + { 'any-string': 'another' }, + ], + 'string-matching-foreign-string': [ + { 'string-with-foreign-key': '1234' }, + { 'string-with-foreign-key': '1234' }, + { 'string-with-foreign-key': 'some string' }, + ], + }; + const result = validateDictionary(dataSet, dictionaryForeignKeySimple); + expect(result.valid).true; + }); + + it('Invalid when breaking foreign key restriction', () => { + const dataSet = { + 'all-data-types': [{ 'any-string': '1234' }, { 'any-string': 'some string' }, { 'any-string': 'another' }], + 'string-matching-foreign-string': [{ 'string-with-foreign-key': 'invalid value' }], + }; + const result = validateDictionary(dataSet, dictionaryForeignKeySimple); + expect(result.valid).false; + assert(result.valid === false); + + expect(result.details.length).equal(1); + expect(result.details[0]?.reason).equal('INVALID_RECORDS'); + assert(result.details[0]?.reason === 'INVALID_RECORDS'); + expect(result.details[0].invalidRecords.length, 'Only 1 invalid record').equal(1); + expect(result.details[0].invalidRecords[0]?.recordIndex).equal(0); + expect(result.details[0].invalidRecords[0]?.recordErrors.length, 'Only 1 error on this record').equal(1); + expect(result.details[0].invalidRecords[0]?.recordErrors[0]?.reason).equal('INVALID_BY_FOREIGNKEY'); + assert(result.details[0].invalidRecords[0]?.recordErrors[0]?.reason === 'INVALID_BY_FOREIGNKEY'); + expect(result.details[0].invalidRecords[0]?.recordErrors[0]?.fieldName).equal('string-with-foreign-key'); + expect(result.details[0].invalidRecords[0]?.recordErrors[0]?.fieldValue).equal('invalid value'); + expect(result.details[0].invalidRecords[0]?.recordErrors[0]?.foreignSchema.schemaName).equal('all-data-types'); + expect(result.details[0].invalidRecords[0]?.recordErrors[0]?.foreignSchema.fieldName).equal('any-string'); + }); + it('Foreign key restriction not applied to undefined values', () => { + const dataSet = { + 'all-data-types': [{ 'any-string': '1234' }, { 'any-string': 'some string' }, { 'any-string': 'another' }], + 'string-matching-foreign-string': [{ 'string-with-foreign-key': undefined }], + }; + const result = validateDictionary(dataSet, dictionaryForeignKeySimple); + expect(result.valid).true; + }); + it('Valid with multiple foreign key restrictions', () => { + const dataSet = { + 'all-data-types': [ + { 'any-string': 'asdf', 'any-number': 123.45 }, + { 'any-number': 111 }, + { 'any-string': 'qwerty' }, + ], + 'single-string': [{ 'any-string': 'asdf' }, { 'any-string': 'lkjh' }, { 'any-string': 'poiuy' }], + 'multiple-foreign-keys': [ + { 'string-field': 'asdf', 'number-field': 111 }, + { 'string-field': 'asdf' }, + { 'number-field': 123.45 }, + ], + }; + const result = validateDictionary(dataSet, dictionaryForeignKeyMultiple); + expect(result.valid).true; + }); + it('Invalid with multiple foreign key restrictions, all errors listed', () => { + const dataSet = { + 'all-data-types': [ + { 'any-string': 'asdf', 'any-number': 123.45 }, + { 'any-number': 111 }, + { 'any-string': 'qwerty' }, + ], + 'single-string': [{ 'any-string': 'asdf' }, { 'any-string': 'lkjh' }, { 'any-string': 'poiuy' }], + 'multiple-foreign-keys': [ + { 'string-field': 'asdf', 'number-field': 123.45 }, + { 'string-field': 'lkjh', 'number-field': 111 }, + { 'string-field': 'unknown' }, + { 'number-field': 0 }, + ], + }; + const result = validateDictionary(dataSet, dictionaryForeignKeyMultiple); + expect(result.valid).false; + assert(result.valid === false); + + expect(result.details.length, 'One schema with errors').equal(1); + expect(result.details[0]?.reason).equal('INVALID_RECORDS'); + assert(result.details[0]?.reason === 'INVALID_RECORDS'); + + expect(result.details[0].schemaName).equal('multiple-foreign-keys'); + expect(result.details[0].invalidRecords.length, 'Expect 3 records to fail the foreign key restriction').equal(3); + + // Index 0 has no errors + expect(result.details[0].invalidRecords.find((invalidRecord) => invalidRecord.recordIndex === 0)).undefined; + + const errorIndex1 = result.details[0].invalidRecords.find((invalidRecord) => invalidRecord.recordIndex === 1); + expect(errorIndex1).not.undefined; + assert(errorIndex1 !== undefined); + expect(errorIndex1.recordErrors.length).equal(1); + expect(errorIndex1.recordErrors[0]?.reason).equal('INVALID_BY_FOREIGNKEY'); + assert(errorIndex1.recordErrors[0]?.reason === 'INVALID_BY_FOREIGNKEY'); + expect(errorIndex1.recordErrors[0].foreignSchema.schemaName).equal('all-data-types'); + expect(errorIndex1.recordErrors[0].foreignSchema.fieldName).equal('any-string'); + expect(errorIndex1.recordErrors[0].fieldName).equal('string-field'); + expect(errorIndex1.recordErrors[0].fieldValue).equal('lkjh'); + + const errorIndex2 = result.details[0].invalidRecords.find((invalidRecord) => invalidRecord.recordIndex === 2); + expect(errorIndex2).not.undefined; + assert(errorIndex2 !== undefined); + expect(errorIndex2.recordErrors.length).equal(2); + expect(errorIndex2.recordErrors[0]?.reason).equal('INVALID_BY_FOREIGNKEY'); + expect(errorIndex2.recordErrors[1]?.reason).equal('INVALID_BY_FOREIGNKEY'); + expect( + errorIndex2.recordErrors.find( + (recordError) => + recordError.reason === 'INVALID_BY_FOREIGNKEY' && recordError.foreignSchema.schemaName === 'all-data-types', + ), + 'Expect one foreign key error vs all-data-types schema', + ).not.undefined; + expect( + errorIndex2.recordErrors.find( + (recordError) => + recordError.reason === 'INVALID_BY_FOREIGNKEY' && recordError.foreignSchema.schemaName === 'single-string', + ), + 'Expect one foreign key error vs single-string schema', + ).not.undefined; + + const errorIndex3 = result.details[0].invalidRecords.find((invalidRecord) => invalidRecord.recordIndex === 3); + expect(errorIndex3).not.undefined; + assert(errorIndex3 !== undefined); + expect(errorIndex3.recordErrors.length).equal(1); + expect(errorIndex3.recordErrors[0]?.reason).equal('INVALID_BY_FOREIGNKEY'); + assert(errorIndex3.recordErrors[0]?.reason === 'INVALID_BY_FOREIGNKEY'); + expect(errorIndex3.recordErrors[0].foreignSchema.schemaName).equal('all-data-types'); + expect(errorIndex3.recordErrors[0].foreignSchema.fieldName).equal('any-number'); + expect(errorIndex3.recordErrors[0].fieldName).equal('number-field'); + expect(errorIndex3.recordErrors[0].fieldValue).equal(0); + }); + it('Invalid with mix of foreign key and field validation errors', () => { + const dataSet = { + 'all-data-types': [{ 'any-string': '1234' }, { 'any-string': 'some string' }, { 'any-string': 'another' }], + 'string-matching-foreign-string': [{ 'string-with-foreign-key': 123 }], + }; + const result = validateDictionary(dataSet, dictionaryForeignKeySimple); + expect(result.valid).false; + assert(result.valid === false); + + expect(result.details.length).equal(1); + expect(result.details[0]?.reason).equal('INVALID_RECORDS'); + assert(result.details[0]?.reason === 'INVALID_RECORDS'); + expect(result.details[0].invalidRecords.length, 'Only 1 invalid record').equal(1); + expect(result.details[0].invalidRecords[0]?.recordIndex).equal(0); + expect(result.details[0].invalidRecords[0]?.recordErrors.length, 'Expect 2 errors on this record').equal(2); + + const valueTypeError = result.details[0].invalidRecords[0]?.recordErrors.find( + (recordError) => recordError.reason === 'INVALID_VALUE_TYPE', + ); + expect(valueTypeError).not.undefined; + assert(valueTypeError !== undefined); + + const foreignKeyError = result.details[0].invalidRecords[0]?.recordErrors.find( + (recordError) => recordError.reason === 'INVALID_BY_FOREIGNKEY', + ); + expect(foreignKeyError).not.undefined; + assert(foreignKeyError !== undefined); + }); + // TODO: foreign keys on fields that are arrays need their own rules defined, currently they will not resolve correctly. + }); +}); diff --git a/packages/validation/test/validateField/restrictions/testCodeList.spec.ts b/packages/validation/test/validateField/restrictions/testCodeList.spec.ts new file mode 100644 index 0000000..a338648 --- /dev/null +++ b/packages/validation/test/validateField/restrictions/testCodeList.spec.ts @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { expect } from 'chai'; +import assert from 'node:assert'; +import { testCodeList } from '../../../src/validateField/restrictions'; +import { codeListInteger, codeListNumber, codeListString } from '../../fixtures/restrictions/codeListsFixtures'; + +describe('Field - Restrictions - testCodeList', () => { + describe('Single Value', () => { + it('Invalid when value is not in string list', () => { + expect(testCodeList(codeListString, 'space shuttle').valid).false; + expect(testCodeList(codeListString, 'asdfl;2309dsajlkxcnm').valid).false; + expect(testCodeList(codeListString, 'donuts').valid).false; // extra letter + expect(testCodeList(codeListString, 'aple').valid).false; // missing letter + expect(testCodeList(codeListString, '').valid).false; // empty string special case + }); + + it('Invalid when value is not in integer list', () => { + expect(testCodeList(codeListInteger, 0).valid).false; // special case 0 + expect(testCodeList(codeListInteger, 4).valid).false; // number not in list + expect(testCodeList(codeListInteger, -1).valid).false; // negated value from list + expect(testCodeList(codeListInteger, 13.1).valid).false; // close with decimal + }); + + it('Invalid when value is not in number list', () => { + expect(testCodeList(codeListNumber, 1).valid).false; // close + expect(testCodeList(codeListNumber, 1.6).valid).false; // closer + expect(testCodeList(codeListNumber, 5.19258).valid).false; // number not in list + expect(testCodeList(codeListNumber, -2.41421).valid).false; // negated value from list + expect(testCodeList(codeListNumber, 3.30277).valid).false; // off by 0.00001 + expect(testCodeList(codeListNumber, NaN).valid).false; // special case NaN + expect(testCodeList(codeListNumber, Infinity).valid).false; // special case Infinity + }); + it('Valid when value is in string list', () => { + expect(testCodeList(codeListString, 'Apple').valid).true; + expect(testCodeList(codeListString, 'Banana').valid).true; + expect(testCodeList(codeListString, 'Carrot').valid).true; + expect(testCodeList(codeListString, 'Donut').valid).true; + }); + it('Valid when value is in string list with different case', () => { + expect(testCodeList(codeListString, 'apple').valid).true; + expect(testCodeList(codeListString, 'aPpLe').valid).true; + expect(testCodeList(codeListString, 'BANANA').valid).true; + }); + it('Valid when value is in string list with extra whitespace', () => { + expect(testCodeList(codeListString, ' Apple').valid).true; + expect(testCodeList(codeListString, ' Banana ').valid).true; + expect(testCodeList(codeListString, 'Donut ').valid).true; + }); + it('Valid when value is in string list with extra whitespace and different case', () => { + expect(testCodeList(codeListString, ' cARRoT ').valid).true; + }); + it('Valid when value is in integer list', () => { + expect(testCodeList(codeListInteger, 1).valid).true; + expect(testCodeList(codeListInteger, 2).valid).true; + expect(testCodeList(codeListInteger, 3).valid).true; + expect(testCodeList(codeListInteger, 5).valid).true; + expect(testCodeList(codeListInteger, 8).valid).true; + expect(testCodeList(codeListInteger, 13).valid).true; + expect(testCodeList(codeListInteger, 24).valid).true; + }); + it('Valid when value is in number list', () => { + expect(testCodeList(codeListNumber, 1.61803).valid).true; + expect(testCodeList(codeListNumber, 2.41421).valid).true; + expect(testCodeList(codeListNumber, 3.30278).valid).true; + expect(testCodeList(codeListNumber, 4.23607).valid).true; + }); + it('Valid when value is undefined', () => { + expect(testCodeList(codeListInteger, undefined).valid).true; + expect(testCodeList(codeListNumber, undefined).valid).true; + expect(testCodeList(codeListString, undefined).valid).true; + }); + it('Valid when value is not a boolean', () => { + expect(testCodeList(codeListInteger, true).valid).true; + expect(testCodeList(codeListNumber, false).valid).true; + }); + }); + describe('Array Value', () => { + it('Valid when all array items are valid', () => { + expect(testCodeList(codeListString, ['apple', 'banana', 'apple', 'donut']).valid).true; + expect(testCodeList(codeListInteger, [1, 1, 1, 1, 1, 5, 8, 13]).valid).true; + expect(testCodeList(codeListNumber, [3.30278, 2.41421]).valid).true; + }); + it('Valid when array is empty', () => { + expect(testCodeList(codeListString, []).valid).true; + expect(testCodeList(codeListInteger, []).valid).true; + expect(testCodeList(codeListNumber, []).valid).true; + }); + it('Invalid when any item is invalid', () => { + expect(testCodeList(codeListString, ['linux', 'apple', 'turnip', 'banana', 'apple', 'donut']).valid).false; + expect(testCodeList(codeListInteger, [1, 1, 1, 1, 1, 6, 8, 13]).valid).false; + expect(testCodeList(codeListNumber, [12.34]).valid).false; + }); + it('Identifies the invalid items', () => { + const result = testCodeList(codeListString, ['linux', 'apple', 'turnip', 'banana', 'apple', 'donut']); + expect(result.valid).false; + assert(!result.valid); + + expect(Array.isArray(result.details.invalidItems)).true; + assert(Array.isArray(result.details.invalidItems)); + + expect(result.details.invalidItems.length).equal(2); + expect(result.details.invalidItems[0]?.position).equal(0); + expect(result.details.invalidItems[0]?.value).equal('linux'); + expect(result.details.invalidItems[1]?.position).equal(2); + expect(result.details.invalidItems[1]?.value).equal('turnip'); + }); + }); +}); diff --git a/packages/validation/test/validateField/restrictions/testRange.spec.ts b/packages/validation/test/validateField/restrictions/testRange.spec.ts new file mode 100644 index 0000000..e69de29 diff --git a/packages/validation/test/validateField/restrictions/testRegex.spec.ts b/packages/validation/test/validateField/restrictions/testRegex.spec.ts new file mode 100644 index 0000000..e015ab3 --- /dev/null +++ b/packages/validation/test/validateField/restrictions/testRegex.spec.ts @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { expect } from 'chai'; +import assert from 'node:assert'; +import { testRegex } from '../../../src/validateField/restrictions'; +import { regexYearMonthDay } from '../../fixtures/restrictions/regexFixtures'; + +const validDateSingle = '1989-11-09'; +const validDateArray = ['1867-07-01', '1969-07-16', '1989-12-17']; + +describe('Field - Restrictions - testRegex', () => { + describe('Single Value', () => { + it('Valid for matching values', () => { + expect(testRegex(regexYearMonthDay, validDateSingle).valid).true; + }); + it('Valid when value is undefined', () => { + expect(testRegex(regexYearMonthDay, undefined).valid).true; + }); + it('Valid for non-string types', () => { + expect(testRegex(regexYearMonthDay, 0).valid).true; + expect(testRegex(regexYearMonthDay, 123).valid).true; + expect(testRegex(regexYearMonthDay, 4.56).valid).true; + expect(testRegex(regexYearMonthDay, true).valid).true; + expect(testRegex(regexYearMonthDay, false).valid).true; + }); + it('Invalid for values with incorrect format', () => { + expect(testRegex(regexYearMonthDay, 'November 9th, 1989').valid).false; + expect(testRegex(regexYearMonthDay, '19891109').valid).false; + expect(testRegex(regexYearMonthDay, 'random string !&^1234').valid).false; + }); + it('Invalid for empty string', () => { + expect(testRegex(regexYearMonthDay, '').valid).false; + }); + }); + describe('Array Value', () => { + it('Valid for array of matching values', () => { + expect(testRegex(regexYearMonthDay, validDateArray).valid).true; + expect(testRegex(regexYearMonthDay, [...validDateArray, validDateSingle]).valid).true; + }); + it('Valid for empty array', () => { + expect(testRegex(regexYearMonthDay, []).valid).true; + }); + it('Invalid when one value in array is invalid', () => { + expect(testRegex(regexYearMonthDay, ['invalid']).valid).false; + expect(testRegex(regexYearMonthDay, [...validDateArray, 'invalid', validDateSingle, 'another invalid']).valid) + .false; + }); + + it('Identifies the invalid items', () => { + const result = testRegex(regexYearMonthDay, [ + ...validDateArray, + 'invalid', + validDateSingle, + 'another invalid', + 'third invalid', + ]); + expect(result.valid).false; + assert(!result.valid); + + expect(Array.isArray(result.details.invalidItems)).true; + assert(Array.isArray(result.details.invalidItems)); + + expect(result.details.invalidItems.length).equal(3); + expect(result.details.invalidItems[0]?.position).equal(validDateArray.length); + expect(result.details.invalidItems[0]?.value).equal('invalid'); + expect(result.details.invalidItems[1]?.position).equal(validDateArray.length + 2); + expect(result.details.invalidItems[1]?.value).equal('another invalid'); + expect(result.details.invalidItems[2]?.position).equal(validDateArray.length + 3); + expect(result.details.invalidItems[2]?.value).equal('third invalid'); + }); + }); +}); diff --git a/packages/validation/test/validateField/restrictions/testRequired.spec.ts b/packages/validation/test/validateField/restrictions/testRequired.spec.ts new file mode 100644 index 0000000..17235ca --- /dev/null +++ b/packages/validation/test/validateField/restrictions/testRequired.spec.ts @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { expect } from 'chai'; +import { testRequired } from '../../../src/validateField/restrictions'; + +describe('Field - Restrictions - testRequired', () => { + describe('Restriction is false', () => { + it('Valid with undefined', () => { + expect(testRequired(false, undefined).valid).true; + }); + it('Valid with empty string', () => { + expect(testRequired(false, '').valid).true; + }); + it('Valid with empty array', () => { + expect(testRequired(false, []).valid).true; + }); + it('Valid with all value types', () => { + // boolean + boolean[] + expect(testRequired(false, false).valid).true; + expect(testRequired(false, true).valid).true; + expect(testRequired(false, [false]).valid).true; + // string + string[] + expect(testRequired(false, 'hello').valid).true; + expect(testRequired(false, ['hello']).valid).true; + // number + number[] + expect(testRequired(false, 0).valid).true; + expect(testRequired(false, [123]).valid).true; + }); + }); + describe('Restriction is true', () => { + it('Valid with any all value types', () => { + // boolean + boolean[] + expect(testRequired(true, false).valid).true; + expect(testRequired(true, true).valid).true; + expect(testRequired(true, [false]).valid).true; + // string + string[] + expect(testRequired(true, 'hello').valid).true; + expect(testRequired(true, ['hello']).valid).true; + // number + number[] + expect(testRequired(true, 0).valid).true; + expect(testRequired(true, [123]).valid).true; + }); + it('Invalid with undefined', () => { + expect(testRequired(true, undefined).valid).false; + }); + it('Invalid with empty string', () => { + expect(testRequired(true, '').valid).false; + }); + it('Invalid with empty array', () => { + expect(testRequired(true, []).valid).false; + }); + it('Invalid with array with some missing values', () => { + expect(testRequired(true, ['hello', '', 'world']).valid).false; + }); + }); +}); diff --git a/packages/validation/test/validateField/validateField.spec.ts b/packages/validation/test/validateField/validateField.spec.ts new file mode 100644 index 0000000..a08a2cd --- /dev/null +++ b/packages/validation/test/validateField/validateField.spec.ts @@ -0,0 +1,399 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { expect } from 'chai'; +import type { SchemaStringField } from 'dictionary'; +import assert from 'node:assert'; +import { validateField } from '../../src'; +import { fieldStringManyRestrictions } from '../fixtures/fields/multipleRestrictions/fieldStringManyRestrictions'; +import { fieldBooleanNoRestriction } from '../fixtures/fields/noRestrictions/fieldBooleanNoRestriction'; +import { fieldIntegerNoRestriction } from '../fixtures/fields/noRestrictions/fieldIntegerNoRestriction'; +import { fieldNumberNoRestriction } from '../fixtures/fields/noRestrictions/fieldNumberNoRestriction'; +import { fieldStringNoRestriction } from '../fixtures/fields/noRestrictions/fieldStringNoRestriction'; +import { fieldNumberArrayCodeList } from '../fixtures/fields/simpleRestrictions/number/fieldNumberArrayCodeList'; +import { fieldNumberRange } from '../fixtures/fields/simpleRestrictions/number/fieldNumberRange'; +import { fieldStringArrayRequired } from '../fixtures/fields/simpleRestrictions/string/fieldStringArrayRequired'; +import { fieldStringCodeList } from '../fixtures/fields/simpleRestrictions/string/fieldStringCodeList'; +import { fieldStringRegex } from '../fixtures/fields/simpleRestrictions/string/fieldStringRegex'; +import { fieldStringRequired } from '../fixtures/fields/simpleRestrictions/string/fieldStringRequired'; + +const emptyDataRecord = {}; + +describe('Field - validateField', () => { + describe('No Restrictions', () => { + // These validations are on the data value type passed to the validator, + // and ensuring that undefined values are accepted since these fields do not have a required restriction. + describe('String type, no restrictions', () => { + it('Valid for undefined', () => { + expect(validateField(undefined, emptyDataRecord, fieldStringNoRestriction).valid).true; + }); + it('Valid for correct type', () => { + // String Type + expect(validateField('asdf', emptyDataRecord, fieldStringNoRestriction).valid).true; + expect(validateField('', emptyDataRecord, fieldStringNoRestriction).valid).true; + expect(validateField('2001-02-03', emptyDataRecord, fieldStringNoRestriction).valid).true; + expect(validateField('this is a story all about how my life', emptyDataRecord, fieldStringNoRestriction).valid) + .true; + }); + it('Invalid when given wrong type', () => { + expect(validateField(123, emptyDataRecord, fieldStringNoRestriction).valid).false; + expect(validateField(0, emptyDataRecord, fieldStringNoRestriction).valid).false; + expect(validateField(NaN, emptyDataRecord, fieldStringNoRestriction).valid).false; + expect(validateField(Infinity, emptyDataRecord, fieldStringNoRestriction).valid).false; + expect(validateField(true, emptyDataRecord, fieldStringNoRestriction).valid).false; + expect(validateField(false, emptyDataRecord, fieldStringNoRestriction).valid).false; + expect(validateField([], emptyDataRecord, fieldStringNoRestriction).valid).false; + expect(validateField([false], emptyDataRecord, fieldStringNoRestriction).valid).false; + expect(validateField([123, 456, 789], emptyDataRecord, fieldStringNoRestriction).valid).false; + expect(validateField(['abc', 'def', 'xyz'], emptyDataRecord, fieldStringNoRestriction).valid).false; + }); + it('Invalid result for wrong type includes expected type info', () => { + const testField: SchemaStringField = fieldStringNoRestriction; + const result = validateField(123, emptyDataRecord, testField); + expect(result.valid).false; + assert(!result.valid); + + expect(result.details.reason).equal('INVALID_VALUE_TYPE'); + assert(result.details.reason === 'INVALID_VALUE_TYPE'); + + expect(result.details.isArray).equal(!!testField.isArray); // needs the !! to convert to boolean since `testField.isArray` is undefined + expect(result.details.valueType).equal(testField.valueType); + }); + }); + describe('Boolean type, no restrictions', () => { + it('Valid for undefined', () => { + expect(validateField(undefined, emptyDataRecord, fieldBooleanNoRestriction).valid).true; + }); + it('Valid for correct type', () => { + // String Type + expect(validateField(true, emptyDataRecord, fieldBooleanNoRestriction).valid).true; + expect(validateField(false, emptyDataRecord, fieldBooleanNoRestriction).valid).true; + }); + it('Invalid when given wrong type', () => { + expect(validateField(123, emptyDataRecord, fieldBooleanNoRestriction).valid).false; + expect(validateField(0, emptyDataRecord, fieldBooleanNoRestriction).valid).false; + expect(validateField('true', emptyDataRecord, fieldBooleanNoRestriction).valid).false; + expect(validateField('false', emptyDataRecord, fieldBooleanNoRestriction).valid).false; + expect(validateField('random string', emptyDataRecord, fieldBooleanNoRestriction).valid).false; + expect(validateField([], emptyDataRecord, fieldBooleanNoRestriction).valid).false; + expect(validateField([false], emptyDataRecord, fieldBooleanNoRestriction).valid).false; + expect(validateField([123, 456, 789], emptyDataRecord, fieldBooleanNoRestriction).valid).false; + expect(validateField(['abc', 'def', 'xyz'], emptyDataRecord, fieldBooleanNoRestriction).valid).false; + expect(validateField(NaN, emptyDataRecord, fieldBooleanNoRestriction).valid).false; + expect(validateField(Infinity, emptyDataRecord, fieldBooleanNoRestriction).valid).false; + }); + }); + describe('Integer type, no restrictions', () => { + it('Valid for undefined', () => { + expect(validateField(undefined, emptyDataRecord, fieldIntegerNoRestriction).valid).true; + }); + it('Valid for correct type', () => { + // String Type + expect(validateField(0, emptyDataRecord, fieldIntegerNoRestriction).valid).true; + expect(validateField(1, emptyDataRecord, fieldIntegerNoRestriction).valid).true; + expect(validateField(5.0, emptyDataRecord, fieldIntegerNoRestriction).valid).true; + expect(validateField(9999, emptyDataRecord, fieldIntegerNoRestriction).valid).true; + expect(validateField(-10, emptyDataRecord, fieldIntegerNoRestriction).valid).true; + }); + it('Invalid when given wrong type', () => { + expect(validateField('random string', emptyDataRecord, fieldIntegerNoRestriction).valid).false; + expect(validateField(123.456, emptyDataRecord, fieldIntegerNoRestriction).valid).false; + expect(validateField(-789.01, emptyDataRecord, fieldIntegerNoRestriction).valid).false; + expect(validateField(NaN, emptyDataRecord, fieldIntegerNoRestriction).valid).false; + expect(validateField(Infinity, emptyDataRecord, fieldIntegerNoRestriction).valid).false; + expect(validateField(true, emptyDataRecord, fieldIntegerNoRestriction).valid).false; + expect(validateField(false, emptyDataRecord, fieldIntegerNoRestriction).valid).false; + expect(validateField([], emptyDataRecord, fieldIntegerNoRestriction).valid).false; + expect(validateField([false], emptyDataRecord, fieldIntegerNoRestriction).valid).false; + expect(validateField([123, 456, 789], emptyDataRecord, fieldIntegerNoRestriction).valid).false; + expect(validateField(['abc', 'def', 'xyz'], emptyDataRecord, fieldIntegerNoRestriction).valid).false; + }); + }); + describe('Number type, no restrictions', () => { + it('Valid for undefined', () => { + expect(validateField(undefined, emptyDataRecord, fieldNumberNoRestriction).valid).true; + }); + it('Valid for correct type', () => { + // String Type + expect(validateField(0, emptyDataRecord, fieldNumberNoRestriction).valid).true; + expect(validateField(0.0001, emptyDataRecord, fieldNumberNoRestriction).valid).true; + expect(validateField(1, emptyDataRecord, fieldNumberNoRestriction).valid).true; + expect(validateField(9999, emptyDataRecord, fieldNumberNoRestriction).valid).true; + expect(validateField(-10.123, emptyDataRecord, fieldNumberNoRestriction).valid).true; + }); + it('Invalid when given wrong type', () => { + expect(validateField('random string', emptyDataRecord, fieldNumberNoRestriction).valid).false; + expect(validateField('123.456', emptyDataRecord, fieldNumberNoRestriction).valid).false; + expect(validateField(NaN, emptyDataRecord, fieldNumberNoRestriction).valid).false; + expect(validateField(Infinity, emptyDataRecord, fieldNumberNoRestriction).valid).false; + expect(validateField(true, emptyDataRecord, fieldNumberNoRestriction).valid).false; + expect(validateField(false, emptyDataRecord, fieldNumberNoRestriction).valid).false; + expect(validateField([], emptyDataRecord, fieldNumberNoRestriction).valid).false; + expect(validateField([false], emptyDataRecord, fieldNumberNoRestriction).valid).false; + expect(validateField([123, 456, 789], emptyDataRecord, fieldNumberNoRestriction).valid).false; + expect(validateField(['abc', 'def', 'xyz'], emptyDataRecord, fieldNumberNoRestriction).valid).false; + }); + }); + }); + + describe('Simple Restrictions', () => { + describe('String with CodeList', () => { + it('Valid with value in list', () => { + expect(validateField('apple', emptyDataRecord, fieldStringCodeList).valid).true; + }); + it('Invalid with value not in list', () => { + const result = validateField('hockey puck', emptyDataRecord, fieldStringCodeList); + expect(result.valid).false; + assert(result.valid === false); + + expect(result.details.reason).equal('INVALID_BY_RESTRICTION'); + assert(result.details.reason === 'INVALID_BY_RESTRICTION'); + + expect(result.details.errors.length).equal(1); + expect(result.details.errors[0]?.invalidItems).undefined; + expect(result.details.errors[0]?.message).not.undefined; + expect(result.details.errors[0]?.restriction.type).equal('codeList'); + expect(result.details.errors[0]?.restriction.rule).deep.equal(fieldStringCodeList.restrictions?.codeList); + }); + }); + describe('String with Regex', () => { + it('Valid with value matching regex', () => { + expect(validateField('2063-04-05', emptyDataRecord, fieldStringRegex).valid).true; + }); + it('Invalid with value not matching regex', () => { + const result = validateField('April 5, 2063', emptyDataRecord, fieldStringRegex); + expect(result.valid).false; + expect(result.valid).false; + assert(result.valid === false); + + expect(result.details.reason).equal('INVALID_BY_RESTRICTION'); + assert(result.details.reason === 'INVALID_BY_RESTRICTION'); + expect(result.details.errors[0]?.restriction.type).equal('regex'); + expect(result.details.errors[0]?.restriction.rule).equal(fieldStringRegex.restrictions?.regex); + }); + }); + describe('String with Required', () => { + it('Valid with any string value', () => { + expect( + validateField('hello from the past. I hope the future is cool.', emptyDataRecord, fieldStringRequired).valid, + ).true; + }); + it('Invalid with empty string', () => { + const testValue = ''; + const result = validateField(testValue, emptyDataRecord, fieldStringRequired); + expect(result.valid).false; + assert(result.valid === false); + + expect(result.details.reason).equal('INVALID_BY_RESTRICTION'); + assert(result.details.reason === 'INVALID_BY_RESTRICTION'); + + expect(result.details.errors.length).equal(1); + expect(result.details.errors[0]?.invalidItems).undefined; + expect(result.details.errors[0]?.message).not.undefined; + expect(result.details.errors[0]?.restriction.type).equal('required'); + expect(result.details.errors[0]?.restriction.rule).equal(fieldStringRequired.restrictions?.required); + }); + }); + describe('Number with Range', () => { + it('Valid with numbers within range', () => { + // range on test fixture is regexPercent which is min 0, max 100 + expect(validateField(0, emptyDataRecord, fieldNumberRange).valid).true; // extreme value low + expect(validateField(100, emptyDataRecord, fieldNumberRange).valid).true; // exterme value high + expect(validateField(10, emptyDataRecord, fieldNumberRange).valid).true; + expect(validateField(20.1, emptyDataRecord, fieldNumberRange).valid).true; + expect(validateField(33, emptyDataRecord, fieldNumberRange).valid).true; + expect(validateField(99, emptyDataRecord, fieldNumberRange).valid).true; + }); + it('Valid with undefined', () => { + expect(validateField(undefined, emptyDataRecord, fieldNumberRange).valid).true; + }); + it('Invalid with number out of range', () => { + // range on test fixture is regexPercent which is min 0, max 100 + const result = validateField(101, emptyDataRecord, fieldNumberRange); + expect(result.valid).false; + assert(result.valid === false); + + expect(result.details.reason).equal('INVALID_BY_RESTRICTION'); + assert(result.details.reason === 'INVALID_BY_RESTRICTION'); + + expect(result.details.errors.length).equal(1); + expect(result.details.errors[0]?.invalidItems).undefined; + expect(result.details.errors[0]?.message).not.undefined; + expect(result.details.errors[0]?.restriction.type).equal('range'); + expect(result.details.errors[0]?.restriction.rule).equal(fieldNumberRange.restrictions?.range); + }); + }); + describe('String Array with Required', () => { + it('Valid with array of any string values', () => { + expect(validateField(['ok'], emptyDataRecord, fieldStringArrayRequired).valid).true; + expect(validateField(['fine', 'great', 'charmed'], emptyDataRecord, fieldStringArrayRequired).valid).true; + }); + + it('Invalid when array is empty', () => { + const testValue: string[] = []; + const result = validateField(testValue, emptyDataRecord, fieldStringArrayRequired); + expect(result.valid).false; + assert(result.valid === false); + + expect(result.details.reason).equal('INVALID_BY_RESTRICTION'); + assert(result.details.reason === 'INVALID_BY_RESTRICTION'); + + expect(result.details.errors.length).equal(1); + expect(result.details.errors[0]?.invalidItems).undefined; + expect(result.details.errors[0]?.message).not.undefined; + expect(result.details.errors[0]?.restriction.type).equal('required'); + expect(result.details.errors[0]?.restriction.rule).equal(fieldStringArrayRequired.restrictions?.required); + }); + it('Invalid when array has an empty string', () => { + const testValue = ['this array has an illegal value', '']; + const result = validateField(testValue, emptyDataRecord, fieldStringArrayRequired); + expect(result.valid).false; + assert(result.valid === false); + + expect(result.details.reason).equal('INVALID_BY_RESTRICTION'); + assert(result.details.reason === 'INVALID_BY_RESTRICTION'); + + expect(result.details.errors.length).equal(1); + expect(result.details.errors[0]?.message).not.undefined; + expect(result.details.errors[0]?.restriction.type).equal('required'); + expect(result.details.errors[0]?.restriction.rule).equal(fieldStringArrayRequired.restrictions?.required); + expect(Array.isArray(result.details.errors[0]?.invalidItems)).true; + assert(Array.isArray(result.details.errors[0]?.invalidItems)); + + expect(result.details.errors[0]?.invalidItems[0]?.position).equal(1); + expect(result.details.errors[0]?.invalidItems[0]?.value).equal(''); + }); + }); + describe('Number Array with codeList', () => { + it('Valid with all values in list', () => { + expect(validateField([2.41421, 1.61803, 4.23607, 4.23607], emptyDataRecord, fieldNumberArrayCodeList).valid) + .true; + }); + it('Valid with empty list', () => { + expect(validateField([], emptyDataRecord, fieldNumberArrayCodeList).valid).true; + }); + it('Valid with undefined', () => { + expect(validateField(undefined, emptyDataRecord, fieldNumberArrayCodeList).valid).true; + }); + it('Invalid with value not in list', () => { + const result = validateField( + [2.41421, 1.61803, 4.23607, 1.61803, 4], + emptyDataRecord, + fieldNumberArrayCodeList, + ); + expect(result.valid).false; + assert(result.valid === false); + + expect(result.details.reason).equal('INVALID_BY_RESTRICTION'); + assert(result.details.reason === 'INVALID_BY_RESTRICTION'); + + expect(result.details.errors.length).equal(1); + expect(result.details.errors[0]?.message).not.undefined; + expect(result.details.errors[0]?.restriction.type).equal('codeList'); + expect(result.details.errors[0]?.restriction.rule).deep.equal(fieldNumberArrayCodeList.restrictions?.codeList); + + expect(Array.isArray(result.details.errors[0]?.invalidItems)).true; + assert(Array.isArray(result.details.errors[0]?.invalidItems)); + + expect(result.details.errors[0]?.invalidItems[0]?.position).equal(4); + expect(result.details.errors[0]?.invalidItems[0]?.value).equal(4); + }); + }); + }); + describe('Multiple Restrictions', () => { + describe('String with multiple restrictions: required, regex, codeList', () => { + it('Valid when matching all conditions', () => { + expect(validateField('2001-01-01', emptyDataRecord, fieldStringManyRestrictions).valid).true; + expect(validateField('2002-02-02', emptyDataRecord, fieldStringManyRestrictions).valid).true; + expect(validateField('2003-03-03', emptyDataRecord, fieldStringManyRestrictions).valid).true; + }); + it('Invalid when failing codeList condition', () => { + const result = validateField('1999-01-02', emptyDataRecord, fieldStringManyRestrictions); + expect(result.valid).false; + assert(result.valid === false); + + expect(result.details.reason).equal('INVALID_BY_RESTRICTION'); + assert(result.details.reason === 'INVALID_BY_RESTRICTION'); + + expect(result.details.errors.length).equal(1); + expect(result.details.errors[0]?.message).not.undefined; + expect(result.details.errors[0]?.invalidItems).undefined; + expect(result.details.errors[0]?.restriction.type).equal('codeList'); + expect(result.details.errors[0]?.restriction.rule).deep.equal( + fieldStringManyRestrictions.restrictions?.codeList, + ); + }); + it('Invalid when failing regex condition', () => { + const result = validateField('April 4, 2004', emptyDataRecord, fieldStringManyRestrictions); + expect(result.valid).false; + assert(result.valid === false); + + expect(result.details.reason).equal('INVALID_BY_RESTRICTION'); + assert(result.details.reason === 'INVALID_BY_RESTRICTION'); + + expect(result.details.errors.length).equal(1); + expect(result.details.errors[0]?.message).not.undefined; + expect(result.details.errors[0]?.invalidItems).undefined; + expect(result.details.errors[0]?.restriction.type).equal('regex'); + expect(result.details.errors[0]?.restriction.rule).deep.equal(fieldStringManyRestrictions.restrictions?.regex); + }); + it('Invalid when failing required condition', () => { + const result = validateField(undefined, emptyDataRecord, fieldStringManyRestrictions); + expect(result.valid).false; + assert(result.valid === false); + + expect(result.details.reason).equal('INVALID_BY_RESTRICTION'); + assert(result.details.reason === 'INVALID_BY_RESTRICTION'); + + expect(result.details.errors.length).equal(1); + expect(result.details.errors[0]?.message).not.undefined; + expect(result.details.errors[0]?.invalidItems).undefined; + expect(result.details.errors[0]?.restriction.type).equal('required'); + expect(result.details.errors[0]?.restriction.rule).deep.equal( + fieldStringManyRestrictions.restrictions?.required, + ); + }); + it('Invalid when failing regex and codeList restrictions', () => { + const result = validateField('this value is wrong', emptyDataRecord, fieldStringManyRestrictions); + expect(result.valid).false; + assert(result.valid === false); + + expect(result.details.reason).equal('INVALID_BY_RESTRICTION'); + assert(result.details.reason === 'INVALID_BY_RESTRICTION'); + + expect(result.details.errors.length).equal(2); + expect(result.details.errors[0]?.message).not.undefined; + expect(result.details.errors[0]?.invalidItems).undefined; + const errors = result.details.errors; + const regexError = errors.find( + (error) => + error.restriction.type === 'regex' && + error.restriction.rule === fieldStringManyRestrictions.restrictions?.regex, + ); + expect(regexError).exist; + const codeListError = errors.find( + (error) => + error.restriction.type === 'codeList' && + error.restriction.rule === fieldStringManyRestrictions.restrictions?.codeList, + ); + expect(codeListError).exist; + }); + }); + }); +}); diff --git a/packages/validation/test/validateRecord/validateRecord.spec.ts b/packages/validation/test/validateRecord/validateRecord.spec.ts new file mode 100644 index 0000000..d338f5d --- /dev/null +++ b/packages/validation/test/validateRecord/validateRecord.spec.ts @@ -0,0 +1,204 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { expect } from 'chai'; +import assert from 'node:assert'; +import { validateRecord } from '../../src'; +import { schemaSingleString } from '../fixtures/schema/schemaSingleString'; + +import { schemaAllDataTypesRequired } from '../fixtures/schema/schemaAllDataTypesRequired'; +import { schemaSingleStringRequired } from '../fixtures/schema/schemaSingleStringRequired'; +import { schemaAllDataTypesMixedRestrictions } from '../fixtures/schema/schemaAllDataMixedRestrictions'; +import { fieldStringManyRestrictions } from '../fixtures/fields/multipleRestrictions/fieldStringManyRestrictions'; +import { fieldNumberArrayCodeList } from '../fixtures/fields/simpleRestrictions/number/fieldNumberArrayCodeList'; +import { fieldIntegerRequired } from '../fixtures/fields/simpleRestrictions/integer/fieldIntegerRequired'; + +describe('Record - validateRecord', () => { + it('Valid with single field with valid value, no restrictions', () => { + expect(validateRecord({ [schemaSingleString.fields[0].name]: 'test value' }, schemaSingleString).valid).true; + }); + it('Valid with all fields with valid values, many restrictions', () => { + const result = validateRecord( + { + 'complicated-multi-restriction-rules': '2001-01-01', + 'number-code-list': [1.61803, 2.41421], + 'integer-required': 123, + 'any-boolean': false, + }, + schemaAllDataTypesMixedRestrictions, + ); + expect(result.valid).true; + }); + it('Valid when missing optional fields', () => { + expect(validateRecord({}, schemaSingleString).valid).true; + }); + describe('Unrecognized Fields', () => { + it('Invalid when given record with unrecognized field', () => { + const unknownFieldName = 'unknown-field-name'; + const unknownFieldValue = 123; + const result = validateRecord({ [unknownFieldName]: unknownFieldValue }, schemaSingleString); + expect(result.valid).false; + assert(result.valid === false); + + expect(result.details.length).equal(1); + expect(result.details[0]?.reason).equal('UNRECOGNIZED_FIELD'); + expect(result.details[0]?.fieldName).equal(unknownFieldName); + expect(result.details[0]?.fieldValue).equal(unknownFieldValue); + }); + it('Invalid with multiple unrecognized fields, reports each error', () => { + const unknownFieldNameA = 'unknown-field-name-a'; + const unknownFieldValueA = 123; + const unknownFieldNameB = 'unknown-field-name-b'; + const unknownFieldValueB = 'abcdefg'; + const record = { [unknownFieldNameA]: unknownFieldValueA, [unknownFieldNameB]: unknownFieldValueB }; + const result = validateRecord(record, schemaSingleString); + + expect(result.valid).false; + assert(result.valid === false); + + // Confirmed result is invalid, make sure we have multiple unrecognized field errors with the correct info + expect(result.details.length).equal(2); + const fieldErrorA = result.details.find((error) => error.fieldName === unknownFieldNameA); + expect(fieldErrorA).not.undefined; + assert(fieldErrorA !== undefined); + expect(fieldErrorA.reason).equal('UNRECOGNIZED_FIELD'); + expect(fieldErrorA.fieldValue).equal(record[unknownFieldNameA]); + + const fieldErrorB = result.details.find((error) => error.fieldName === unknownFieldNameB); + expect(fieldErrorB).not.undefined; + assert(fieldErrorB !== undefined); + expect(fieldErrorB.reason).equal('UNRECOGNIZED_FIELD'); + expect(fieldErrorB.fieldValue).equal(record[unknownFieldNameB]); + }); + }); + + describe('Field Restriction Errors', () => { + it('Invalid when missing required field', () => { + const result = validateRecord({}, schemaSingleStringRequired); + expect(result.valid).false; + assert(result.valid === false); + + expect(result.details.length).equal(1); + expect(result.details[0]?.reason).equal('INVALID_BY_RESTRICTION'); + expect(result.details[0]?.fieldName).equal('string-required'); + expect(result.details[0]?.fieldValue).equal(undefined); + }); + + it('Invalid with multiple missing required fields, reports each error', () => { + const result = validateRecord({}, schemaAllDataTypesRequired); + expect(result.valid).false; + assert(result.valid === false); + + expect(result.details.length).equal(4); + + const fieldError0 = result.details.find((error) => error.fieldName === schemaAllDataTypesRequired.fields[0].name); + const fieldError1 = result.details.find((error) => error.fieldName === schemaAllDataTypesRequired.fields[1].name); + const fieldError2 = result.details.find((error) => error.fieldName === schemaAllDataTypesRequired.fields[2].name); + const fieldError3 = result.details.find((error) => error.fieldName === schemaAllDataTypesRequired.fields[3].name); + expect(fieldError0).not.undefined; + expect(fieldError1).not.undefined; + expect(fieldError2).not.undefined; + expect(fieldError3).not.undefined; + assert( + fieldError0 !== undefined && + fieldError1 !== undefined && + fieldError2 !== undefined && + fieldError3 !== undefined, + ); + + expect(fieldError0.reason).equal('INVALID_BY_RESTRICTION'); + expect(fieldError0.fieldValue).equal(undefined); + expect(fieldError1.reason).equal('INVALID_BY_RESTRICTION'); + expect(fieldError1.fieldValue).equal(undefined); + expect(fieldError2.reason).equal('INVALID_BY_RESTRICTION'); + expect(fieldError2.fieldValue).equal(undefined); + expect(fieldError3.reason).equal('INVALID_BY_RESTRICTION'); + expect(fieldError3.fieldValue).equal(undefined); + + // each of these errors will have fieldValidationErrors inside them which are validated in validateField.spec.ts + }); + + it('Invalid with multiple failed restrictions', () => { + const result = validateRecord( + { + 'complicated-multi-restriction-rules': 'this value is wrong', // triggers both regex and codeList error + 'number-code-list': [1.61803, 2.41421, 0], // 0 is codeList error in position 2 of invalidItems + // missing 'integer-required' and 'any-boolean' fields. ''integer-required' should add an INVALID_FIELD_VALUE error + }, + schemaAllDataTypesMixedRestrictions, + ); + console.log(JSON.stringify(result, null, 2)); + expect(result.valid).false; + assert(result.valid === false); + + expect(result.details.length).equal(3); + + const fieldErrorStringMany = result.details.find((error) => error.fieldName === fieldStringManyRestrictions.name); + expect(fieldErrorStringMany).not.undefined; + const fieldErrorNumberCodeList = result.details.find( + (error) => error.fieldName === fieldNumberArrayCodeList.name, + ); + expect(fieldErrorNumberCodeList).not.undefined; + const fieldErrorIntegerRequired = result.details.find((error) => error.fieldName === fieldIntegerRequired.name); + expect(fieldErrorIntegerRequired).not.undefined; + assert( + fieldErrorStringMany !== undefined && + fieldErrorNumberCodeList !== undefined && + fieldErrorIntegerRequired !== undefined, + ); + + expect(fieldErrorStringMany.reason).equal('INVALID_BY_RESTRICTION'); + expect(fieldErrorStringMany.fieldValue).equal('this value is wrong'); + expect(fieldErrorNumberCodeList.reason).equal('INVALID_BY_RESTRICTION'); + expect(fieldErrorNumberCodeList.fieldValue).deep.equal([1.61803, 2.41421, 0]); + expect(fieldErrorIntegerRequired.reason).equal('INVALID_BY_RESTRICTION'); + expect(fieldErrorIntegerRequired.fieldValue).equal(undefined); + }); + }); + describe('Mixed Errors', () => { + it('Invalid with mix of unrecognized field and failed restrictions', () => { + const result = validateRecord( + { + 'complicated-multi-restriction-rules': '2001-01-01', // correct + 'number-code-list': 'should be a number', // wrong type, should add a INVALID_FIELD_VALUE error + 'integer-required': 1, + // missing 'any-boolean' field, this is optional so no problem. + 'unknown-field': 123, // unknown field this, should add a UNRECOGNIZED_FIELD error + }, + schemaAllDataTypesMixedRestrictions, + ); + expect(result.valid).false; + assert(result.valid === false); + + expect(result.details.length).equal(2); + + const fieldValueError = result.details.find((error) => error.fieldName === 'number-code-list'); + const unknownFieldError = result.details.find((error) => error.fieldName === 'unknown-field'); + expect(fieldValueError).not.undefined; + expect(unknownFieldError).not.undefined; + assert(fieldValueError !== undefined && unknownFieldError !== undefined); + + expect(fieldValueError.reason).equal('INVALID_VALUE_TYPE'); + expect(fieldValueError.fieldValue).equal('should be a number'); + + expect(unknownFieldError.reason).equal('UNRECOGNIZED_FIELD'); + expect(unknownFieldError.fieldValue).equal(123); + }); + }); +}); diff --git a/packages/validation/test/validateSchema/restrictions/generateDataSetHashMap.spec.ts b/packages/validation/test/validateSchema/restrictions/generateDataSetHashMap.spec.ts new file mode 100644 index 0000000..9efb3ef --- /dev/null +++ b/packages/validation/test/validateSchema/restrictions/generateDataSetHashMap.spec.ts @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { expect } from 'chai'; +import { generateDataSetHashMap } from '../../../src/validateSchema/restrictions/generateDataSetHashMap'; +import { hashDataRecord } from '../../../src/utils/hashDataRecord'; +import { getUniqueKeyValues } from '../../../src/validateSchema/restrictions/uniqueKey/getUniqueKeyValues'; + +describe('Schema - generateDataSetHashMap', () => { + it('It contains an entry for each record', () => { + const records = [ + { a: 1, b: 1, c: 1 }, + { a: 1, b: 1, c: 1 }, + { a: 2, b: 2, c: 2 }, + { other: 'stuff' }, + { more: 'other stuff' }, + { a: 5, b: '5' }, + { a: 'six', b: 'six', other: true }, + ]; + const result = generateDataSetHashMap(records, ['a', 'b']); + + let totalEntries = 0; + for (const value of result.values()) { + totalEntries += value.length; + } + expect(totalEntries).equal(records.length); + }); + it('It contains a different hash for every record when all different', () => { + const records = [ + { a: 1, b: 1, c: 1 }, + { a: 2, b: 2, c: 2 }, + { a: 3, b: 3, c: 3 }, + { a: 4, b: 4, c: 4 }, + { a: 5, b: 5, c: 5 }, + { a: 6, b: 6, c: 6 }, + ]; + const result = generateDataSetHashMap(records, ['a', 'b']); + + let keys = 0; + for (const key of result.keys()) { + keys++; + } + expect(keys).equal(records.length); + }); + it('Duplicate records are both listed under the same hash', () => { + const duplicateA = { a: 1, b: 1, c: 1 }; + const duplicateB = { a: 2, b: 2, c: 2 }; + const records = [ + duplicateA, + duplicateA, + duplicateA, + duplicateB, + duplicateB, + { a: 3, b: 3, c: 3 }, + { a: 4, b: 4, c: 4 }, + { a: 5, b: 5, c: 5 }, + { a: 6, b: 6, c: 6 }, + ]; + const uniqueKeyRule = ['a', 'b']; + const result = generateDataSetHashMap(records, uniqueKeyRule); + + const hashA = hashDataRecord(getUniqueKeyValues(duplicateA, uniqueKeyRule)); + const recordA = result.get(hashA); + expect(recordA?.length).equal(3); + expect(recordA).contain(0); + expect(recordA).contain(1); + expect(recordA).contain(2); + + const hashB = hashDataRecord(getUniqueKeyValues(duplicateB, uniqueKeyRule)); + const recordB = result.get(hashB); + expect(recordB?.length).equal(2); + expect(recordB).contain(3); + expect(recordB).contain(4); + }); + it('Works with a single key', () => { + // single key in uniqueKeyRule + const uniqueKeyRule = ['a']; + + const duplicateA = { a: 1, b: 1, c: 1 }; + const duplicateB = { a: 2, b: 2, c: 2 }; + const records = [ + duplicateA, + duplicateA, + duplicateA, + duplicateB, + duplicateB, + { a: 3, b: 3, c: 3 }, + { a: 4, b: 4, c: 4 }, + { a: 5, b: 5, c: 5 }, + { a: 6, b: 6, c: 6 }, + ]; + const result = generateDataSetHashMap(records, uniqueKeyRule); + + const hashA = hashDataRecord(getUniqueKeyValues(duplicateA, uniqueKeyRule)); + const recordA = result.get(hashA); + expect(recordA?.length).equal(3); + expect(recordA).contain(0); + expect(recordA).contain(1); + expect(recordA).contain(2); + + const hashB = hashDataRecord(getUniqueKeyValues(duplicateB, uniqueKeyRule)); + const recordB = result.get(hashB); + expect(recordB?.length).equal(2); + expect(recordB).contain(3); + expect(recordB).contain(4); + }); +}); diff --git a/packages/validation/test/validateSchema/validateSchema.spec.ts b/packages/validation/test/validateSchema/validateSchema.spec.ts new file mode 100644 index 0000000..70789f1 --- /dev/null +++ b/packages/validation/test/validateSchema/validateSchema.spec.ts @@ -0,0 +1,378 @@ +import { expect } from 'chai'; +import type { DataRecord } from 'dictionary'; +import assert from 'node:assert'; +import { validateSchema } from '../../src'; +import { schemaSingleStringRequired } from '../fixtures/schema/schemaSingleStringRequired'; +import { schemaUniqueKey } from '../fixtures/schema/schemaUniqueKey'; +import { schemaUniqueString } from '../fixtures/schema/schemaUniqueString'; +import { schemaUniqueStringArray } from '../fixtures/schema/schemaUniqueStringArray'; +import { schemaUniqueKeyWithArray } from '../fixtures/schema/schemaUniqueKeyWithArray'; + +describe('Schema - validateSchema', () => { + describe('Restriction - unique', () => { + it('Valid for empty data set', () => { + const records: DataRecord[] = []; + const result = validateSchema(records, schemaUniqueString); + expect(result.valid).true; + }); + it('Valid for single entry', () => { + const records: DataRecord[] = [{ 'unique-string': 'asdf' }]; + const result = validateSchema(records, schemaUniqueString); + expect(result.valid).true; + }); + it('Valid for multiple distinct entires', () => { + const records: DataRecord[] = [ + { 'unique-string': 'qwerty' }, + { 'unique-string': 'uiop' }, + { 'unique-string': 'asdf' }, + { 'unique-string': 'ghjkl' }, + { 'unique-string': 'zxcv' }, + { 'unique-string': 'bnm' }, + ]; + const result = validateSchema(records, schemaUniqueString); + expect(result.valid).true; + }); + it('Valid for multiple undefined entires', () => { + const records: DataRecord[] = [ + { 'unique-string': 'qwerty' }, + { 'unique-string': 'uiop' }, + { 'unique-string': 'asdf' }, + { 'unique-string': undefined }, + { 'unique-string': undefined }, + {}, + {}, + ]; + const result = validateSchema(records, schemaUniqueString); + expect(result.valid).true; + }); + it('Invalid for repeated value', () => { + const records: DataRecord[] = [ + { 'unique-string': 'asdf' }, + { 'unique-string': 'asdf' }, + { 'unique-string': 'asdf' }, + ]; + const result = validateSchema(records, schemaUniqueString); + expect(result.valid).false; + assert(result.valid === false); + + expect(result.details.length, 'There should be an invalid record for each field with a duplicate value').equal( + records.length, + ); + expect( + result.details.every((invalidRecord) => invalidRecord.recordErrors.length === 1), + 'Each invalid record should have a single error', + ).true; + expect( + result.details.every((invalidRecord) => invalidRecord.recordErrors[0]?.reason === 'INVALID_BY_UNIQUE'), + 'Every invalid record should have a unique restriction error', + ).true; + }); + // TODO: Test unique on array field + describe('Array fields', () => { + it('Valid when arrays have different values', () => { + const records: DataRecord[] = [ + { 'unique-string-array': ['qwerty', 'uiop[]'] }, + { 'unique-string-array': ['asdf'] }, + { 'unique-string-array': ['ghjkl'] }, + { 'unique-string-array': ['zxcv', 'bn', 'm,./'] }, + ]; + const result = validateSchema(records, schemaUniqueStringArray); + expect(result.valid).true; + }); + it('Valid for arrays with same values in different order', () => { + const records: DataRecord[] = [ + { 'unique-string-array': ['qwerty'] }, + { 'unique-string-array': ['uiop[]'] }, + { 'unique-string-array': ['qwerty', 'uiop[]'] }, + { 'unique-string-array': ['uiop[]', 'qwerty'] }, + { 'unique-string-array': ['qwerty', 'uiop[]', 'asdf'] }, + { 'unique-string-array': ['ghjkl', 'qwerty', 'uiop[]'] }, + ]; + const result = validateSchema(records, schemaUniqueStringArray); + expect(result.valid).true; + }); + it('Valid with multiple empty arrays', () => { + // empty arrays are used when no value is provided + // and records with no value provided should not fail a unique restriction + const records: DataRecord[] = [ + { 'unique-string-array': ['qwerty'] }, + { 'unique-string-array': [] }, + { 'unique-string-array': [] }, + ]; + const result = validateSchema(records, schemaUniqueStringArray); + expect(result.valid).true; + }); + it('Rejects arrays with same values in same order', () => { + const records: DataRecord[] = [ + { 'unique-string-array': ['qwerty'] }, + { 'unique-string-array': ['uiop[]'] }, + + // duplicate entry x3 + { 'unique-string-array': ['qwerty', 'uiop[]'] }, + { 'unique-string-array': ['qwerty', 'uiop[]'] }, + + { 'unique-string-array': ['uiop[]', 'qwerty'] }, + { 'unique-string-array': ['qwerty', 'uiop[]', 'asdf'] }, + + // duplicate entry + { 'unique-string-array': ['ghjkl', 'qwerty', 'uiop[]'] }, + { 'unique-string-array': ['ghjkl', 'qwerty', 'uiop[]'] }, + + // third copy of earlier entry + { 'unique-string-array': ['qwerty', 'uiop[]'] }, + ]; + const result = validateSchema(records, schemaUniqueStringArray); + expect(result.valid).false; + assert(result.valid === false); + expect(result.details.length).equal(5); + + const failedIndices = [2, 3, 6, 7, 8]; + const allIndicesListed = failedIndices.every( + (index) => + result.details.find((error) => error.recordIndex === index)?.recordErrors[0]?.reason === + 'INVALID_BY_UNIQUE', + ); + expect(allIndicesListed).true; + }); + }); + }); + describe('Restriction - uniqueKey', () => { + // TODO: Test uniqueKey when one or more fields is an array + it('Valid for empty data set', () => { + const records: DataRecord[] = []; + const result = validateSchema(records, schemaUniqueKey); + expect(result.valid).true; + }); + it('Valid for single entry', () => { + const records: DataRecord[] = [ + { 'any-string': 'asdf', 'any-number': 12.34, 'any-integer': 123, 'any-boolean': true }, + ]; + const result = validateSchema(records, schemaUniqueKey); + expect(result.valid).true; + }); + it('Valid for entries with distinct unique key values', () => { + const records: DataRecord[] = [ + { 'any-string': 'asdf', 'any-number': 12.34, 'any-integer': 123, 'any-boolean': true }, + { 'any-string': 'qwerty', 'any-number': 12.34, 'any-integer': 123, 'any-boolean': true }, + { 'any-string': 'asdf', 'any-number': 56.78, 'any-integer': 123, 'any-boolean': true }, + { 'any-string': 'asdf', 'any-number': 12.34, 'any-integer': 456, 'any-boolean': true }, + { 'any-string': 'asdf', 'any-number': 12.34, 'any-integer': 123, 'any-boolean': false }, + ]; + const result = validateSchema(records, schemaUniqueKey); + expect(result.valid).true; + }); + it('Valid when one element of unique key value changes to undefined', () => { + const records: DataRecord[] = [ + { 'any-string': 'asdf', 'any-number': 12.34, 'any-integer': 123, 'any-boolean': true }, + { 'any-string': undefined, 'any-number': 12.34, 'any-integer': 123, 'any-boolean': true }, + { 'any-string': 'asdf', 'any-number': undefined, 'any-integer': 123, 'any-boolean': true }, + { 'any-string': 'asdf', 'any-number': 12.34, 'any-integer': undefined, 'any-boolean': true }, + { 'any-string': 'asdf', 'any-number': 12.34, 'any-integer': 123, 'any-boolean': undefined }, + ]; + const result = validateSchema(records, schemaUniqueKey); + expect(result.valid).true; + }); + it('Invalid for repeated key', () => { + const repeatedRecord = { 'any-string': 'asdf', 'any-number': 12.34, 'any-integer': 123, 'any-boolean': true }; + const records: DataRecord[] = [{ ...repeatedRecord }, { ...repeatedRecord }]; + const result = validateSchema(records, schemaUniqueKey); + expect(result.valid).false; + assert(result.valid === false); + + expect(result.details.length, 'There should be an invalid record for each field with a duplicate value').equal( + records.length, + ); + expect( + result.details.every((invalidRecord) => invalidRecord.recordErrors.length === 1), + 'Each invalid record should have a single error', + ).true; + expect( + result.details.every((invalidRecord) => invalidRecord.recordErrors[0]?.reason === 'INVALID_BY_UNIQUE_KEY'), + 'Every invalid record should have a uniqueKey error', + ).true; + assert( + result.details[0] !== undefined && + result.details[0].recordErrors[0] !== undefined && + result.details[0].recordErrors[0].reason === 'INVALID_BY_UNIQUE_KEY', + ); + assert( + result.details[1] !== undefined && + result.details[1].recordErrors[0] !== undefined && + result.details[1].recordErrors[0].reason === 'INVALID_BY_UNIQUE_KEY', + ); + + expect(result.details[0].recordErrors[0].uniqueKey).deep.equal({ ...repeatedRecord }); + expect(result.details[0].recordErrors[0].matchingRecords).include(0); + expect(result.details[0].recordErrors[0].matchingRecords).include(1); + + expect(result.details[1].recordErrors[0].uniqueKey).deep.equal({ ...repeatedRecord }); + expect(result.details[1].recordErrors[0].matchingRecords).include(0); + expect(result.details[1].recordErrors[0].matchingRecords).include(1); + }); + it('Invalid for repeated key including an undefined', () => { + const records: DataRecord[] = [ + { 'any-string': 'asdf', 'any-number': 12.34, 'any-integer': 123, 'any-boolean': undefined }, + { 'any-string': 'asdf', 'any-number': 12.34, 'any-integer': 123, 'any-boolean': undefined }, + ]; + const result = validateSchema(records, schemaUniqueKey); + expect(result.valid).false; + assert(result.valid === false); + + expect(result.details.length, 'There should be an invalid record for each field with a duplicate value').equal( + records.length, + ); + expect( + result.details.every((invalidRecord) => invalidRecord.recordErrors.length === 1), + 'Each invalid record should have a single error', + ).true; + expect( + result.details.every((invalidRecord) => invalidRecord.recordErrors[0]?.reason === 'INVALID_BY_UNIQUE_KEY'), + 'Every invalid record should have a uniqueKey error', + ).true; + }); + it('Invalid for repeated key with all entries missing/undefined for all unique key fields', () => { + const records: DataRecord[] = [ + { 'any-string': undefined, 'any-number': undefined, 'any-integer': undefined, 'any-boolean': undefined }, + {}, + ]; + const result = validateSchema(records, schemaUniqueKey); + expect(result.valid).false; + assert(result.valid === false); + + expect(result.details.length, 'There should be an invalid record for each field with a duplicate value').equal( + records.length, + ); + expect( + result.details.every((invalidRecord) => invalidRecord.recordErrors.length === 1), + 'Each invalid record should have a single error', + ).true; + expect( + result.details.every((invalidRecord) => invalidRecord.recordErrors[0]?.reason === 'INVALID_BY_UNIQUE_KEY'), + 'Every invalid record should have a uniqueKey error', + ).true; + }); + describe('Array field in uniqueKey', () => { + it('Valid when unique key has an array field', () => { + const records: DataRecord[] = [ + { 'any-string-array': ['asdf'], 'any-number': 12.34, 'any-integer': 123, 'any-boolean': true }, + { 'any-string-array': ['qwerty', 'asdf'], 'any-number': 12.34, 'any-integer': 123, 'any-boolean': true }, + { 'any-string-array': ['asdf', 'qwerty'], 'any-number': 12.34, 'any-integer': 123, 'any-boolean': true }, + { 'any-string-array': ['asdf'], 'any-number': 56.78, 'any-integer': 123, 'any-boolean': true }, + { 'any-string-array': ['asdf'], 'any-number': 12.34, 'any-integer': 456, 'any-boolean': true }, + { 'any-string-array': ['asdf'], 'any-number': 12.34, 'any-integer': 123, 'any-boolean': false }, + ]; + const result = validateSchema(records, schemaUniqueKeyWithArray); + expect(result.valid).true; + }); + it('Invalid when unique key has a repeated array field', () => { + // first two records are duplicates + const records: DataRecord[] = [ + { 'any-string-array': ['asdf'], 'any-number': 12.34, 'any-integer': 123, 'any-boolean': true }, + { 'any-string-array': ['asdf'], 'any-number': 12.34, 'any-integer': 123, 'any-boolean': true }, + { 'any-string-array': ['asdf'], 'any-number': 56.78, 'any-integer': 123, 'any-boolean': true }, + { 'any-string-array': ['asdf'], 'any-number': 12.34, 'any-integer': 456, 'any-boolean': true }, + { 'any-string-array': ['asdf'], 'any-number': 12.34, 'any-integer': 123, 'any-boolean': false }, + ]; + const result = validateSchema(records, schemaUniqueKeyWithArray); + expect(result.valid).false; + assert(result.valid === false); + expect(result.details.length).equal(2); + }); + it('Invalid when repeated value is empty array', () => { + // first two records are duplicates + const records: DataRecord[] = [ + { 'any-string-array': [], 'any-number': 12.34, 'any-integer': 123, 'any-boolean': true }, + { 'any-string-array': [], 'any-number': 12.34, 'any-integer': 123, 'any-boolean': true }, + { 'any-string-array': [], 'any-number': 56.78, 'any-integer': 123, 'any-boolean': true }, + { 'any-string-array': [], 'any-number': 12.34, 'any-integer': 456, 'any-boolean': true }, + { 'any-string-array': [], 'any-number': 12.34, 'any-integer': 123, 'any-boolean': false }, + ]; + const result = validateSchema(records, schemaUniqueKeyWithArray); + expect(result.valid).false; + assert(result.valid === false); + expect(result.details.length).equal(2); + }); + }); + }); + describe('Record validations', () => { + it('Invalid when one record has invalid field values', () => { + const records: DataRecord[] = [{ 'string-required': 'asdf' }, {}]; + const result = validateSchema(records, schemaSingleStringRequired); + expect(result.valid).false; + assert(result.valid === false); + + expect(result.details.length).equal(1); + expect( + result.details[0]?.recordIndex, + 'Invalid record needs to indicate the correct position in the array.', + ).equal(1); + expect( + result.details[0]?.recordErrors[0]?.reason, + 'Invalid record needs to indicate it failed by restriction.', + ).equal('INVALID_BY_RESTRICTION'); + }); + it('Invalid when one record has invalid value type', () => { + const records: DataRecord[] = [ + { 'string-required': 'asdf' }, + { 'string-required': 'qwerty' }, + { 'string-required': 123 }, // invalid value type + ]; + const result = validateSchema(records, schemaSingleStringRequired); + expect(result.valid).false; + assert(result.valid === false); + + expect(result.details.length).equal(1); + expect( + result.details[0]?.recordIndex, + 'Invalid record needs to indicate the correct position in the array.', + ).equal(2); + expect( + result.details[0]?.recordErrors[0]?.reason, + 'Invalid record needs to indicate it failed by invalid value type.', + ).equal('INVALID_VALUE_TYPE'); + }); + it('Invalid and reporting multiple invalid records, all invalid records are reported', () => { + const records: DataRecord[] = [ + { 'string-required': 'asdf' }, + {}, // missing value + { 'string-required': 123 }, // invalid value type + { 'string-required': '123', 'unknown-field': 123 }, // unrecognized field + ]; + const result = validateSchema(records, schemaSingleStringRequired); + expect(result.valid).false; + assert(result.valid === false); + + expect(result.details.length).equal(3); + + const missingValueRecord = result.details.find( + (invalidRecord) => invalidRecord.recordErrors[0]?.reason === 'INVALID_BY_RESTRICTION', + ); + const invalidValueTypeRecord = result.details.find( + (invalidRecord) => invalidRecord.recordErrors[0]?.reason === 'INVALID_VALUE_TYPE', + ); + const unrecognizedFieldRecord = result.details.find( + (invalidRecord) => invalidRecord.recordErrors[0]?.reason === 'UNRECOGNIZED_FIELD', + ); + + expect(missingValueRecord).not.undefined; + assert(missingValueRecord !== undefined); + expect(invalidValueTypeRecord).not.undefined; + assert(invalidValueTypeRecord !== undefined); + expect(unrecognizedFieldRecord).not.undefined; + assert(unrecognizedFieldRecord !== undefined); + + expect(missingValueRecord.recordIndex, 'Invalid record should report correct index in records array.').equal(1); + expect(missingValueRecord.recordErrors.length, 'Invalid record should only have a single error.').equal(1); + + expect(invalidValueTypeRecord.recordIndex, 'Invalid record should report correct index in records array.').equal( + 2, + ); + expect(invalidValueTypeRecord.recordErrors.length, 'Invalid record should only have a single error.').equal(1); + + expect(unrecognizedFieldRecord.recordIndex, 'Invalid record should report correct index in records array.').equal( + 3, + ); + expect(unrecognizedFieldRecord.recordErrors.length, 'Invalid record should only have a single error.').equal(1); + }); + }); +}); diff --git a/packages/validation/tsconfig.build.json b/packages/validation/tsconfig.build.json new file mode 100644 index 0000000..df0d74a --- /dev/null +++ b/packages/validation/tsconfig.build.json @@ -0,0 +1,4 @@ +{ + "extends": "./tsconfig.json", + "exclude": ["test/**/*.ts"] +} diff --git a/packages/validation/tsconfig.json b/packages/validation/tsconfig.json index b474986..70cd80a 100644 --- a/packages/validation/tsconfig.json +++ b/packages/validation/tsconfig.json @@ -8,6 +8,7 @@ "strict": true, "noImplicitAny": true, "noUnusedParameters": true, + "noUncheckedIndexedAccess": true, "esModuleInterop": true, "declaration": true, "sourceMap": true, @@ -16,5 +17,5 @@ "downlevelIteration": true, "skipLibCheck": true }, - "include": ["./src/**/*.ts"] + "include": ["./src/**/*.ts", "./test/**/*.ts"] } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9b21509..0dc57af 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -280,6 +280,15 @@ importers: specifier: ^5.0.0 version: 5.0.7 + scripts/generate: + dependencies: + dictionary: + specifier: workspace:^ + version: link:../../packages/dictionary + zod-to-json-schema: + specifier: ^3.21.3 + version: 3.21.3(zod@3.21.4) + packages: /@ampproject/remapping@2.3.0: @@ -5651,7 +5660,6 @@ packages: zod: ^3.21.4 dependencies: zod: 3.21.4 - dev: true /zod@3.21.4: resolution: {integrity: sha512-m46AKbrzKVzOzs/DZgVnG5H55N1sv1M8qZU3A8RIKbs3mrACDNeIOeilDymVb2HdmP8uwshOCF4uJ8uM9rCqJw==} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index c53e539..82c9034 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,3 +1,4 @@ packages: - 'apps/*' - - 'packages/*' \ No newline at end of file + - 'packages/*' + - 'scripts/*' \ No newline at end of file diff --git a/scripts/package.json b/scripts/generate/package.json similarity index 95% rename from scripts/package.json rename to scripts/generate/package.json index 4da51f7..43147da 100644 --- a/scripts/package.json +++ b/scripts/generate/package.json @@ -1,6 +1,7 @@ { "name": "scripts", "version": "1.0.0", + "private": true, "description": "", "main": "index.js", "scripts": { diff --git a/scripts/src/generateMetaSchema.ts b/scripts/generate/src/generateMetaSchema.ts similarity index 100% rename from scripts/src/generateMetaSchema.ts rename to scripts/generate/src/generateMetaSchema.ts From 9e74501b3756ead91669a1d0d32238d321dce80b Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Tue, 30 Jul 2024 23:40:38 -0400 Subject: [PATCH 22/27] Add copyright header text --- packages/common/src/types/defined.ts | 19 +++++++++++++++++++ packages/dictionary/src/types/index.ts | 19 +++++++++++++++++++ .../dictionary/src/types/restrictionsTypes.ts | 19 +++++++++++++++++++ .../parseValues/matchCodeListFormatting.ts | 19 +++++++++++++++++++ .../collectSchemaReferenceData.ts | 19 +++++++++++++++++++ .../testForeignKeyRestriction.ts | 19 +++++++++++++++++++ 6 files changed, 114 insertions(+) diff --git a/packages/common/src/types/defined.ts b/packages/common/src/types/defined.ts index 09cd56e..b052129 100644 --- a/packages/common/src/types/defined.ts +++ b/packages/common/src/types/defined.ts @@ -1,3 +1,22 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + /** * Remove `undefined` from a type union. * diff --git a/packages/dictionary/src/types/index.ts b/packages/dictionary/src/types/index.ts index 456f247..220663d 100644 --- a/packages/dictionary/src/types/index.ts +++ b/packages/dictionary/src/types/index.ts @@ -1,3 +1,22 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + export * from './commonTypes'; export * from './dataTypes'; export * from './dbTypes'; diff --git a/packages/dictionary/src/types/restrictionsTypes.ts b/packages/dictionary/src/types/restrictionsTypes.ts index e01d0a0..d4c7b4c 100644 --- a/packages/dictionary/src/types/restrictionsTypes.ts +++ b/packages/dictionary/src/types/restrictionsTypes.ts @@ -1,3 +1,22 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + import { z as zod } from 'zod'; import { ReferenceTag } from './referenceTypes'; import { Integer } from './commonTypes'; diff --git a/packages/validation/src/parseValues/matchCodeListFormatting.ts b/packages/validation/src/parseValues/matchCodeListFormatting.ts index 8d0d082..8b6d860 100644 --- a/packages/validation/src/parseValues/matchCodeListFormatting.ts +++ b/packages/validation/src/parseValues/matchCodeListFormatting.ts @@ -1,3 +1,22 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + import type { SchemaField } from 'dictionary'; /** diff --git a/packages/validation/src/validateDictionary/collectSchemaReferenceData.ts b/packages/validation/src/validateDictionary/collectSchemaReferenceData.ts index d786560..0ea6589 100644 --- a/packages/validation/src/validateDictionary/collectSchemaReferenceData.ts +++ b/packages/validation/src/validateDictionary/collectSchemaReferenceData.ts @@ -1,3 +1,22 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + import { isDefined } from 'common'; import type { DataRecord, DataRecordValue, Dictionary, ForeignKeyRestriction, Schema } from 'dictionary'; diff --git a/packages/validation/src/validateDictionary/testForeignKeyRestriction.ts b/packages/validation/src/validateDictionary/testForeignKeyRestriction.ts index f746dd1..2755c13 100644 --- a/packages/validation/src/validateDictionary/testForeignKeyRestriction.ts +++ b/packages/validation/src/validateDictionary/testForeignKeyRestriction.ts @@ -1,3 +1,22 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + import type { DataRecord, ForeignKeyRestriction } from 'dictionary'; import type { SchemaDataReference } from './collectSchemaReferenceData'; import type { DictionaryValidationErrorRecordForeignKey } from './DictionaryValidationError'; From b484be4d74abbfa43c0a6f4b3b7c0cf0d89068a8 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Tue, 30 Jul 2024 23:46:33 -0400 Subject: [PATCH 23/27] TS comments for functions rangeToText and isValidValueType --- packages/validation/src/utils/isValidValueType.ts | 8 ++++++++ packages/validation/src/utils/rangeToText.ts | 10 ++++++++++ 2 files changed, 18 insertions(+) diff --git a/packages/validation/src/utils/isValidValueType.ts b/packages/validation/src/utils/isValidValueType.ts index 5f2639a..4e00706 100644 --- a/packages/validation/src/utils/isValidValueType.ts +++ b/packages/validation/src/utils/isValidValueType.ts @@ -20,6 +20,14 @@ import type { DataRecordValue, SchemaField } from 'dictionary'; import { isBooleanArray, isInteger, isIntegerArray, isNumber, isNumberArray, isStringArray } from './typeUtils'; +/** + * Checks that a value matches the expected type for a given field, based on the value type specified in its field + * definition. + * + * @param value Value to check + * @param fieldDefinition Field definition that specifies the expected value type + * @returns `true` if value matches the expected type; `false` otherwise. + */ export const isValidValueType = (value: DataRecordValue, fieldDefinition: SchemaField): boolean => { switch (fieldDefinition.valueType) { case 'boolean': { diff --git a/packages/validation/src/utils/rangeToText.ts b/packages/validation/src/utils/rangeToText.ts index 57b8d67..afc8bcc 100644 --- a/packages/validation/src/utils/rangeToText.ts +++ b/packages/validation/src/utils/rangeToText.ts @@ -19,6 +19,16 @@ import { RestrictionRange } from 'dictionary'; +/** + * Convert a RestrictionRange object to a simple string representation. + * + * @example + * const range = { exclusiveMin: 0, max: 10 }; + * rangeToText(range); // "> 0 and <= 10" + * + * @param range + * @returns + */ export const rangeToText = (range: RestrictionRange): string => { let minString = ''; let maxString = ''; From 29d87090944889ca7beb6477dbe02283a23e8263 Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Thu, 1 Aug 2024 11:19:34 -0400 Subject: [PATCH 24/27] Consolidate Common and Dictionary packages (#220) * Combine common package with dictionary * Rename dictionary package to `@overture-stack/lectern-dictionary` - Updates all imports of the dictionary package - Update root README to reflect this change - Runs `pnpm update` to make use latest version of all dependencies - dependency udpate was required to get all packages onto the same version of zod and work with the zod-to-json-schema utility in the generate script - Relocate scripts to a single package in the `/script` dir. - Adds README to scripts package to describe available scripts - Designed to have a single scripts package with multiple utilities instead of separate package per script * Update dependency versions in validation package * Update lectern server package name in Dockerfile * Adds processSchema usage example to Client README * Updates Dictionary README to match packages format * Rename processing Success types to not say Error --- README.md | 24 +- apps/server/Dockerfile | 2 +- apps/server/package.json | 41 +- .../src/controllers/dictionaryController.ts | 9 +- .../types => apps/server/src/db}/dbTypes.ts | 2 +- apps/server/src/db/dictionary.ts | 3 +- apps/server/src/external/ego.ts | 2 +- apps/server/src/routers/diffRouter.ts | 3 +- apps/server/src/services/dictionaryService.ts | 11 +- apps/server/src/services/schemaService.ts | 3 +- .../test/fixtures/dictionaries/1_base.ts | 2 +- .../dictionaries/2_base_schema_added.ts | 2 +- .../dictionaries/3_base_schema_updated.ts | 2 +- .../dictionaries/invalidReferences.ts | 2 +- .../fixtures/dictionaries/linebreak/input1.ts | 2 +- .../fixtures/dictionaries/linebreak/input2.ts | 2 +- .../dictionaries/linebreak/nLinebreaks.ts | 2 +- .../linebreak/normalizedLinebreaks.ts | 2 +- .../fixtures/dictionaries/linebreak/output.ts | 2 +- .../dictionaries/linebreak/rnLinebreaks.ts | 2 +- .../test/fixtures/dictionaries/simple.ts | 2 +- .../fixtures/dictionaries/validReferences.ts | 2 +- .../test/fixtures/schemas/primitives.ts | 2 +- .../test/fixtures/schemas/references.ts | 2 +- .../fixtures/schemas/schemaChangesBase.ts | 2 +- .../fixtures/schemas/schemaChangesUpdated.ts | 2 +- .../services/dictionaryService.spec.ts | 3 +- package.json | 30 +- packages/client/README.md | 56 +- packages/client/package.json | 13 +- .../src/changeAnalysis/changeAnalysisTypes.ts | 2 +- .../src/changeAnalysis/changeAnalyzer.ts | 8 +- packages/client/src/processing/index.ts | 2 +- .../src/processing/processingResultTypes.ts | 14 +- packages/client/src/rest/index.ts | 9 +- packages/client/test/changeAnalyzer.spec.ts | 2 +- packages/client/test/fixtures/diffResponse.ts | 2 +- .../test/fixtures/registrationSchema.ts | 2 +- packages/common/.mocharc.json | 5 - packages/common/README.md | 5 - packages/common/package.json | 16 - packages/common/src/index.ts | 3 - packages/common/src/utils/objectUtils.ts | 93 -- packages/dictionary/README.md | 11 +- packages/dictionary/package.json | 13 +- packages/{common => dictionary}/src/errors.ts | 0 packages/dictionary/src/index.ts | 23 +- .../dictionarySchemas.ts} | 32 +- .../src/metaSchema}/index.ts | 9 +- .../referenceSchemas.ts} | 0 .../restrictionsSchemas.ts} | 7 +- packages/dictionary/src/references.ts | 12 +- packages/dictionary/src/types/commonTypes.ts | 18 - .../src/types/{dataTypes.ts => dataRecord.ts} | 2 +- .../src/types/defined.ts | 0 packages/dictionary/src/types/diffTypes.ts | 2 +- .../src/types/generics.ts | 0 packages/dictionary/src/types/index.ts | 11 +- .../src/types/result.ts | 0 .../src/types/singular.ts | 0 .../{common => dictionary}/src/utils/index.ts | 5 +- packages/dictionary/src/utils/schemaUtils.ts | 2 +- .../src/utils/stringUtils.ts | 0 .../src/utils/typeUtils.ts | 0 .../src/utils/{version.ts => versionUtils.ts} | 2 +- .../dictionary/test/dictionaryTypes.spec.ts | 22 +- packages/validation/package.json | 11 +- .../src/parseValues/ParseValuesResult.ts | 3 +- .../parseValues/matchCodeListFormatting.ts | 2 +- .../validation/src/parseValues/parseValues.ts | 12 +- packages/validation/src/utils/datasetUtils.ts | 2 +- .../validation/src/utils/hashDataRecord.ts | 2 +- .../validation/src/utils/isValidValueType.ts | 2 +- .../validation/src/utils/isWithinRange.ts | 2 +- packages/validation/src/utils/rangeToText.ts | 2 +- .../collectSchemaReferenceData.ts | 12 +- .../testForeignKeyRestriction.ts | 7 +- .../testUnrecognizedSchema.ts | 2 +- .../validateDictionary/validateDictionary.ts | 9 +- .../src/validateField/FieldRestrictionRule.ts | 2 +- .../src/validateField/FieldRestrictionTest.ts | 2 +- .../src/validateField/FieldValidationError.ts | 2 +- .../validateField/resolveFieldRestrictions.ts | 2 +- .../createFieldRestrictionTestForArrays.ts | 4 +- .../restrictions/testCodeList.ts | 2 +- .../validateField/restrictions/testRange.ts | 2 +- .../validateField/restrictions/testRegex.ts | 2 +- .../restrictions/testRequired.ts | 2 +- .../src/validateField/validateField.ts | 2 +- .../validateRecord/RecordValidationError.ts | 2 +- .../src/validateRecord/validateRecord.ts | 2 +- .../validateSchema/SchemaValidationError.ts | 2 +- .../restrictions/generateDataSetHashMap.ts | 2 +- .../uniqueField/testUniqueFieldRestriction.ts | 2 +- .../uniqueKey/getUniqueKeyValues.ts | 2 +- .../restrictions/uniqueKey/testUniqueKey.ts | 2 +- .../src/validateSchema/validateSchema.ts | 10 +- .../dictionaries/dictionaryFourSchemas.ts | 2 +- ...dictionaryMultipleSchemasNoRestrictions.ts | 2 +- .../dictionarySingleSchemaNoRestrictions.ts | 2 +- ...tionarySingleSchemaRequiredRestrictions.ts | 2 +- ...tionarySingleSchemaUniqueKeyRestriction.ts | 2 +- .../dictionaryForeignKeyMultiple.ts | 2 +- .../foreignKey/dictionaryForeignKeySimple.ts | 2 +- .../fieldStringManyRestrictions.ts | 2 +- .../fieldBooleanNoRestriction.ts | 2 +- .../fieldIntegerNoRestriction.ts | 2 +- .../fieldNumberNoRestriction.ts | 2 +- .../fieldStringArrayNoRestriction.ts | 2 +- .../fieldStringNoRestriction.ts | 2 +- .../schemaRestrictions/fieldStringUnique.ts | 2 +- .../fieldStringUniqueArray.ts | 2 +- .../boolean/fieldBooleanArrayRequired.ts | 2 +- .../boolean/fieldBooleanRequired.ts | 2 +- .../integer/fieldIntegerArrayRequired.ts | 2 +- .../integer/fieldIntegerRequired.ts | 2 +- .../number/fieldNumberArrayCodeList.ts | 2 +- .../number/fieldNumberRange.ts | 2 +- .../number/fieldNumberRequired.ts | 2 +- .../string/fieldStringArrayCodeList.ts | 2 +- .../string/fieldStringArrayRequired.ts | 2 +- .../string/fieldStringCodeList.ts | 2 +- .../string/fieldStringRegex.ts | 2 +- .../string/fieldStringRequired.ts | 2 +- .../restrictions/codeListsFixtures.ts | 6 +- .../fixtures/restrictions/rangeFixtures.ts | 2 +- .../fixtures/restrictions/regexFixtures.ts | 2 +- .../schema/schemaAllDataMixedRestrictions.ts | 2 +- .../fixtures/schema/schemaAllDataTypes.ts | 2 +- .../schema/schemaAllDataTypesRequired.ts | 2 +- .../fixtures/schema/schemaSingleString.ts | 2 +- .../schema/schemaSingleStringRequired.ts | 2 +- .../test/fixtures/schema/schemaUniqueKey.ts | 2 +- .../schema/schemaUniqueKeyWithArray.ts | 2 +- .../fixtures/schema/schemaUniqueString.ts | 2 +- .../schema/schemaUniqueStringArray.ts | 2 +- .../validation/test/utils/rangeTest.spec.ts | 2 +- .../test/validateField/validateField.spec.ts | 2 +- .../validateSchema/validateSchema.spec.ts | 2 +- pnpm-lock.yaml | 1270 +++++++---------- pnpm-workspace.yaml | 2 +- scripts/README.md | 12 + scripts/generate/package.json | 18 - scripts/package.json | 16 + .../{generate => }/src/generateMetaSchema.ts | 7 +- {packages/common => scripts}/tsconfig.json | 9 +- 146 files changed, 908 insertions(+), 1196 deletions(-) rename {packages/dictionary/src/types => apps/server/src/db}/dbTypes.ts (96%) delete mode 100644 packages/common/.mocharc.json delete mode 100644 packages/common/README.md delete mode 100644 packages/common/package.json delete mode 100644 packages/common/src/index.ts delete mode 100644 packages/common/src/utils/objectUtils.ts rename packages/{common => dictionary}/src/errors.ts (100%) rename packages/dictionary/src/{types/dictionaryTypes.ts => metaSchema/dictionarySchemas.ts} (94%) rename packages/{common/src/types => dictionary/src/metaSchema}/index.ts (86%) rename packages/dictionary/src/{types/referenceTypes.ts => metaSchema/referenceSchemas.ts} (100%) rename packages/dictionary/src/{types/restrictionsTypes.ts => metaSchema/restrictionsSchemas.ts} (96%) delete mode 100644 packages/dictionary/src/types/commonTypes.ts rename packages/dictionary/src/types/{dataTypes.ts => dataRecord.ts} (98%) rename packages/{common => dictionary}/src/types/defined.ts (100%) rename packages/{common => dictionary}/src/types/generics.ts (100%) rename packages/{common => dictionary}/src/types/result.ts (100%) rename packages/{common => dictionary}/src/types/singular.ts (100%) rename packages/{common => dictionary}/src/utils/index.ts (88%) rename packages/{common => dictionary}/src/utils/stringUtils.ts (100%) rename packages/{common => dictionary}/src/utils/typeUtils.ts (100%) rename packages/dictionary/src/utils/{version.ts => versionUtils.ts} (97%) create mode 100644 scripts/README.md delete mode 100644 scripts/generate/package.json create mode 100644 scripts/package.json rename scripts/{generate => }/src/generateMetaSchema.ts (90%) rename {packages/common => scripts}/tsconfig.json (66%) diff --git a/README.md b/README.md index d51a298..eb4d01c 100644 --- a/README.md +++ b/README.md @@ -33,15 +33,15 @@ The repository is organized with the following directory structure: The modules in the monorepo are organized into two categories: * __apps/__ - Standalone processes meant to be run. These are published to [ghcr.io](https://ghcr.io) as container images. - * __packages/__ - Reusable packages shared between applications and other packages. Packages are published to [NPM](https://npmjs.com) . + * __packages/__ - Reusable packages shared between applications and other packages. Packages are published to [NPM](https://npmjs.com). + * __scripts__ - Utility scripts for use within this repo. -| Component | Type | Package Name | Path | Published Location | Description | -| ------------------------------------------- | ----------- | ---------------------------------- | -------------------- | ------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [Lectern Server](apps/server/README.md) | Application | server | apps/server/ | [![Lectern GHCR Packages](https://img.shields.io/badge/GHCR-lectern-brightgreen?style=for-the-badge&logo=github)](https://github.com/overture-stack/lectern/pkgs/container/lectern) | Lectern Server web application. | -| [Lectern Client](packages/client/README.md) | Package | @overture-stack/lectern-client | packages/client | [![Lectern Client NPM Package](https://img.shields.io/npm/v/@overture-stack/lectern-client?color=%23cb3837&style=for-the-badge&logo=npm)](https://www.npmjs.com/package/@overture-stack/lectern-client) | TypeScript Client to interact with Lectern Server and perform data validation. | -| [common](packages/common/README.md) | Package | common | packages/common/ | N/A | Non-specific but commonly reusable utilities. Includes shared Error classes. | -| [dictionary](packages/dictionary/README.md) | Package | dictionary | packages/dictionary/ | N/A | Dictionary meta-schema definition, includes TS types, and Zod schemas. This also exports all utilities for getting the diff of two dictionaries, and for validating data records with a Dictionary. | -| [validation](packages/validation/README.md) | Package | @overture-stack/lectern-validation | packages/validation/ | [![Lectern Validation NPM Package](https://img.shields.io/npm/v/@overture-stack/lectern-client?color=%23cb3837&style=for-the-badge&logo=npm)](https://www.npmjs.com/package/@overture-stack/lectern-client) | Validate data using Lectern Dictionaries. | +| Component | Package Name | Path | Published Location | Description | +| --------------------------------------------------- | ---------------------------------- | ---------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [Lectern Server](apps/server/README.md) | @overture-stack/lectern-server | apps/server/ | [![Lectern GHCR Packages](https://img.shields.io/badge/GHCR-lectern-brightgreen?style=for-the-badge&logo=github)](https://github.com/overture-stack/lectern/pkgs/container/lectern) | Lectern Server web application. | +| [Lectern Client](packages/client/README.md) | @overture-stack/lectern-client | packages/client | [![Lectern Client NPM Package](https://img.shields.io/npm/v/@overture-stack/lectern-client?color=%23cb3837&style=for-the-badge&logo=npm)](https://www.npmjs.com/package/@overture-stack/lectern-client) | TypeScript Client to interact with Lectern Server and Lectern data dictionaries. This library provides a REST client to assist in fetching data from the Lectern server. It also exposes the functionality from the Lectern Validation library to use a Lectern data dictionary to validate data. | +| [Lectern Dictionary](packages/dictionary/README.md) | | @overture-stack/lectern-dictionary | packages/dictionary/ | [![Lectern Client NPM Package](https://img.shields.io/npm/v/@overture-stack/lectern-dictionary?color=%23cb3837&style=for-the-badge&logo=npm)](https://www.npmjs.com/package/@overture-stack/lectern-dictionary) | Dictionary meta-schema definition, includes TS types, and Zod schemas. This also exports all utilities for getting the diff of two dictionaries. | +| [Lectern Validation](packages/validation/README.md) | @overture-stack/lectern-validation | packages/validation/ | [![Lectern Validation NPM Package](https://img.shields.io/npm/v/@overture-stack/lectern-client?color=%23cb3837&style=for-the-badge&logo=npm)](https://www.npmjs.com/package/@overture-stack/lectern-client) | Validate data using Lectern Dictionaries. | ## Developer Instructions @@ -85,9 +85,9 @@ This will build all modules. `pnpm test:all` -This will test everything, building all dependencies needed to fully test. +This will test all modules in the repo. -## Additonal Content +## Additional Content In addition to the code for Lectern, this repository contains some useful reference material. @@ -95,9 +95,9 @@ In addition to the code for Lectern, this repository contains some useful refere Lectern provides a meta-schema definition that describes the structure of Lectern Dictionaries. The generated JSON Schema formatted copy of this schema can be found at [`./generated/DictionaryMetaSchema.json`](./generated/DictionaryMetaSchema.json). -This can be used as a programing language agnostic schema for external applications that want to validate, generate, or interact with Lectern Dictionaries. +This can be used as a programing language agnostic schema for external applications to validate Lectern Dictionaries. -> **Note:** +> [!NOTE] > > Don't manually update any files in the `./generated` path. This content is programatically generated from the source code. diff --git a/apps/server/Dockerfile b/apps/server/Dockerfile index 0e1d6d8..90913db 100644 --- a/apps/server/Dockerfile +++ b/apps/server/Dockerfile @@ -25,7 +25,7 @@ RUN chown -R ${APP_USER}:${APP_USER} $(npm config get prefix) USER ${APP_USER}:${APP_USER} RUN pnpm install --ignore-scripts -RUN pnpm nx build server +RUN pnpm nx build @overture-stack/lectern-server EXPOSE 3000 diff --git a/apps/server/package.json b/apps/server/package.json index 5d87de4..b7c2947 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -1,5 +1,6 @@ { - "name": "server", + "name": "@overture-stack/lectern-server", + "private": true, "version": "2.0.0-next.0", "description": "Overture Data Dictionary Management", "scripts": { @@ -19,40 +20,40 @@ }, "homepage": "https://github.com/overture-stack/lectern#readme", "devDependencies": { - "@types/body-parser": "^1.19.2", + "@types/body-parser": "^1.19.5", "@types/errorhandler": "0.0.32", - "@types/express": "^4.17.17", + "@types/express": "^4.17.21", "@types/jsonwebtoken": "^8.5.9", - "@types/lodash": "^4.14.195", - "@types/memoizee": "^0.4.8", - "@types/ms": "^0.7.31", - "@types/superagent": "^4.1.18", + "@types/lodash": "^4.17.7", + "@types/memoizee": "^0.4.11", + "@types/ms": "^0.7.34", + "@types/superagent": "^4.1.24", "@types/swagger-ui-express": "^3.0.1", "concurrently": "^5.3.0", "husky": "^3.1.0", "nodemon": "^2.0.22", - "prettier": "^3", + "prettier": "^3.3.3", "testcontainers": "^1.3.1", - "zod-to-json-schema": "^3.21.3" + "typescript": "^5.5.4", + "zod-to-json-schema": "^3.23.2" }, "dependencies": { - "ajv": "^8.12.0", - "axios": "^1.4.0", + "@overture-stack/lectern-dictionary": "workspace:^", + "ajv": "^8.17.1", + "axios": "^1.7.2", "body-parser": "^1.20.2", - "common": "workspace:^", - "dictionary": "workspace:^", - "dotenv": "^16.3.1", + "dotenv": "^16.4.5", "errorhandler": "^1.5.1", - "express": "^4.18.2", - "immer": "^10.0.2", + "express": "^4.19.2", + "immer": "^10.1.1", "jsonwebtoken": "^8.5.1", "lodash": "^4.17.21", - "memoizee": "^0.4.15", - "mongoose": "^7.3.2", + "memoizee": "^0.4.17", + "mongoose": "^7.8.0", "ms": "^2.1.3", "node-vault": "^0.9.22", "swagger-ui-express": "^4.6.3", - "winston": "^3.9.0", - "zod": "^3.21.4" + "winston": "^3.13.1", + "zod": "^3.23.8" } } diff --git a/apps/server/src/controllers/dictionaryController.ts b/apps/server/src/controllers/dictionaryController.ts index 53c25ff..fb5aadd 100644 --- a/apps/server/src/controllers/dictionaryController.ts +++ b/apps/server/src/controllers/dictionaryController.ts @@ -17,8 +17,13 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { BadRequestError, NotFoundError } from 'common'; -import { Dictionary, Schema, replaceReferences } from 'dictionary'; +import { + BadRequestError, + Dictionary, + NotFoundError, + Schema, + replaceReferences, +} from '@overture-stack/lectern-dictionary'; import { Request, Response } from 'express'; import * as dictionaryService from '../services/dictionaryService'; diff --git a/packages/dictionary/src/types/dbTypes.ts b/apps/server/src/db/dbTypes.ts similarity index 96% rename from packages/dictionary/src/types/dbTypes.ts rename to apps/server/src/db/dbTypes.ts index 71d38ad..209c031 100644 --- a/packages/dictionary/src/types/dbTypes.ts +++ b/apps/server/src/db/dbTypes.ts @@ -17,8 +17,8 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +import { DictionaryBase } from '@overture-stack/lectern-dictionary'; import { z as zod } from 'zod'; -import { DictionaryBase } from './dictionaryTypes'; /** * A Dictionary stored in the DB is represented as a document and gets an `_id` property diff --git a/apps/server/src/db/dictionary.ts b/apps/server/src/db/dictionary.ts index 09a30b5..aafa822 100644 --- a/apps/server/src/db/dictionary.ts +++ b/apps/server/src/db/dictionary.ts @@ -17,9 +17,10 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { Dictionary, DictionaryDocument, DictionaryDocumentSummary } from 'dictionary'; +import { Dictionary } from '@overture-stack/lectern-dictionary'; import { omit } from 'lodash'; import mongoose from 'mongoose'; +import type { DictionaryDocument, DictionaryDocumentSummary } from './dbTypes'; export const DictionaryModel = mongoose.model( 'Dictionary', diff --git a/apps/server/src/external/ego.ts b/apps/server/src/external/ego.ts index 5d7bab7..693e525 100644 --- a/apps/server/src/external/ego.ts +++ b/apps/server/src/external/ego.ts @@ -18,7 +18,7 @@ */ import axios from 'axios'; -import { ForbiddenError, UnauthorizedError } from 'common'; +import { ForbiddenError, UnauthorizedError } from '@overture-stack/lectern-dictionary'; import { NextFunction, Request, RequestHandler, Response } from 'express'; import jwt from 'jsonwebtoken'; import memoize from 'memoizee'; diff --git a/apps/server/src/routers/diffRouter.ts b/apps/server/src/routers/diffRouter.ts index cd31a27..083620b 100644 --- a/apps/server/src/routers/diffRouter.ts +++ b/apps/server/src/routers/diffRouter.ts @@ -16,8 +16,7 @@ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { BadRequestError } from 'common'; -import { DictionaryDiffArray, DiffUtils, replaceReferences } from 'dictionary'; +import { BadRequestError, DictionaryDiffArray, DiffUtils, replaceReferences } from '@overture-stack/lectern-dictionary'; import { Router } from 'express'; import * as dictionaryService from '../services/dictionaryService'; import { wrapAsync } from './wrappers'; diff --git a/apps/server/src/services/dictionaryService.ts b/apps/server/src/services/dictionaryService.ts index ce9bc07..8b3d914 100644 --- a/apps/server/src/services/dictionaryService.ts +++ b/apps/server/src/services/dictionaryService.ts @@ -17,13 +17,20 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { BadRequestError, ConflictError, NotFoundError } from 'common'; -import { Dictionary, DictionaryDocument, DictionaryDocumentSummary, Schema, VersionUtils } from 'dictionary'; +import { + BadRequestError, + ConflictError, + Dictionary, + NotFoundError, + Schema, + VersionUtils, +} from '@overture-stack/lectern-dictionary'; import * as immer from 'immer'; import { omit } from 'lodash'; import logger from '../config/logger'; import * as DictionaryRepo from '../db/dictionary'; import { normalizeSchema, validate } from '../services/schemaService'; +import type { DictionaryDocument, DictionaryDocumentSummary } from '../db/dbTypes'; /** * Get latest version for all dictionaries with the provided name diff --git a/apps/server/src/services/schemaService.ts b/apps/server/src/services/schemaService.ts index fdb4cd7..2bbae87 100644 --- a/apps/server/src/services/schemaService.ts +++ b/apps/server/src/services/schemaService.ts @@ -17,10 +17,9 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { References, Schema } from 'dictionary'; +import { References, replaceSchemaReferences, Schema } from '@overture-stack/lectern-dictionary'; import * as immer from 'immer'; import { ZodError } from 'zod'; -import { replaceSchemaReferences } from 'dictionary'; export function validate(schema: Schema, references: References): { valid: boolean; errors?: ZodError } { const schemaWithReplacements = replaceSchemaReferences(schema, references); diff --git a/apps/server/test/fixtures/dictionaries/1_base.ts b/apps/server/test/fixtures/dictionaries/1_base.ts index 82d51d5..4d94be1 100644 --- a/apps/server/test/fixtures/dictionaries/1_base.ts +++ b/apps/server/test/fixtures/dictionaries/1_base.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { Dictionary } from 'dictionary'; +import { Dictionary } from '@overture-stack/lectern-dictionary'; import primitivesSchema from '../schemas/primitives'; const dictionary: Dictionary = { diff --git a/apps/server/test/fixtures/dictionaries/2_base_schema_added.ts b/apps/server/test/fixtures/dictionaries/2_base_schema_added.ts index 39213b1..88962b2 100644 --- a/apps/server/test/fixtures/dictionaries/2_base_schema_added.ts +++ b/apps/server/test/fixtures/dictionaries/2_base_schema_added.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { Dictionary } from 'dictionary'; +import { Dictionary } from '@overture-stack/lectern-dictionary'; import primitivesSchema from '../schemas/primitives'; import additionalSchema from '../schemas/schemaChangesBase'; const dictionary: Dictionary = { diff --git a/apps/server/test/fixtures/dictionaries/3_base_schema_updated.ts b/apps/server/test/fixtures/dictionaries/3_base_schema_updated.ts index 17aeefc..ac34d66 100644 --- a/apps/server/test/fixtures/dictionaries/3_base_schema_updated.ts +++ b/apps/server/test/fixtures/dictionaries/3_base_schema_updated.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { Dictionary } from 'dictionary'; +import { Dictionary } from '@overture-stack/lectern-dictionary'; import primitivesSchema from '../schemas/primitives'; import updatedSchema from '../schemas/schemaChangesUpdated'; const dictionary: Dictionary = { diff --git a/apps/server/test/fixtures/dictionaries/invalidReferences.ts b/apps/server/test/fixtures/dictionaries/invalidReferences.ts index f58987b..900fb68 100644 --- a/apps/server/test/fixtures/dictionaries/invalidReferences.ts +++ b/apps/server/test/fixtures/dictionaries/invalidReferences.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { Dictionary } from 'dictionary'; +import { Dictionary } from '@overture-stack/lectern-dictionary'; import referencesSchema, { references } from '../schemas/references'; const dictionary: Dictionary = { name: 'Invalid References Dictionary', diff --git a/apps/server/test/fixtures/dictionaries/linebreak/input1.ts b/apps/server/test/fixtures/dictionaries/linebreak/input1.ts index 28d48a7..d7be203 100644 --- a/apps/server/test/fixtures/dictionaries/linebreak/input1.ts +++ b/apps/server/test/fixtures/dictionaries/linebreak/input1.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { Dictionary } from 'dictionary'; +import { Dictionary } from '@overture-stack/lectern-dictionary'; const output: Dictionary = { name: 'Test Dictionary', diff --git a/apps/server/test/fixtures/dictionaries/linebreak/input2.ts b/apps/server/test/fixtures/dictionaries/linebreak/input2.ts index a36b5ff..449786a 100644 --- a/apps/server/test/fixtures/dictionaries/linebreak/input2.ts +++ b/apps/server/test/fixtures/dictionaries/linebreak/input2.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { Dictionary } from 'dictionary'; +import { Dictionary } from '@overture-stack/lectern-dictionary'; const output: Dictionary = { name: 'Test Dictionary', diff --git a/apps/server/test/fixtures/dictionaries/linebreak/nLinebreaks.ts b/apps/server/test/fixtures/dictionaries/linebreak/nLinebreaks.ts index 9660470..3880947 100644 --- a/apps/server/test/fixtures/dictionaries/linebreak/nLinebreaks.ts +++ b/apps/server/test/fixtures/dictionaries/linebreak/nLinebreaks.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { Dictionary } from 'dictionary'; +import { Dictionary } from '@overture-stack/lectern-dictionary'; const output: Dictionary = { name: 'Line Breaks', diff --git a/apps/server/test/fixtures/dictionaries/linebreak/normalizedLinebreaks.ts b/apps/server/test/fixtures/dictionaries/linebreak/normalizedLinebreaks.ts index 9660470..3880947 100644 --- a/apps/server/test/fixtures/dictionaries/linebreak/normalizedLinebreaks.ts +++ b/apps/server/test/fixtures/dictionaries/linebreak/normalizedLinebreaks.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { Dictionary } from 'dictionary'; +import { Dictionary } from '@overture-stack/lectern-dictionary'; const output: Dictionary = { name: 'Line Breaks', diff --git a/apps/server/test/fixtures/dictionaries/linebreak/output.ts b/apps/server/test/fixtures/dictionaries/linebreak/output.ts index 28d48a7..d7be203 100644 --- a/apps/server/test/fixtures/dictionaries/linebreak/output.ts +++ b/apps/server/test/fixtures/dictionaries/linebreak/output.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { Dictionary } from 'dictionary'; +import { Dictionary } from '@overture-stack/lectern-dictionary'; const output: Dictionary = { name: 'Test Dictionary', diff --git a/apps/server/test/fixtures/dictionaries/linebreak/rnLinebreaks.ts b/apps/server/test/fixtures/dictionaries/linebreak/rnLinebreaks.ts index 13dc165..1c0c3a5 100644 --- a/apps/server/test/fixtures/dictionaries/linebreak/rnLinebreaks.ts +++ b/apps/server/test/fixtures/dictionaries/linebreak/rnLinebreaks.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { Dictionary } from 'dictionary'; +import { Dictionary } from '@overture-stack/lectern-dictionary'; const output: Dictionary = { name: 'Line Breaks', diff --git a/apps/server/test/fixtures/dictionaries/simple.ts b/apps/server/test/fixtures/dictionaries/simple.ts index 1a237dc..6741fd4 100644 --- a/apps/server/test/fixtures/dictionaries/simple.ts +++ b/apps/server/test/fixtures/dictionaries/simple.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { Dictionary } from 'dictionary'; +import { Dictionary } from '@overture-stack/lectern-dictionary'; import primitivesSchema from '../schemas/primitives'; export default { name: 'Simple Dictionary', diff --git a/apps/server/test/fixtures/dictionaries/validReferences.ts b/apps/server/test/fixtures/dictionaries/validReferences.ts index 9bdc4b7..b567101 100644 --- a/apps/server/test/fixtures/dictionaries/validReferences.ts +++ b/apps/server/test/fixtures/dictionaries/validReferences.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { Dictionary } from 'dictionary'; +import { Dictionary } from '@overture-stack/lectern-dictionary'; import referencesSchema, { references } from '../schemas/references'; const dictionary: Dictionary = { name: 'Valid References Dictionary', diff --git a/apps/server/test/fixtures/schemas/primitives.ts b/apps/server/test/fixtures/schemas/primitives.ts index 80af9ba..6595083 100644 --- a/apps/server/test/fixtures/schemas/primitives.ts +++ b/apps/server/test/fixtures/schemas/primitives.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { Schema } from 'dictionary'; +import { Schema } from '@overture-stack/lectern-dictionary'; // Just a simple schema with no frills. diff --git a/apps/server/test/fixtures/schemas/references.ts b/apps/server/test/fixtures/schemas/references.ts index 011f21b..abb4f01 100644 --- a/apps/server/test/fixtures/schemas/references.ts +++ b/apps/server/test/fixtures/schemas/references.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { Schema } from 'dictionary'; +import { Schema } from '@overture-stack/lectern-dictionary'; export const references = { listA: ['a', 'b', 'c'], diff --git a/apps/server/test/fixtures/schemas/schemaChangesBase.ts b/apps/server/test/fixtures/schemas/schemaChangesBase.ts index 7751a85..e80d66b 100644 --- a/apps/server/test/fixtures/schemas/schemaChangesBase.ts +++ b/apps/server/test/fixtures/schemas/schemaChangesBase.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { Schema } from 'dictionary'; +import { Schema } from '@overture-stack/lectern-dictionary'; export default { name: 'schema_tests', diff --git a/apps/server/test/fixtures/schemas/schemaChangesUpdated.ts b/apps/server/test/fixtures/schemas/schemaChangesUpdated.ts index 4ca12cb..cb0d8ad 100644 --- a/apps/server/test/fixtures/schemas/schemaChangesUpdated.ts +++ b/apps/server/test/fixtures/schemas/schemaChangesUpdated.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { Schema } from 'dictionary'; +import { Schema } from '@overture-stack/lectern-dictionary'; export default { name: 'schema_tests', diff --git a/apps/server/test/functional/services/dictionaryService.spec.ts b/apps/server/test/functional/services/dictionaryService.spec.ts index 9bfadbe..607460f 100644 --- a/apps/server/test/functional/services/dictionaryService.spec.ts +++ b/apps/server/test/functional/services/dictionaryService.spec.ts @@ -19,7 +19,7 @@ import chai, { expect } from 'chai'; import chaiAsPromised from 'chai-as-promised'; -import { DictionaryDocument, VersionUtils } from 'dictionary'; +import { VersionUtils } from '@overture-stack/lectern-dictionary'; import * as immer from 'immer'; import { Error as MongooseError } from 'mongoose'; import sinon from 'sinon'; @@ -34,6 +34,7 @@ import SCHEMA_1_EXISTING from '../../fixtures/schemas/primitives'; import SCHEMA_REFERENCES, { references } from '../../fixtures/schemas/references'; import SCHEMA_2_ADD from '../../fixtures/schemas/schemaChangesBase'; import SCHEMA_3_UPDATES from '../../fixtures/schemas/schemaChangesUpdated'; +import type { DictionaryDocument } from '../../../src/db/dbTypes'; chai.use(chaiAsPromised); diff --git a/package.json b/package.json index 9398856..fadba1d 100644 --- a/package.json +++ b/package.json @@ -1,22 +1,20 @@ { - "name": "lectern", + "name": "@overture-stack/lectern", + "private": true, "version": "2.0.0-next.0", - "description": "Data Dictionary Schema Manager and Validator", - "main": "index.js", + "description": "Schema Manager and Validation for Data Dictionaries", "scripts": { "build:all": "pnpm nx run-many --all --target=build", - "build:common": "pnpm nx build common", - "build:dictionary": "pnpm nx build dictionary", + "build:dictionary": "pnpm nx build @overture-stack/lectern-dictionary", "build:validation": "pnpm nx build @overture-stack/lectern-validation", - "build:server": "pnpm nx build server", + "build:server": "pnpm nx build @overture-stack/lectern-server", "build:client": "pnpm nx build @overture-stack/lectern-client", "test:all": "pnpm nx run-many --all --target=test", - "test:common": "pnpm nx test common", - "test:dictionary": "pnpm nx test dictionary", + "test:dictionary": "pnpm nx test @overture-stack/lectern-dictionary", "test:validation": "pnpm nx test @overture-stack/lectern-validation", - "test:server": "pnpm nx test server", + "test:server": "pnpm nx test @overture-stack/lectern-server", "test:client": "pnpm nx test @overture-stack/lectern-client", - "generate": "pnpm build:dictionary && pnpm -C generate generate" + "generate": "pnpm build:dictionary && pnpm -C scripts generate" }, "keywords": [], "author": "Ontario Institute for Cancer Research", @@ -24,18 +22,18 @@ "devDependencies": { "@types/chai": "^4.3.16", "@types/chai-as-promised": "^7.1.8", - "@types/mocha": "^10.0.6", - "@types/node": "^20.14.7", + "@types/mocha": "^10.0.7", + "@types/node": "^20.14.13", "@types/sinon": "^10.0.20", - "chai": "^4.4.1", + "chai": "^4.5.0", "chai-as-promised": "^7.1.2", "chai-http": "^4.4.0", - "mocha": "^10.4.0", + "mocha": "^10.7.0", "nx": "^16.10.0", "nyc": "^15.1.0", - "prettier": "^3.3.2", + "prettier": "^3.3.3", "sinon": "^15.2.0", "ts-node": "^10.9.2", - "typescript": "^5.5.2" + "typescript": "^5.5.4" } } diff --git a/packages/client/README.md b/packages/client/README.md index f517a4f..ba0390b 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -5,43 +5,61 @@ The Lectern client provides developers the mechanism to interact with Lectern servers and to use Lectern Dictionaries. The client provides all the validation logic to check that submitted data is valid based on the structure and restrictions of a Lectern dictionary. It also provides a REST client to fetch Lectern Dictionary data from a Lectern server. ## Features +- Interact with lectern servers: + - Fetch dictionary by name and version + - Fetch difference summaries between dictionary versions - Process data using a Lectern Dictionary: - Convert raw string inputs into properly typed values. - Check the structure of input data is valid. - Apply all restrictions, both across schemas and on individual fields, to validate input data. - Report all validation errors found in the input data. -- Interact with lectern servers: - - Fetch dictionary by name and version - - Fetch difference summaries between dictionary versions -## Data Processing Usage -### Process Data for a Single Schema - -The following example shows how to process data using the Lectern Client. The input `donorData` is presented as hardcoded, but in a typical scenario this would be submitted to the application through an uploaded TSV, form entry, or similar user submission system. - -To process data records which all belong to the same schema we use the `processRecords` function: +## Data Fetching Example ```ts import * as lectern from '@overture-stack/lectern-client'; -import type { Dictionary, UnprocessedDataRecord } from '@overture-stack/lectern-client'; +const lecternUrl = 'http://lectern.example.com'; +const dictionaryName = 'dictionary-name'; +const currentVersion = "2.3"; +const previousVersion = "2.1"; + +const dictionary = lectern.restClient.fetchSchema(lecternUrl, dictionaryName, currentVersion); +const versionUpdates = lectern.restClient.fetchDiff(lecternUrl, dictionaryName, currentVersion, previousVersion); +``` -const dictionary: Dictionary = await getLecternDictionary(); +## Data Processing Usage -const donorData: UnprocessedDataRecord = [{submitter_donor_id: "abc123", gender: "Male", age: "28"}, {submitter_donor_id: "def456", gender: "Female", age: "37"}] +### Process Data for a Single Schema -const { processedRecords, validationErrors } = lectern.process.schema(dictionary, "donors", donorData); -``` +The following example shows how to process data using the Lectern Client. The input `donorData` is presented as hardcoded, but in a typical scenario this would be submitted to the application through an uploaded TSV, form entry, or similar user submission system. -## Data Fetching Example +To process data records which all belong to the same schema we use the `processSchema` function: ```ts import * as lectern from '@overture-stack/lectern-client'; -const currentVersion = "2.3"; -const previousVersion = "2.1"; -const dictionary = client.restClient.fetchSchema('http://lectern.example.com', 'dictionary-name', currentVersion); -const versionUpdates = client.restClient.fetchDiff('http://lectern.example.com', 'dictionary-name', currentVersion, previousVersion); +const dictionary = await getLecternDictionary(); + +const donorData = [{submitter_donor_id: "abc123", gender: "Male", age: "28"}, {submitter_donor_id: "def456", gender: "Female", age: "37"}] + +const schemaProcessingResult = lectern.functions.processSchema(dictionary, "donors", donorData); + +switch (schemaProcessingResult.status) { + case 'SUCCESS': { + const { records } = schemaProcessingResult; + // use converted and validated records + } + case 'ERROR_PARSING': { + const { errors, records } = schemaProcessingResult; + // errors occured parsing records. read the errors that occurred + // records have been return with their values parsed where possible. If an error occurred, the original input string value is returned + } + case 'ERROR_VALIDATION': { + const { records, errors } = schemaProcessingResult; + // errors occured validating records. these errors have been returned + // records were parsed successfully, so this returns all parsed records + } ``` diff --git a/packages/client/package.json b/packages/client/package.json index 3f21f95..8a24840 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -22,30 +22,29 @@ "license": "AGPL-3.0", "devDependencies": { "@types/chai": "^4.3.16", - "@types/lodash": "^4.17.5", + "@types/lodash": "^4.17.7", "@types/mocha": "^8.2.3", "@types/node": "^12.20.55", "@types/node-fetch": "^2.6.11", - "chai": "^4.4.1", + "chai": "^4.5.0", "husky": "^6.0.0", "mocha": "^8.4.0", "prettier": "^2.8.8", "pretty-quick": "^3.3.1", - "rimraf": "^5.0.7", + "rimraf": "^5.0.9", "ts-node": "^9.1.1", "tslint": "^6.1.3", "typedoc": "^0.17.8", - "typescript": "^5.5.2" + "typescript": "^5.5.4" }, "dependencies": { + "@overture-stack/lectern-dictionary": "workspace:^", "@overture-stack/lectern-validation": "workspace:^", "cd": "^0.3.3", - "common": "workspace:^", - "dictionary": "workspace:^", "lodash": "^4.17.21", "node-fetch": "^2.7.0", "promise-tools": "^2.1.0", - "winston": "^3.13.0" + "winston": "^3.13.1" }, "author": "Ontario Institute for Cancer Research" } diff --git a/packages/client/src/changeAnalysis/changeAnalysisTypes.ts b/packages/client/src/changeAnalysis/changeAnalysisTypes.ts index 853a918..79c3561 100644 --- a/packages/client/src/changeAnalysis/changeAnalysisTypes.ts +++ b/packages/client/src/changeAnalysis/changeAnalysisTypes.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { SchemaField, ValueChangeTypeName } from 'dictionary'; +import { SchemaField, ValueChangeTypeName } from '@overture-stack/lectern-dictionary'; type ChangeOnlyTypeNames = Exclude; diff --git a/packages/client/src/changeAnalysis/changeAnalyzer.ts b/packages/client/src/changeAnalysis/changeAnalyzer.ts index 77adcdf..55f50ca 100644 --- a/packages/client/src/changeAnalysis/changeAnalyzer.ts +++ b/packages/client/src/changeAnalysis/changeAnalyzer.ts @@ -17,7 +17,13 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { DictionaryDiff, FieldChanges, RestrictionRange, SchemaField, ValueChange } from 'dictionary'; +import { + DictionaryDiff, + FieldChanges, + RestrictionRange, + SchemaField, + ValueChange, +} from '@overture-stack/lectern-dictionary'; import { restClient } from '../rest'; import { ChangeAnalysis, RestrictionChanges } from './changeAnalysisTypes'; diff --git a/packages/client/src/processing/index.ts b/packages/client/src/processing/index.ts index 942ebad..28f6894 100644 --- a/packages/client/src/processing/index.ts +++ b/packages/client/src/processing/index.ts @@ -18,7 +18,7 @@ */ import * as validation from '@overture-stack/lectern-validation'; -import { DataRecord, Dictionary, Schema, UnprocessedDataRecord } from 'dictionary'; +import { DataRecord, Dictionary, Schema, UnprocessedDataRecord } from '@overture-stack/lectern-dictionary'; import { loggerFor } from '../logger'; import { SchemaProcessingResult, diff --git a/packages/client/src/processing/processingResultTypes.ts b/packages/client/src/processing/processingResultTypes.ts index 4769b9c..899a177 100644 --- a/packages/client/src/processing/processingResultTypes.ts +++ b/packages/client/src/processing/processingResultTypes.ts @@ -25,11 +25,11 @@ import { type ParseSchemaFailureData, type ParseDictionaryData, } from '@overture-stack/lectern-validation'; -import { DataRecord, Schema } from 'dictionary'; +import { DataRecord, Schema } from '@overture-stack/lectern-dictionary'; export type ProcessingFunction = (schema: Schema, rec: Readonly, index: number) => any; -type RecordProcessingErrorSuccess = { +type RecordProcessingSuccess = { status: 'SUCCESS'; record: DataRecord; }; @@ -42,11 +42,11 @@ type RecordProcessingErrorValidation = { errors: RecordValidationError[]; }; export type RecordProcessingResult = - | RecordProcessingErrorSuccess + | RecordProcessingSuccess | RecordProcessingErrorParsing | RecordProcessingErrorValidation; -type SchemaProcessingErrorSuccess = { +type SchemaProcessingSuccess = { status: 'SUCCESS'; records: DataRecord[]; }; @@ -59,11 +59,11 @@ type SchemaProcessingErrorValidation = { errors: SchemaValidationError[]; }; export type SchemaProcessingResult = - | SchemaProcessingErrorSuccess + | SchemaProcessingSuccess | SchemaProcessingErrorParsing | SchemaProcessingErrorValidation; -type DictionaryProcessingErrorSuccess = { +type DictionaryProcessingSuccess = { status: 'SUCCESS'; data: Record; }; @@ -81,7 +81,7 @@ type DictionaryProcessingErrorValidation = { errors: DictionaryValidationError[]; }; export type DictionaryProcessingResult = - | DictionaryProcessingErrorSuccess + | DictionaryProcessingSuccess | DictionaryProcessingErrorParsing | DictionaryProcessingErrorValidation; diff --git a/packages/client/src/rest/index.ts b/packages/client/src/rest/index.ts index 1d39073..0e49d18 100644 --- a/packages/client/src/rest/index.ts +++ b/packages/client/src/rest/index.ts @@ -17,8 +17,13 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { unknownToString } from 'common'; -import { Dictionary, DictionaryDiff, DictionaryDiffArray, FieldDiff } from 'dictionary'; +import { + Dictionary, + DictionaryDiff, + DictionaryDiffArray, + FieldDiff, + unknownToString, +} from '@overture-stack/lectern-dictionary'; import fetch from 'node-fetch'; import promiseTools from 'promise-tools'; import { loggerFor } from '../logger'; diff --git a/packages/client/test/changeAnalyzer.spec.ts b/packages/client/test/changeAnalyzer.spec.ts index c2bbff0..426cd58 100644 --- a/packages/client/test/changeAnalyzer.spec.ts +++ b/packages/client/test/changeAnalyzer.spec.ts @@ -18,7 +18,7 @@ */ import chai from 'chai'; -import { DiffUtils } from 'dictionary'; +import { DiffUtils } from '@overture-stack/lectern-dictionary'; import { analyzer } from '../src'; import { ChangeAnalysis } from '../src/changeAnalysis'; import diffResponse from './fixtures/diffResponse'; diff --git a/packages/client/test/fixtures/diffResponse.ts b/packages/client/test/fixtures/diffResponse.ts index cf16490..fdb45f6 100644 --- a/packages/client/test/fixtures/diffResponse.ts +++ b/packages/client/test/fixtures/diffResponse.ts @@ -1,4 +1,4 @@ -import { DictionaryDiffArray } from 'dictionary'; +import { DictionaryDiffArray } from '@overture-stack/lectern-dictionary'; const diffResponse = [ [ diff --git a/packages/client/test/fixtures/registrationSchema.ts b/packages/client/test/fixtures/registrationSchema.ts index 4d57053..5fbbd3c 100644 --- a/packages/client/test/fixtures/registrationSchema.ts +++ b/packages/client/test/fixtures/registrationSchema.ts @@ -1,4 +1,4 @@ -import { Dictionary } from 'dictionary'; +import { Dictionary } from '@overture-stack/lectern-dictionary'; const dictionary: Dictionary = { schemas: [ diff --git a/packages/common/.mocharc.json b/packages/common/.mocharc.json deleted file mode 100644 index 6d0022d..0000000 --- a/packages/common/.mocharc.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "extension": ["ts"], - "require": "ts-node/register", - "spec": "test/**/*.spec.ts" -} diff --git a/packages/common/README.md b/packages/common/README.md deleted file mode 100644 index e9c1247..0000000 --- a/packages/common/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Lectern Common Utils - -Contains generic types and utilities used across Lectern modules. - -Also exported are custom error classes. Please note: it is prefered to return Result objects with typed error details rather than throw unexpectedly. Use of thrown errors is being refactored out where possible. \ No newline at end of file diff --git a/packages/common/package.json b/packages/common/package.json deleted file mode 100644 index 80bb858..0000000 --- a/packages/common/package.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "name": "common", - "version": "1.0.0", - "description": "", - "main": "dist/index.js", - "scripts": { - "build": "pnpm build:clean && tsc", - "build:clean": "rimraf dist/ && mkdir dist" - }, - "keywords": [], - "author": "Ontario Institute for Cancer Research", - "license": "AGPL-3.0", - "devDependencies": { - "rimraf": "^5.0.0" - } -} diff --git a/packages/common/src/index.ts b/packages/common/src/index.ts deleted file mode 100644 index c85c35d..0000000 --- a/packages/common/src/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './errors'; -export * from './types'; -export * from './utils'; diff --git a/packages/common/src/utils/objectUtils.ts b/packages/common/src/utils/objectUtils.ts deleted file mode 100644 index 48dcba8..0000000 --- a/packages/common/src/utils/objectUtils.ts +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright (c) 2023 The Ontario Institute for Cancer Research. All rights reserved - * - * This program and the accompanying materials are made available under the terms of - * the GNU Affero General Public License v3.0. You should have received a copy of the - * GNU Affero General Public License along with this program. - * If not, see . - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT - * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER - * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -type CompareFunction = (a: T, b: T) => number; - -const defaultSort = (a: T, b: T) => { - return a > b ? 1 : -1; -}; - -/** - * Wrapper for a search function that can define a priority order for specific values. This will return a - * search function that will check if values are in the priorityArray, and if so will sort them in the - * order found in the array. All values not found in the array will be sorted according to the provided - * sortingFunction. - * @param priorityArray Array of values in a specified priority sort order - * @param sortingFunction Optional sorting function for case where neither value is found in the priorityArray. - * Default sorting function is the defaultSort which does a JavaScript greater than operation to compare values. - * @returns A new CompareFunction that will prioritize items found in the priorityArray. - * - * @example - * ``` - * const array = ["c", "z", "x", "b", "a", "y"]; - * const customSortFunction = sortWithPriority(["z", "y", "x"]); - * - * const sortedArray = array.sort(customSortFunction); - * // Result: ["z", "y", "x", "a", "b", "c"]; - * ``` - */ -export const sortWithPriority = - (priorityArray: T[], sortingFunction: CompareFunction = defaultSort): CompareFunction => - (a, b) => { - const indexA = priorityArray.indexOf(a); - const indexB = priorityArray.indexOf(b); - if (indexA >= 0) { - if (indexB >= 0) { - // Has A and B - return indexA > indexB ? 1 : -1; - } else { - // Has A not B - return -1; - } - } else if (indexB >= 0) { - // Has B not A - return 1; - } else { - // Has neither, use sortingFunction for remainder - return sortingFunction(a, b); - } - }; - -/** - * Create a deep clone of an object with all properties sorted. This is helpful for preparing a data structure - * for presentation in a predictable way. For example, a user submitted JSON object will have - * properties sorted randomly, while an API response that includes this user data should be always returned with - * object properties in a consistent order. - * - * The sort order will by default be alphabetical. A custom sortingFunction can be provided to - * sort the object properties with custom order or logic. - * - * Note: This only operates on own properties, not inherited properties. - * @param input An object to be cloned with sorted properties - * @param sortingFunction Optional property, a sorting function to determine the order of properties - * in the output object - * @returns Deep clone of the input object - */ -export const sortProperties = (input: T, sortingFunction = defaultSort): T => { - const sortedEntries = Object.keys(input).sort(sortingFunction); - return sortedEntries.reduce((acc, key) => { - const value = (input as any)[key]; - if (typeof value === 'object' && value !== null) { - acc[key] = sortProperties(value, sortingFunction); - } else { - acc[key] = value; - } - return acc; - }, {}); -}; diff --git a/packages/dictionary/README.md b/packages/dictionary/README.md index a466b00..f6a26cd 100644 --- a/packages/dictionary/README.md +++ b/packages/dictionary/README.md @@ -1,9 +1,10 @@ -# Lectern Dictionary Schema and Utilities +# Lectern Dictionary Meta-Schema and Utilities -## Schema Definition for Lectern Developers +[![NPM Version](https://img.shields.io/npm/v/@overture-stack/lectern-dictionary?color=%23cb3837&style=for-the-badge&logo=npm)](https://www.npmjs.com/package/@overture-stack/lectern-dictionary) -The Lectern server uses a TypeScript native schema generated with [Zod](https://zod.dev/) that can be found in the [`./src/types/dictionaryTypes.ts`](./src/types/dictionaryTypes.ts) file. There is a custom script `npm run generate` that will regenerate the content in the `DictionaryMetaSchema.json`. The JSON Schema file is generated thanks to the libary [zod-to-JSON Schema](https://www.npmjs.com/package/zod-to-JSON Schema). +This package defines the structure of Lectern Dictionaries, including providing the TypeScript type definitions to use the dictionary in code and the schemas to validate that a given JSON object is a valid Lectern Dictionary. -Whenever changes are made to the Zod Schemas in `./src/types` the generation script needs to be re-run and the updated Dictionary Meta Schema committed to the repository. +The Lectern Dictionary meta-schema is formally defined in TypeScript and exported as the type `Dictionary`. This definition is created using `Zod` schemas, which are also exported from this package and available for use to confirm a given object is a valid Lectern Dictionary. + +A [JSON Schema definition](https://github.com/overture-stack/lectern/blob/develop/generated/DictionaryMetaSchema.json) of the Lectern Dictionary structure is also available if you are looking for a non-TypeScript mechanism for validating Lectern Dictionaries. -If the generation script needs updating, it can be found at [`./scripts/buildMetaSchema.ts`](./scripts/buildMetaSchema.ts). diff --git a/packages/dictionary/package.json b/packages/dictionary/package.json index fc24a33..735c321 100644 --- a/packages/dictionary/package.json +++ b/packages/dictionary/package.json @@ -1,6 +1,6 @@ { - "name": "dictionary", - "version": "1.0.0", + "name": "@overture-stack/lectern-dictionary", + "version": "0.1.0-beta.1", "description": "", "main": "dist/index.js", "scripts": { @@ -12,13 +12,12 @@ "author": "Ontario Institute for Cancer Research", "license": "AGPL-3.0", "dependencies": { - "common": "workspace:^", - "immer": "^10.0.2", + "immer": "^10.1.1", "lodash": "^4.17.21", - "zod": "^3.21.4" + "zod": "^3.23.8" }, "devDependencies": { - "@types/lodash": "^4.14.195", - "rimraf": "^5.0.0" + "@types/lodash": "^4.17.7", + "rimraf": "^5.0.9" } } diff --git a/packages/common/src/errors.ts b/packages/dictionary/src/errors.ts similarity index 100% rename from packages/common/src/errors.ts rename to packages/dictionary/src/errors.ts diff --git a/packages/dictionary/src/index.ts b/packages/dictionary/src/index.ts index 27be2f7..f3ddde2 100644 --- a/packages/dictionary/src/index.ts +++ b/packages/dictionary/src/index.ts @@ -1,5 +1,26 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + export * from './constants'; +export * from './errors'; export * as DiffUtils from './diff'; +export * from './metaSchema'; export * from './references'; export * from './types'; -export * as VersionUtils from './utils/version'; +export * from './utils'; diff --git a/packages/dictionary/src/types/dictionaryTypes.ts b/packages/dictionary/src/metaSchema/dictionarySchemas.ts similarity index 94% rename from packages/dictionary/src/types/dictionaryTypes.ts rename to packages/dictionary/src/metaSchema/dictionarySchemas.ts index 42885d1..1e159b3 100644 --- a/packages/dictionary/src/types/dictionaryTypes.ts +++ b/packages/dictionary/src/metaSchema/dictionarySchemas.ts @@ -19,8 +19,7 @@ import { z as zod } from 'zod'; import allUnique from '../utils/allUnique'; -import { NameString } from './commonTypes'; -import { ReferenceTag, References } from './referenceTypes'; +import { ReferenceTag, References } from './referenceSchemas'; import { RestrictionCodeListInteger, RestrictionCodeListNumber, @@ -29,7 +28,22 @@ import { RestrictionNumberRange, RestrictionRegex, RestrictionScript, -} from './restrictionsTypes'; +} from './restrictionsSchemas'; + +/** + * String rules for all name fields used in dictionary, including Dictionary, Schema, and Fields. + * This validates the format of the string since names are not allowed to have `.` characters. + * + * Example Values: + * - `donors` + * - `primary-site` + * - `maximumVelocity` + */ +export const NameValue = zod + .string() + .min(1, 'Name fields cannot be empty.') + .regex(/^[^.]+$/, 'Name fields cannot have `.` characters.'); +export type NameValue = zod.infer; // Meta accepts as values only strings, numbers, booleans, arrays of numbers or arrays of strings // Another Meta object can be nested inside a Meta property @@ -95,7 +109,7 @@ export type BooleanFieldRestrictions = zod.infer; export const Schema = zod .object({ - name: NameString, + name: NameValue, description: zod.string().optional(), fields: zod.array(SchemaField).min(1), meta: DictionaryMeta.optional(), restrictions: zod .object({ foreignKey: zod.array(ForeignKeyRestriction).min(1), - uniqueKey: zod.array(NameString).min(1), + uniqueKey: zod.array(NameValue).min(1), }) .partial() .optional(), diff --git a/packages/common/src/types/index.ts b/packages/dictionary/src/metaSchema/index.ts similarity index 86% rename from packages/common/src/types/index.ts rename to packages/dictionary/src/metaSchema/index.ts index bf51180..cf7f5be 100644 --- a/packages/common/src/types/index.ts +++ b/packages/dictionary/src/metaSchema/index.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 The Ontario Institute for Cancer Research. All rights reserved + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved * * This program and the accompanying materials are made available under the terms of * the GNU Affero General Public License v3.0. You should have received a copy of the @@ -17,7 +17,6 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -export * from './defined'; -export * from './generics'; -export * from './result'; -export * from './singular'; +export * from './dictionarySchemas'; +export * from './referenceSchemas'; +export * from './restrictionsSchemas'; diff --git a/packages/dictionary/src/types/referenceTypes.ts b/packages/dictionary/src/metaSchema/referenceSchemas.ts similarity index 100% rename from packages/dictionary/src/types/referenceTypes.ts rename to packages/dictionary/src/metaSchema/referenceSchemas.ts diff --git a/packages/dictionary/src/types/restrictionsTypes.ts b/packages/dictionary/src/metaSchema/restrictionsSchemas.ts similarity index 96% rename from packages/dictionary/src/types/restrictionsTypes.ts rename to packages/dictionary/src/metaSchema/restrictionsSchemas.ts index d4c7b4c..f9835e4 100644 --- a/packages/dictionary/src/types/restrictionsTypes.ts +++ b/packages/dictionary/src/metaSchema/restrictionsSchemas.ts @@ -18,9 +18,10 @@ */ import { z as zod } from 'zod'; -import { ReferenceTag } from './referenceTypes'; -import { Integer } from './commonTypes'; -import type { Values } from 'common'; +import { ReferenceTag } from './referenceSchemas'; +import type { Values } from '../types'; + +export const Integer = zod.number().int(); export const FieldRestrictionTypes = { codeList: 'codeList', diff --git a/packages/dictionary/src/references.ts b/packages/dictionary/src/references.ts index 35c4821..8c3a9df 100644 --- a/packages/dictionary/src/references.ts +++ b/packages/dictionary/src/references.ts @@ -19,8 +19,8 @@ import * as immer from 'immer'; import { cloneDeep, get, isObject, omit } from 'lodash'; -import { Dictionary, ReferenceArray, ReferenceTag, ReferenceValue, References, Schema } from '.'; -import { InvalidReferenceError, asArray } from 'common'; +import { Dictionary, ReferenceArray, ReferenceTag, ReferenceValue, References, Schema, TypeUtils } from '.'; +import { InvalidReferenceError } from './errors'; // This is the union of all schema sections that could have reference values type OutputReferenceValues = ReferenceArray | ReferenceValue; @@ -83,23 +83,23 @@ const internalReplaceSchemaReferences = ( // All the checking for undefined prevents us from adding properties with value undefined into the field's ouput JSON case 'string': if (field.restrictions.codeList !== undefined) { - field.restrictions.codeList = asArray(resolveRestriction(field.restrictions.codeList)); + field.restrictions.codeList = TypeUtils.asArray(resolveRestriction(field.restrictions.codeList)); } if (field.restrictions.regex !== undefined) { field.restrictions.regex = resolveNoArrays(field.restrictions.regex, 'regex'); } if (field.restrictions.script !== undefined) { - field.restrictions.script = asArray(resolveRestriction(field.restrictions.script)); + field.restrictions.script = TypeUtils.asArray(resolveRestriction(field.restrictions.script)); } break; case 'number': if (field.restrictions.script !== undefined) { - field.restrictions.script = asArray(resolveRestriction(field.restrictions.script)); + field.restrictions.script = TypeUtils.asArray(resolveRestriction(field.restrictions.script)); } break; case 'integer': if (field.restrictions.script !== undefined) { - field.restrictions.script = asArray(resolveRestriction(field.restrictions.script)); + field.restrictions.script = TypeUtils.asArray(resolveRestriction(field.restrictions.script)); } break; case 'boolean': diff --git a/packages/dictionary/src/types/commonTypes.ts b/packages/dictionary/src/types/commonTypes.ts deleted file mode 100644 index bfaab91..0000000 --- a/packages/dictionary/src/types/commonTypes.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { z as zod } from 'zod'; - -export const Integer = zod.number().int(); - -/** - * String rules for all name fields used in dictionary, including Dictionary, Schema, and Fields. - * This validates the format of the string since names are not allowed to have `.` characters. - * - * Example Values: - * - `donors` - * - `primary-site` - * - `maximumVelocity` - */ -export const NameString = zod - .string() - .min(1, 'Name fields cannot be empty.') - .regex(/^[^.]+$/, 'Name fields cannot have `.` characters.'); -export type NameString = zod.infer; diff --git a/packages/dictionary/src/types/dataTypes.ts b/packages/dictionary/src/types/dataRecord.ts similarity index 98% rename from packages/dictionary/src/types/dataTypes.ts rename to packages/dictionary/src/types/dataRecord.ts index 917f447..aba8842 100644 --- a/packages/dictionary/src/types/dataTypes.ts +++ b/packages/dictionary/src/types/dataRecord.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import type { Singular } from 'common'; +import type { Singular } from '.'; /** * Represents a data record as taken from an input file. All values are the original strings and have not been validated into diff --git a/packages/common/src/types/defined.ts b/packages/dictionary/src/types/defined.ts similarity index 100% rename from packages/common/src/types/defined.ts rename to packages/dictionary/src/types/defined.ts diff --git a/packages/dictionary/src/types/diffTypes.ts b/packages/dictionary/src/types/diffTypes.ts index 976e3a4..77eeee5 100644 --- a/packages/dictionary/src/types/diffTypes.ts +++ b/packages/dictionary/src/types/diffTypes.ts @@ -18,7 +18,7 @@ */ import { z as zod } from 'zod'; -import { SchemaField } from './dictionaryTypes'; +import { SchemaField } from '../metaSchema'; export const ValueChangeTypeNames = { CREATED: 'created', diff --git a/packages/common/src/types/generics.ts b/packages/dictionary/src/types/generics.ts similarity index 100% rename from packages/common/src/types/generics.ts rename to packages/dictionary/src/types/generics.ts diff --git a/packages/dictionary/src/types/index.ts b/packages/dictionary/src/types/index.ts index 220663d..b8e308f 100644 --- a/packages/dictionary/src/types/index.ts +++ b/packages/dictionary/src/types/index.ts @@ -17,10 +17,9 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -export * from './commonTypes'; -export * from './dataTypes'; -export * from './dbTypes'; -export * from './dictionaryTypes'; +export * from './dataRecord'; +export * from './defined'; export * from './diffTypes'; -export * from './referenceTypes'; -export * from './restrictionsTypes'; +export * from './generics'; +export * from './result'; +export * from './singular'; diff --git a/packages/common/src/types/result.ts b/packages/dictionary/src/types/result.ts similarity index 100% rename from packages/common/src/types/result.ts rename to packages/dictionary/src/types/result.ts diff --git a/packages/common/src/types/singular.ts b/packages/dictionary/src/types/singular.ts similarity index 100% rename from packages/common/src/types/singular.ts rename to packages/dictionary/src/types/singular.ts diff --git a/packages/common/src/utils/index.ts b/packages/dictionary/src/utils/index.ts similarity index 88% rename from packages/common/src/utils/index.ts rename to packages/dictionary/src/utils/index.ts index 8bcd0f1..6b9e180 100644 --- a/packages/common/src/utils/index.ts +++ b/packages/dictionary/src/utils/index.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 The Ontario Institute for Cancer Research. All rights reserved + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved * * This program and the accompanying materials are made available under the terms of * the GNU Affero General Public License v3.0. You should have received a copy of the @@ -18,4 +18,5 @@ */ export * from './stringUtils'; -export * from './typeUtils'; +export * as TypeUtils from './typeUtils'; +export * as VersionUtils from './versionUtils'; diff --git a/packages/dictionary/src/utils/schemaUtils.ts b/packages/dictionary/src/utils/schemaUtils.ts index 6c55fdf..7f6b520 100644 --- a/packages/dictionary/src/utils/schemaUtils.ts +++ b/packages/dictionary/src/utils/schemaUtils.ts @@ -1,4 +1,4 @@ -import { Schema, SchemaField } from 'types'; +import type { Schema, SchemaField } from '../metaSchema'; /** * Get an array of fields from this schema that have the required restriction set to true diff --git a/packages/common/src/utils/stringUtils.ts b/packages/dictionary/src/utils/stringUtils.ts similarity index 100% rename from packages/common/src/utils/stringUtils.ts rename to packages/dictionary/src/utils/stringUtils.ts diff --git a/packages/common/src/utils/typeUtils.ts b/packages/dictionary/src/utils/typeUtils.ts similarity index 100% rename from packages/common/src/utils/typeUtils.ts rename to packages/dictionary/src/utils/typeUtils.ts diff --git a/packages/dictionary/src/utils/version.ts b/packages/dictionary/src/utils/versionUtils.ts similarity index 97% rename from packages/dictionary/src/utils/version.ts rename to packages/dictionary/src/utils/versionUtils.ts index 65d038e..925c338 100644 --- a/packages/dictionary/src/utils/version.ts +++ b/packages/dictionary/src/utils/versionUtils.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { MalformedVersionError } from 'common'; +import { MalformedVersionError } from '../errors'; export const isValidVersion = (version: string): boolean => { return /^[0-9]+\.[0-9]+$/.test(version); diff --git a/packages/dictionary/test/dictionaryTypes.spec.ts b/packages/dictionary/test/dictionaryTypes.spec.ts index 560cada..2637f8a 100644 --- a/packages/dictionary/test/dictionaryTypes.spec.ts +++ b/packages/dictionary/test/dictionaryTypes.spec.ts @@ -24,7 +24,7 @@ import { DictionaryMeta, Integer, IntegerFieldRestrictions, - NameString, + NameValue, NumberFieldRestrictions, RestrictionIntegerRange, RestrictionNumberRange, @@ -35,22 +35,22 @@ import { } from '../src'; describe('Dictionary Types', () => { - describe('NameString', () => { + describe('NameValue', () => { it('Rejects empty string', () => { - expect(NameString.safeParse('').success).false; + expect(NameValue.safeParse('').success).false; }); it('Can be string', () => { - expect(NameString.safeParse('any').success).true; - expect(NameString.safeParse('123').success).true; - expect(NameString.safeParse('_').success).true; + expect(NameValue.safeParse('any').success).true; + expect(NameValue.safeParse('123').success).true; + expect(NameValue.safeParse('_').success).true; // NOTE: if we want to limit the property names we should explicitly declare those reules, right now all characters are valid and the strings dont have to start with a letter }); it("Can't contain a `.`", () => { - expect(NameString.safeParse('asdf.asdf').success).false; - expect(NameString.safeParse('.').success).false; - expect(NameString.safeParse('.asdf').success).false; - expect(NameString.safeParse('adsf.').success).false; - expect(NameString.safeParse('\\.').success).false; + expect(NameValue.safeParse('asdf.asdf').success).false; + expect(NameValue.safeParse('.').success).false; + expect(NameValue.safeParse('.asdf').success).false; + expect(NameValue.safeParse('adsf.').success).false; + expect(NameValue.safeParse('\\.').success).false; }); }); diff --git a/packages/validation/package.json b/packages/validation/package.json index 2ad15e3..0572342 100644 --- a/packages/validation/package.json +++ b/packages/validation/package.json @@ -1,6 +1,6 @@ { "name": "@overture-stack/lectern-validation", - "version": "0.1.0", + "version": "0.1.0-beta.1", "description": "Logic for validating data using a Lectern dictionary", "main": "dist/index.js", "scripts": { @@ -12,13 +12,12 @@ "author": "Ontario Institute for Cancer Research", "license": "AGPL-3.0", "dependencies": { - "common": "workspace:^", - "dictionary": "workspace:^", + "@overture-stack/lectern-dictionary": "workspace:^", "lodash": "^4.17.21", - "zod": "^3.21.4" + "zod": "^3.23.8" }, "devDependencies": { - "@types/lodash": "^4.14.195", - "rimraf": "^5.0.0" + "@types/lodash": "^4.17.7", + "rimraf": "^5.0.9" } } diff --git a/packages/validation/src/parseValues/ParseValuesResult.ts b/packages/validation/src/parseValues/ParseValuesResult.ts index c8b561a..3d4e87a 100644 --- a/packages/validation/src/parseValues/ParseValuesResult.ts +++ b/packages/validation/src/parseValues/ParseValuesResult.ts @@ -17,10 +17,9 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import type { DataRecord } from 'dictionary'; +import type { DataRecord, Result } from '@overture-stack/lectern-dictionary'; import type { RecordValidationErrorInvalidValue, RecordValidationErrorUnrecognizedField } from '../validateRecord'; import type { SchemaRecordError } from '../validateSchema'; -import type { Result } from 'common'; export type ParseFieldError = RecordValidationErrorInvalidValue | RecordValidationErrorUnrecognizedField; diff --git a/packages/validation/src/parseValues/matchCodeListFormatting.ts b/packages/validation/src/parseValues/matchCodeListFormatting.ts index 8b6d860..7276746 100644 --- a/packages/validation/src/parseValues/matchCodeListFormatting.ts +++ b/packages/validation/src/parseValues/matchCodeListFormatting.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import type { SchemaField } from 'dictionary'; +import type { SchemaField } from '@overture-stack/lectern-dictionary'; /** * Given a string value, look for any matching values in code list restrictions and return that diff --git a/packages/validation/src/parseValues/parseValues.ts b/packages/validation/src/parseValues/parseValues.ts index b076dde..0a3d241 100644 --- a/packages/validation/src/parseValues/parseValues.ts +++ b/packages/validation/src/parseValues/parseValues.ts @@ -17,28 +17,30 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { failure, failWith, success, type Result } from 'common'; import { DEFAULT_DELIMITER, + failure, + failWith, + success, type ArrayDataValue, type DataRecord, type DataRecordValue, type Dictionary, - type RestrictionCodeListString, + type Result, type Schema, type SchemaField, type SchemaFieldValueType, type UnprocessedDataRecord, -} from 'dictionary'; +} from '@overture-stack/lectern-dictionary'; import { isInteger, isNumber } from '../utils/typeUtils'; import type { - ParseFieldError, ParseDictionaryData, ParseDictionaryFailure, ParseDictionaryResult, + ParseFieldError, ParseRecordResult, - ParseSchemaResult, ParseSchemaError, + ParseSchemaResult, } from './ParseValuesResult'; import { matchCodeListFormatting } from './matchCodeListFormatting'; diff --git a/packages/validation/src/utils/datasetUtils.ts b/packages/validation/src/utils/datasetUtils.ts index b757d45..1837c45 100644 --- a/packages/validation/src/utils/datasetUtils.ts +++ b/packages/validation/src/utils/datasetUtils.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { DataRecord } from 'dictionary'; +import { DataRecord } from '@overture-stack/lectern-dictionary'; /** * Returns a string representation of a record. The record is sorted by its properties so diff --git a/packages/validation/src/utils/hashDataRecord.ts b/packages/validation/src/utils/hashDataRecord.ts index 346027c..4c4dcd9 100644 --- a/packages/validation/src/utils/hashDataRecord.ts +++ b/packages/validation/src/utils/hashDataRecord.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import type { DataRecord, DataRecordValue } from 'dictionary'; +import type { DataRecord, DataRecordValue } from '@overture-stack/lectern-dictionary'; /** * Create a unique string out of an arbitrary array of data record values. This can be used diff --git a/packages/validation/src/utils/isValidValueType.ts b/packages/validation/src/utils/isValidValueType.ts index 4e00706..f7c8bc6 100644 --- a/packages/validation/src/utils/isValidValueType.ts +++ b/packages/validation/src/utils/isValidValueType.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import type { DataRecordValue, SchemaField } from 'dictionary'; +import type { DataRecordValue, SchemaField } from '@overture-stack/lectern-dictionary'; import { isBooleanArray, isInteger, isIntegerArray, isNumber, isNumberArray, isStringArray } from './typeUtils'; /** diff --git a/packages/validation/src/utils/isWithinRange.ts b/packages/validation/src/utils/isWithinRange.ts index 67e4947..6f1c367 100644 --- a/packages/validation/src/utils/isWithinRange.ts +++ b/packages/validation/src/utils/isWithinRange.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import type { RestrictionRange } from 'dictionary'; +import type { RestrictionRange } from '@overture-stack/lectern-dictionary'; /** * Test if a number is within a range. Returns true when the number is within the range. diff --git a/packages/validation/src/utils/rangeToText.ts b/packages/validation/src/utils/rangeToText.ts index afc8bcc..d447361 100644 --- a/packages/validation/src/utils/rangeToText.ts +++ b/packages/validation/src/utils/rangeToText.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { RestrictionRange } from 'dictionary'; +import { RestrictionRange } from '@overture-stack/lectern-dictionary'; /** * Convert a RestrictionRange object to a simple string representation. diff --git a/packages/validation/src/validateDictionary/collectSchemaReferenceData.ts b/packages/validation/src/validateDictionary/collectSchemaReferenceData.ts index 0ea6589..7cae7cc 100644 --- a/packages/validation/src/validateDictionary/collectSchemaReferenceData.ts +++ b/packages/validation/src/validateDictionary/collectSchemaReferenceData.ts @@ -17,8 +17,14 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { isDefined } from 'common'; -import type { DataRecord, DataRecordValue, Dictionary, ForeignKeyRestriction, Schema } from 'dictionary'; +import { + TypeUtils, + type DataRecord, + type DataRecordValue, + type Dictionary, + type ForeignKeyRestriction, + type Schema, +} from '@overture-stack/lectern-dictionary'; /** * This type alias is for a structure used to collect then lookup values from schemas. It is helpful @@ -94,7 +100,7 @@ export const collectSchemaReferenceData = ( .map((schema) => schema.restrictions?.foreignKey ? { schema, foreignKeyRestriction: schema.restrictions?.foreignKey } : undefined, ) - .filter(isDefined); + .filter(TypeUtils.isDefined); const foreignKeyMappings = schemasWithForeignKeyRestrictions.flatMap( ({ foreignKeyRestriction: foreignKeyMappings }) => foreignKeyMappings, diff --git a/packages/validation/src/validateDictionary/testForeignKeyRestriction.ts b/packages/validation/src/validateDictionary/testForeignKeyRestriction.ts index 2755c13..bdcab4e 100644 --- a/packages/validation/src/validateDictionary/testForeignKeyRestriction.ts +++ b/packages/validation/src/validateDictionary/testForeignKeyRestriction.ts @@ -17,11 +17,10 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import type { DataRecord, ForeignKeyRestriction } from 'dictionary'; +import { TypeUtils, type DataRecord, type ForeignKeyRestriction } from '@overture-stack/lectern-dictionary'; +import { invalid, valid, type TestResult } from '../types'; import type { SchemaDataReference } from './collectSchemaReferenceData'; import type { DictionaryValidationErrorRecordForeignKey } from './DictionaryValidationError'; -import { invalid, valid, type TestResult } from '../types'; -import { isDefined } from 'common'; /** * Test foreignKey restrictions on a single DataRecord, checked against a pre-calculated `foreignSchemaRefereceData` with the values @@ -70,7 +69,7 @@ export const testForeignKeyRestriction = ( fieldValue: localValue, }; }) - .filter(isDefined); + .filter(TypeUtils.isDefined); return foreignKeyErrors; }); diff --git a/packages/validation/src/validateDictionary/testUnrecognizedSchema.ts b/packages/validation/src/validateDictionary/testUnrecognizedSchema.ts index bd02362..6891433 100644 --- a/packages/validation/src/validateDictionary/testUnrecognizedSchema.ts +++ b/packages/validation/src/validateDictionary/testUnrecognizedSchema.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import type { Dictionary } from 'dictionary'; +import type { Dictionary } from '@overture-stack/lectern-dictionary'; import { invalid, valid, type TestResult } from '../types'; import type { DictionaryValidationErrorUnrecognizedSchema } from './DictionaryValidationError'; diff --git a/packages/validation/src/validateDictionary/validateDictionary.ts b/packages/validation/src/validateDictionary/validateDictionary.ts index 81d2831..e97ae7d 100644 --- a/packages/validation/src/validateDictionary/validateDictionary.ts +++ b/packages/validation/src/validateDictionary/validateDictionary.ts @@ -17,15 +17,14 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { isDefined } from 'common'; -import { DataRecord, Dictionary } from 'dictionary'; +import { DataRecord, Dictionary, TypeUtils } from '@overture-stack/lectern-dictionary'; import { invalid, valid, type TestResult } from '../types'; import { validateSchema, type SchemaRecordError } from '../validateSchema'; import { collectSchemaReferenceData } from './collectSchemaReferenceData'; import type { DictionaryValidationError, - DictionaryValidationRecordErrorDetails, DictionaryValidationErrorRecordForeignKey, + DictionaryValidationRecordErrorDetails, } from './DictionaryValidationError'; import { testForeignKeyRestriction } from './testForeignKeyRestriction'; import { testUnrecognizedSchema } from './testUnrecognizedSchema'; @@ -88,7 +87,7 @@ export const validateDictionary = ( recordErrors: foreignKeyTestResult.details, }; }) - .filter(isDefined) + .filter(TypeUtils.isDefined) : []; const combinedErrors = mergeSchemaRecordValidationErrors( schemaValidationResult.valid ? [] : schemaValidationResult.details, @@ -99,7 +98,7 @@ export const validateDictionary = ( ? { reason: 'INVALID_RECORDS', schemaName: schema.name, invalidRecords: combinedErrors } : undefined; }) - .filter(isDefined); + .filter(TypeUtils.isDefined); const collectedResults: DictionaryValidationError[] = [...unrecognizedSchemaErrors, ...recognizedSchemaErrors]; diff --git a/packages/validation/src/validateField/FieldRestrictionRule.ts b/packages/validation/src/validateField/FieldRestrictionRule.ts index 31b1acc..41117cb 100644 --- a/packages/validation/src/validateField/FieldRestrictionRule.ts +++ b/packages/validation/src/validateField/FieldRestrictionRule.ts @@ -23,7 +23,7 @@ import type { RestrictionRange, RestrictionRegex, RestrictionScript, -} from 'dictionary'; +} from '@overture-stack/lectern-dictionary'; export type FieldRestrictionRuleCodeList = { type: typeof FieldRestrictionTypes.codeList; diff --git a/packages/validation/src/validateField/FieldRestrictionTest.ts b/packages/validation/src/validateField/FieldRestrictionTest.ts index 48a4bc4..743cb16 100644 --- a/packages/validation/src/validateField/FieldRestrictionTest.ts +++ b/packages/validation/src/validateField/FieldRestrictionTest.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import type { ArrayDataValue, DataRecordValue, SingleDataValue } from 'dictionary'; +import type { ArrayDataValue, DataRecordValue, SingleDataValue } from '@overture-stack/lectern-dictionary'; import type { TestResult } from '../types'; export type RestrictionTestInvalidArrayItem = { diff --git a/packages/validation/src/validateField/FieldValidationError.ts b/packages/validation/src/validateField/FieldValidationError.ts index c1c8328..bc88653 100644 --- a/packages/validation/src/validateField/FieldValidationError.ts +++ b/packages/validation/src/validateField/FieldValidationError.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import type { SchemaFieldValueType } from 'dictionary'; +import type { SchemaFieldValueType } from '@overture-stack/lectern-dictionary'; import type { FieldRestrictionRule } from '../validateField/FieldRestrictionRule'; import type { RestrictionTestInvalidInfo } from '../validateField/FieldRestrictionTest'; diff --git a/packages/validation/src/validateField/resolveFieldRestrictions.ts b/packages/validation/src/validateField/resolveFieldRestrictions.ts index b8b4a54..1f6952e 100644 --- a/packages/validation/src/validateField/resolveFieldRestrictions.ts +++ b/packages/validation/src/validateField/resolveFieldRestrictions.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import type { DataRecord, DataRecordValue, SchemaField } from 'dictionary'; +import type { DataRecord, DataRecordValue, SchemaField } from '@overture-stack/lectern-dictionary'; import type { FieldRestrictionRule } from './FieldRestrictionRule'; /** diff --git a/packages/validation/src/validateField/restrictions/createFieldRestrictionTestForArrays.ts b/packages/validation/src/validateField/restrictions/createFieldRestrictionTestForArrays.ts index a2a7e87..e09e1bc 100644 --- a/packages/validation/src/validateField/restrictions/createFieldRestrictionTestForArrays.ts +++ b/packages/validation/src/validateField/restrictions/createFieldRestrictionTestForArrays.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { isDefined } from 'common'; +import { TypeUtils } from '@overture-stack/lectern-dictionary'; import { invalid, valid } from '../../types'; import type { FieldRestrictionArrayTestFunction, @@ -41,7 +41,7 @@ export const createFieldRestrictionTestForArrays = (rule, values) => { const invalidItems = values .map((value, position) => (test(rule, value).valid ? undefined : { position, value })) - .filter(isDefined); + .filter(TypeUtils.isDefined); if (invalidItems.length) { const message = typeof errorMessage === 'function' ? errorMessage(rule) : errorMessage; diff --git a/packages/validation/src/validateField/restrictions/testCodeList.ts b/packages/validation/src/validateField/restrictions/testCodeList.ts index d789bba..4d233a4 100644 --- a/packages/validation/src/validateField/restrictions/testCodeList.ts +++ b/packages/validation/src/validateField/restrictions/testCodeList.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import type { RestrictionCodeList } from 'dictionary'; +import type { RestrictionCodeList } from '@overture-stack/lectern-dictionary'; import { invalid, valid } from '../../types/testResult'; import type { FieldRestrictionSingleValueTestFunction, FieldRestrictionTestFunction } from '../FieldRestrictionTest'; import { createFieldRestrictionTestForArrays } from './createFieldRestrictionTestForArrays'; diff --git a/packages/validation/src/validateField/restrictions/testRange.ts b/packages/validation/src/validateField/restrictions/testRange.ts index 1a7eaf6..5f88358 100644 --- a/packages/validation/src/validateField/restrictions/testRange.ts +++ b/packages/validation/src/validateField/restrictions/testRange.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { RestrictionRange } from 'dictionary'; +import { RestrictionRange } from '@overture-stack/lectern-dictionary'; import { invalid, valid } from '../../types/testResult'; import { isWithinRange } from '../../utils/isWithinRange'; import type { FieldRestrictionSingleValueTestFunction, FieldRestrictionTestFunction } from '../FieldRestrictionTest'; diff --git a/packages/validation/src/validateField/restrictions/testRegex.ts b/packages/validation/src/validateField/restrictions/testRegex.ts index 20e3f05..4681b22 100644 --- a/packages/validation/src/validateField/restrictions/testRegex.ts +++ b/packages/validation/src/validateField/restrictions/testRegex.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { type RestrictionRegex } from 'dictionary'; +import { type RestrictionRegex } from '@overture-stack/lectern-dictionary'; import { invalid, valid } from '../../types/testResult'; import type { FieldRestrictionSingleValueTestFunction, FieldRestrictionTestFunction } from '../FieldRestrictionTest'; import { createFieldRestrictionTestForArrays } from './createFieldRestrictionTestForArrays'; diff --git a/packages/validation/src/validateField/restrictions/testRequired.ts b/packages/validation/src/validateField/restrictions/testRequired.ts index b4556c5..63c5157 100644 --- a/packages/validation/src/validateField/restrictions/testRequired.ts +++ b/packages/validation/src/validateField/restrictions/testRequired.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { type ArrayDataValue } from 'dictionary'; +import { type ArrayDataValue } from '@overture-stack/lectern-dictionary'; import { invalid, valid, type TestResult } from '../../types/testResult'; import type { FieldRestrictionSingleValueTestFunction, diff --git a/packages/validation/src/validateField/validateField.ts b/packages/validation/src/validateField/validateField.ts index 3929de8..11823f4 100644 --- a/packages/validation/src/validateField/validateField.ts +++ b/packages/validation/src/validateField/validateField.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import type { DataRecord, DataRecordValue, SchemaField } from 'dictionary'; +import type { DataRecord, DataRecordValue, SchemaField } from '@overture-stack/lectern-dictionary'; import { invalid, valid, type TestResult } from '../types'; import { isValidValueType } from '../utils/isValidValueType'; import type { FieldRestrictionRule } from './FieldRestrictionRule'; diff --git a/packages/validation/src/validateRecord/RecordValidationError.ts b/packages/validation/src/validateRecord/RecordValidationError.ts index ff6d409..c3e4923 100644 --- a/packages/validation/src/validateRecord/RecordValidationError.ts +++ b/packages/validation/src/validateRecord/RecordValidationError.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import type { DataRecordValue } from 'dictionary'; +import type { DataRecordValue } from '@overture-stack/lectern-dictionary'; import type { FieldValidationErrorRestrictions, FieldValidationErrorValueType, diff --git a/packages/validation/src/validateRecord/validateRecord.ts b/packages/validation/src/validateRecord/validateRecord.ts index cd29d87..57577ef 100644 --- a/packages/validation/src/validateRecord/validateRecord.ts +++ b/packages/validation/src/validateRecord/validateRecord.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import type { DataRecord, Schema } from 'dictionary'; +import type { DataRecord, Schema } from '@overture-stack/lectern-dictionary'; import { validateField } from '../validateField/validateField'; import { invalid, valid, type TestResult } from '../types'; import type { diff --git a/packages/validation/src/validateSchema/SchemaValidationError.ts b/packages/validation/src/validateSchema/SchemaValidationError.ts index fa556df..7cdb9b0 100644 --- a/packages/validation/src/validateSchema/SchemaValidationError.ts +++ b/packages/validation/src/validateSchema/SchemaValidationError.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import type { DataRecord } from 'dictionary'; +import type { DataRecord } from '@overture-stack/lectern-dictionary'; import type { RecordValidationError, FieldDetails } from '../validateRecord'; export type SchemaValidationRecordErrorUniqueKey = { diff --git a/packages/validation/src/validateSchema/restrictions/generateDataSetHashMap.ts b/packages/validation/src/validateSchema/restrictions/generateDataSetHashMap.ts index 990da9f..fe596e3 100644 --- a/packages/validation/src/validateSchema/restrictions/generateDataSetHashMap.ts +++ b/packages/validation/src/validateSchema/restrictions/generateDataSetHashMap.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import type { DataRecord } from 'dictionary'; +import type { DataRecord } from '@overture-stack/lectern-dictionary'; import { hashDataRecord } from '../../utils/hashDataRecord'; import { getUniqueKeyValues } from './uniqueKey/getUniqueKeyValues'; diff --git a/packages/validation/src/validateSchema/restrictions/uniqueField/testUniqueFieldRestriction.ts b/packages/validation/src/validateSchema/restrictions/uniqueField/testUniqueFieldRestriction.ts index 32189b1..f466710 100644 --- a/packages/validation/src/validateSchema/restrictions/uniqueField/testUniqueFieldRestriction.ts +++ b/packages/validation/src/validateSchema/restrictions/uniqueField/testUniqueFieldRestriction.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import type { DataRecordValue } from 'dictionary'; +import type { DataRecordValue } from '@overture-stack/lectern-dictionary'; import { invalid, valid, type TestResult } from '../../../types'; import { hashDataRecord } from '../../../utils/hashDataRecord'; import type { SchemaValidationRecordErrorUnique } from '../../SchemaValidationError'; diff --git a/packages/validation/src/validateSchema/restrictions/uniqueKey/getUniqueKeyValues.ts b/packages/validation/src/validateSchema/restrictions/uniqueKey/getUniqueKeyValues.ts index 1d16dea..49a8538 100644 --- a/packages/validation/src/validateSchema/restrictions/uniqueKey/getUniqueKeyValues.ts +++ b/packages/validation/src/validateSchema/restrictions/uniqueKey/getUniqueKeyValues.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import type { DataRecord } from 'dictionary'; +import type { DataRecord } from '@overture-stack/lectern-dictionary'; /** * Extract from a data record an object with only the unique key field values. * @param record diff --git a/packages/validation/src/validateSchema/restrictions/uniqueKey/testUniqueKey.ts b/packages/validation/src/validateSchema/restrictions/uniqueKey/testUniqueKey.ts index 87c467a..8ab2260 100644 --- a/packages/validation/src/validateSchema/restrictions/uniqueKey/testUniqueKey.ts +++ b/packages/validation/src/validateSchema/restrictions/uniqueKey/testUniqueKey.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import type { DataRecord, Schema, SchemaField } from 'dictionary'; +import type { DataRecord, Schema, SchemaField } from '@overture-stack/lectern-dictionary'; import type { SchemaValidationRecordErrorUniqueKey } from '../../SchemaValidationError'; import { invalid, valid, type TestResult } from '../../../types'; import { hashDataRecord } from '../../../utils/hashDataRecord'; diff --git a/packages/validation/src/validateSchema/validateSchema.ts b/packages/validation/src/validateSchema/validateSchema.ts index 594b3e2..96a0831 100644 --- a/packages/validation/src/validateSchema/validateSchema.ts +++ b/packages/validation/src/validateSchema/validateSchema.ts @@ -17,14 +17,14 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import type { DataRecord, Schema } from 'dictionary'; +import type { DataRecord, Schema } from '@overture-stack/lectern-dictionary'; +import { TypeUtils } from '@overture-stack/lectern-dictionary'; import { invalid, valid, type TestResult } from '../types'; -import type { SchemaValidationError, SchemaValidationRecordErrorDetails } from './SchemaValidationError'; import { validateRecord } from '../validateRecord'; import { generateDataSetHashMap } from './restrictions/generateDataSetHashMap'; -import { testUniqueKey } from './restrictions/uniqueKey/testUniqueKey'; -import { isDefined } from 'common'; import { testUniqueFieldRestriction } from './restrictions/uniqueField/testUniqueFieldRestriction'; +import { testUniqueKey } from './restrictions/uniqueKey/testUniqueKey'; +import type { SchemaValidationError, SchemaValidationRecordErrorDetails } from './SchemaValidationError'; /** * Validate a data set using a Lectern Schema. The data to validate is an array of DataRecords that contains all @@ -82,6 +82,6 @@ export const validateSchema = (records: Array, schema: Schema): Test } return recordErrors.length ? { recordIndex, recordErrors } : undefined; }) - .filter(isDefined); + .filter(TypeUtils.isDefined); return schemaValidationErrors.length ? invalid(schemaValidationErrors) : valid(); }; diff --git a/packages/validation/test/fixtures/dictionaries/dictionaryFourSchemas.ts b/packages/validation/test/fixtures/dictionaries/dictionaryFourSchemas.ts index 991c715..2b114b2 100644 --- a/packages/validation/test/fixtures/dictionaries/dictionaryFourSchemas.ts +++ b/packages/validation/test/fixtures/dictionaries/dictionaryFourSchemas.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { Dictionary } from 'dictionary'; +import { Dictionary } from '@overture-stack/lectern-dictionary'; import { validateFixture } from '../../testUtils/validateFixture'; import { schemaSingleString } from '../schema/schemaSingleString'; import { schemaAllDataTypes } from '../schema/schemaAllDataTypes'; diff --git a/packages/validation/test/fixtures/dictionaries/dictionaryMultipleSchemasNoRestrictions.ts b/packages/validation/test/fixtures/dictionaries/dictionaryMultipleSchemasNoRestrictions.ts index 662e1e0..d785471 100644 --- a/packages/validation/test/fixtures/dictionaries/dictionaryMultipleSchemasNoRestrictions.ts +++ b/packages/validation/test/fixtures/dictionaries/dictionaryMultipleSchemasNoRestrictions.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { Dictionary } from 'dictionary'; +import { Dictionary } from '@overture-stack/lectern-dictionary'; import { validateFixture } from '../../testUtils/validateFixture'; import { schemaSingleString } from '../schema/schemaSingleString'; import { schemaAllDataTypes } from '../schema/schemaAllDataTypes'; diff --git a/packages/validation/test/fixtures/dictionaries/dictionarySingleSchemaNoRestrictions.ts b/packages/validation/test/fixtures/dictionaries/dictionarySingleSchemaNoRestrictions.ts index b933953..a7bf760 100644 --- a/packages/validation/test/fixtures/dictionaries/dictionarySingleSchemaNoRestrictions.ts +++ b/packages/validation/test/fixtures/dictionaries/dictionarySingleSchemaNoRestrictions.ts @@ -1,4 +1,4 @@ -import { Dictionary } from 'dictionary'; +import { Dictionary } from '@overture-stack/lectern-dictionary'; import { validateFixture } from '../../testUtils/validateFixture'; import { schemaSingleString } from '../schema/schemaSingleString'; diff --git a/packages/validation/test/fixtures/dictionaries/dictionarySingleSchemaRequiredRestrictions.ts b/packages/validation/test/fixtures/dictionaries/dictionarySingleSchemaRequiredRestrictions.ts index 9b25ddc..e76e01a 100644 --- a/packages/validation/test/fixtures/dictionaries/dictionarySingleSchemaRequiredRestrictions.ts +++ b/packages/validation/test/fixtures/dictionaries/dictionarySingleSchemaRequiredRestrictions.ts @@ -1,4 +1,4 @@ -import { Dictionary } from 'dictionary'; +import { Dictionary } from '@overture-stack/lectern-dictionary'; import { validateFixture } from '../../testUtils/validateFixture'; import { schemaAllDataTypesRequired } from '../schema/schemaAllDataTypesRequired'; diff --git a/packages/validation/test/fixtures/dictionaries/dictionarySingleSchemaUniqueKeyRestriction.ts b/packages/validation/test/fixtures/dictionaries/dictionarySingleSchemaUniqueKeyRestriction.ts index 094ee02..40913dd 100644 --- a/packages/validation/test/fixtures/dictionaries/dictionarySingleSchemaUniqueKeyRestriction.ts +++ b/packages/validation/test/fixtures/dictionaries/dictionarySingleSchemaUniqueKeyRestriction.ts @@ -1,4 +1,4 @@ -import { Dictionary } from 'dictionary'; +import { Dictionary } from '@overture-stack/lectern-dictionary'; import { validateFixture } from '../../testUtils/validateFixture'; import { schemaUniqueKey } from '../schema/schemaUniqueKey'; diff --git a/packages/validation/test/fixtures/dictionaries/foreignKey/dictionaryForeignKeyMultiple.ts b/packages/validation/test/fixtures/dictionaries/foreignKey/dictionaryForeignKeyMultiple.ts index 7ecdd04..9dabdb0 100644 --- a/packages/validation/test/fixtures/dictionaries/foreignKey/dictionaryForeignKeyMultiple.ts +++ b/packages/validation/test/fixtures/dictionaries/foreignKey/dictionaryForeignKeyMultiple.ts @@ -1,4 +1,4 @@ -import { Dictionary, Schema } from 'dictionary'; +import { Dictionary, Schema } from '@overture-stack/lectern-dictionary'; import { validateFixture } from '../../../testUtils/validateFixture'; import { schemaAllDataTypes } from '../../schema/schemaAllDataTypes'; import { schemaSingleString } from '../../schema/schemaSingleString'; diff --git a/packages/validation/test/fixtures/dictionaries/foreignKey/dictionaryForeignKeySimple.ts b/packages/validation/test/fixtures/dictionaries/foreignKey/dictionaryForeignKeySimple.ts index 558e9e5..57ab5f5 100644 --- a/packages/validation/test/fixtures/dictionaries/foreignKey/dictionaryForeignKeySimple.ts +++ b/packages/validation/test/fixtures/dictionaries/foreignKey/dictionaryForeignKeySimple.ts @@ -1,4 +1,4 @@ -import { Dictionary, Schema } from 'dictionary'; +import { Dictionary, Schema } from '@overture-stack/lectern-dictionary'; import { validateFixture } from '../../../testUtils/validateFixture'; import { schemaAllDataTypes } from '../../schema/schemaAllDataTypes'; diff --git a/packages/validation/test/fixtures/fields/multipleRestrictions/fieldStringManyRestrictions.ts b/packages/validation/test/fixtures/fields/multipleRestrictions/fieldStringManyRestrictions.ts index f057da8..789858a 100644 --- a/packages/validation/test/fixtures/fields/multipleRestrictions/fieldStringManyRestrictions.ts +++ b/packages/validation/test/fixtures/fields/multipleRestrictions/fieldStringManyRestrictions.ts @@ -1,4 +1,4 @@ -import type { SchemaStringField } from 'dictionary'; +import type { SchemaStringField } from '@overture-stack/lectern-dictionary'; import { regexYearMonthDay } from '../../restrictions/regexFixtures'; export const fieldStringManyRestrictions = { diff --git a/packages/validation/test/fixtures/fields/noRestrictions/fieldBooleanNoRestriction.ts b/packages/validation/test/fixtures/fields/noRestrictions/fieldBooleanNoRestriction.ts index 825a75a..db3bdfd 100644 --- a/packages/validation/test/fixtures/fields/noRestrictions/fieldBooleanNoRestriction.ts +++ b/packages/validation/test/fixtures/fields/noRestrictions/fieldBooleanNoRestriction.ts @@ -1,4 +1,4 @@ -import type { SchemaBooleanField } from 'dictionary'; +import type { SchemaBooleanField } from '@overture-stack/lectern-dictionary'; export const fieldBooleanNoRestriction = { name: 'any-boolean', diff --git a/packages/validation/test/fixtures/fields/noRestrictions/fieldIntegerNoRestriction.ts b/packages/validation/test/fixtures/fields/noRestrictions/fieldIntegerNoRestriction.ts index 6c4a66b..3a6cfeb 100644 --- a/packages/validation/test/fixtures/fields/noRestrictions/fieldIntegerNoRestriction.ts +++ b/packages/validation/test/fixtures/fields/noRestrictions/fieldIntegerNoRestriction.ts @@ -1,4 +1,4 @@ -import type { SchemaIntegerField } from 'dictionary'; +import type { SchemaIntegerField } from '@overture-stack/lectern-dictionary'; export const fieldIntegerNoRestriction = { name: 'any-integer', diff --git a/packages/validation/test/fixtures/fields/noRestrictions/fieldNumberNoRestriction.ts b/packages/validation/test/fixtures/fields/noRestrictions/fieldNumberNoRestriction.ts index 824e585..c2f7c06 100644 --- a/packages/validation/test/fixtures/fields/noRestrictions/fieldNumberNoRestriction.ts +++ b/packages/validation/test/fixtures/fields/noRestrictions/fieldNumberNoRestriction.ts @@ -1,4 +1,4 @@ -import type { SchemaNumberField } from 'dictionary'; +import type { SchemaNumberField } from '@overture-stack/lectern-dictionary'; export const fieldNumberNoRestriction = { name: 'any-number', diff --git a/packages/validation/test/fixtures/fields/noRestrictions/fieldStringArrayNoRestriction.ts b/packages/validation/test/fixtures/fields/noRestrictions/fieldStringArrayNoRestriction.ts index 1d7098e..b1b954d 100644 --- a/packages/validation/test/fixtures/fields/noRestrictions/fieldStringArrayNoRestriction.ts +++ b/packages/validation/test/fixtures/fields/noRestrictions/fieldStringArrayNoRestriction.ts @@ -1,4 +1,4 @@ -import type { SchemaStringField } from 'dictionary'; +import type { SchemaStringField } from '@overture-stack/lectern-dictionary'; export const fieldStringArrayNoRestriction = { name: 'any-string-array', diff --git a/packages/validation/test/fixtures/fields/noRestrictions/fieldStringNoRestriction.ts b/packages/validation/test/fixtures/fields/noRestrictions/fieldStringNoRestriction.ts index 96cc975..669ff2f 100644 --- a/packages/validation/test/fixtures/fields/noRestrictions/fieldStringNoRestriction.ts +++ b/packages/validation/test/fixtures/fields/noRestrictions/fieldStringNoRestriction.ts @@ -1,4 +1,4 @@ -import type { SchemaStringField } from 'dictionary'; +import type { SchemaStringField } from '@overture-stack/lectern-dictionary'; export const fieldStringNoRestriction = { name: 'any-string', diff --git a/packages/validation/test/fixtures/fields/schemaRestrictions/fieldStringUnique.ts b/packages/validation/test/fixtures/fields/schemaRestrictions/fieldStringUnique.ts index b8c6d82..364b358 100644 --- a/packages/validation/test/fixtures/fields/schemaRestrictions/fieldStringUnique.ts +++ b/packages/validation/test/fixtures/fields/schemaRestrictions/fieldStringUnique.ts @@ -1,4 +1,4 @@ -import type { SchemaStringField } from 'dictionary'; +import type { SchemaStringField } from '@overture-stack/lectern-dictionary'; export const fieldStringUnique = { name: 'unique-string', diff --git a/packages/validation/test/fixtures/fields/schemaRestrictions/fieldStringUniqueArray.ts b/packages/validation/test/fixtures/fields/schemaRestrictions/fieldStringUniqueArray.ts index c652d82..3b486aa 100644 --- a/packages/validation/test/fixtures/fields/schemaRestrictions/fieldStringUniqueArray.ts +++ b/packages/validation/test/fixtures/fields/schemaRestrictions/fieldStringUniqueArray.ts @@ -1,4 +1,4 @@ -import type { SchemaStringField } from 'dictionary'; +import type { SchemaStringField } from '@overture-stack/lectern-dictionary'; export const fieldStringUniqueArray = { name: 'unique-string-array', diff --git a/packages/validation/test/fixtures/fields/simpleRestrictions/boolean/fieldBooleanArrayRequired.ts b/packages/validation/test/fixtures/fields/simpleRestrictions/boolean/fieldBooleanArrayRequired.ts index b85e03e..0fa999f 100644 --- a/packages/validation/test/fixtures/fields/simpleRestrictions/boolean/fieldBooleanArrayRequired.ts +++ b/packages/validation/test/fixtures/fields/simpleRestrictions/boolean/fieldBooleanArrayRequired.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import type { SchemaBooleanField } from 'dictionary'; +import type { SchemaBooleanField } from '@overture-stack/lectern-dictionary'; export const fieldBooleanArrayRequired = { name: 'boolean-array-required', diff --git a/packages/validation/test/fixtures/fields/simpleRestrictions/boolean/fieldBooleanRequired.ts b/packages/validation/test/fixtures/fields/simpleRestrictions/boolean/fieldBooleanRequired.ts index 88f8792..eada2ff 100644 --- a/packages/validation/test/fixtures/fields/simpleRestrictions/boolean/fieldBooleanRequired.ts +++ b/packages/validation/test/fixtures/fields/simpleRestrictions/boolean/fieldBooleanRequired.ts @@ -1,4 +1,4 @@ -import type { SchemaBooleanField } from 'dictionary'; +import type { SchemaBooleanField } from '@overture-stack/lectern-dictionary'; export const fieldBooleanRequired = { name: 'boolean-required', diff --git a/packages/validation/test/fixtures/fields/simpleRestrictions/integer/fieldIntegerArrayRequired.ts b/packages/validation/test/fixtures/fields/simpleRestrictions/integer/fieldIntegerArrayRequired.ts index c08c82f..a6cfe4e 100644 --- a/packages/validation/test/fixtures/fields/simpleRestrictions/integer/fieldIntegerArrayRequired.ts +++ b/packages/validation/test/fixtures/fields/simpleRestrictions/integer/fieldIntegerArrayRequired.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import type { SchemaIntegerField } from 'dictionary'; +import type { SchemaIntegerField } from '@overture-stack/lectern-dictionary'; export const fieldIntegerArrayRequired = { name: 'integer-array-required', diff --git a/packages/validation/test/fixtures/fields/simpleRestrictions/integer/fieldIntegerRequired.ts b/packages/validation/test/fixtures/fields/simpleRestrictions/integer/fieldIntegerRequired.ts index 8459948..13b345a 100644 --- a/packages/validation/test/fixtures/fields/simpleRestrictions/integer/fieldIntegerRequired.ts +++ b/packages/validation/test/fixtures/fields/simpleRestrictions/integer/fieldIntegerRequired.ts @@ -1,4 +1,4 @@ -import type { SchemaIntegerField } from 'dictionary'; +import type { SchemaIntegerField } from '@overture-stack/lectern-dictionary'; export const fieldIntegerRequired = { name: 'integer-required', diff --git a/packages/validation/test/fixtures/fields/simpleRestrictions/number/fieldNumberArrayCodeList.ts b/packages/validation/test/fixtures/fields/simpleRestrictions/number/fieldNumberArrayCodeList.ts index a1edbdc..97124e7 100644 --- a/packages/validation/test/fixtures/fields/simpleRestrictions/number/fieldNumberArrayCodeList.ts +++ b/packages/validation/test/fixtures/fields/simpleRestrictions/number/fieldNumberArrayCodeList.ts @@ -1,4 +1,4 @@ -import type { SchemaNumberField } from 'dictionary'; +import type { SchemaNumberField } from '@overture-stack/lectern-dictionary'; import { codeListNumber } from '../../../restrictions/codeListsFixtures'; export const fieldNumberArrayCodeList = { diff --git a/packages/validation/test/fixtures/fields/simpleRestrictions/number/fieldNumberRange.ts b/packages/validation/test/fixtures/fields/simpleRestrictions/number/fieldNumberRange.ts index a5e65cd..6797eee 100644 --- a/packages/validation/test/fixtures/fields/simpleRestrictions/number/fieldNumberRange.ts +++ b/packages/validation/test/fixtures/fields/simpleRestrictions/number/fieldNumberRange.ts @@ -1,4 +1,4 @@ -import type { SchemaNumberField } from 'dictionary'; +import type { SchemaNumberField } from '@overture-stack/lectern-dictionary'; import { rangePercent } from '../../../restrictions/rangeFixtures'; export const fieldNumberRange = { diff --git a/packages/validation/test/fixtures/fields/simpleRestrictions/number/fieldNumberRequired.ts b/packages/validation/test/fixtures/fields/simpleRestrictions/number/fieldNumberRequired.ts index ea4b052..aaedc31 100644 --- a/packages/validation/test/fixtures/fields/simpleRestrictions/number/fieldNumberRequired.ts +++ b/packages/validation/test/fixtures/fields/simpleRestrictions/number/fieldNumberRequired.ts @@ -1,4 +1,4 @@ -import type { SchemaNumberField } from 'dictionary'; +import type { SchemaNumberField } from '@overture-stack/lectern-dictionary'; export const fieldNumberRequired = { name: 'number-required', diff --git a/packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringArrayCodeList.ts b/packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringArrayCodeList.ts index 48b04c7..04b9a1b 100644 --- a/packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringArrayCodeList.ts +++ b/packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringArrayCodeList.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import type { SchemaStringField } from 'dictionary'; +import type { SchemaStringField } from '@overture-stack/lectern-dictionary'; import { codeListString } from '../../../restrictions/codeListsFixtures'; export const fieldStringArrayCodeList = { diff --git a/packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringArrayRequired.ts b/packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringArrayRequired.ts index 9873a58..0a59f34 100644 --- a/packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringArrayRequired.ts +++ b/packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringArrayRequired.ts @@ -1,4 +1,4 @@ -import type { SchemaStringField } from 'dictionary'; +import type { SchemaStringField } from '@overture-stack/lectern-dictionary'; export const fieldStringArrayRequired = { name: 'string-array-required', diff --git a/packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringCodeList.ts b/packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringCodeList.ts index 0839f9c..859517e 100644 --- a/packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringCodeList.ts +++ b/packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringCodeList.ts @@ -1,4 +1,4 @@ -import type { SchemaStringField } from 'dictionary'; +import type { SchemaStringField } from '@overture-stack/lectern-dictionary'; import { codeListString } from '../../../restrictions/codeListsFixtures'; export const fieldStringCodeList = { diff --git a/packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringRegex.ts b/packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringRegex.ts index 52cdc90..ecc25fb 100644 --- a/packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringRegex.ts +++ b/packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringRegex.ts @@ -1,4 +1,4 @@ -import type { SchemaStringField } from 'dictionary'; +import type { SchemaStringField } from '@overture-stack/lectern-dictionary'; import { regexYearMonthDay } from '../../../restrictions/regexFixtures'; export const fieldStringRegex = { diff --git a/packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringRequired.ts b/packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringRequired.ts index 9b2d1ca..58b417c 100644 --- a/packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringRequired.ts +++ b/packages/validation/test/fixtures/fields/simpleRestrictions/string/fieldStringRequired.ts @@ -1,4 +1,4 @@ -import type { SchemaStringField } from 'dictionary'; +import type { SchemaStringField } from '@overture-stack/lectern-dictionary'; export const fieldStringRequired = { name: 'string-required', diff --git a/packages/validation/test/fixtures/restrictions/codeListsFixtures.ts b/packages/validation/test/fixtures/restrictions/codeListsFixtures.ts index 36cd8ed..654856b 100644 --- a/packages/validation/test/fixtures/restrictions/codeListsFixtures.ts +++ b/packages/validation/test/fixtures/restrictions/codeListsFixtures.ts @@ -1,4 +1,8 @@ -import type { RestrictionCodeListInteger, RestrictionCodeListNumber, RestrictionCodeListString } from 'dictionary'; +import type { + RestrictionCodeListInteger, + RestrictionCodeListNumber, + RestrictionCodeListString, +} from '@overture-stack/lectern-dictionary'; export const codeListString: RestrictionCodeListString = ['Apple ', ' Banana', ' Carrot ', 'Donut']; // Food, extra whitespace on items in order to test that matching is being trimmed. export const codeListInteger: RestrictionCodeListInteger = [1, 1, 2, 3, 5, 8, 13, 24]; // Fibonacci diff --git a/packages/validation/test/fixtures/restrictions/rangeFixtures.ts b/packages/validation/test/fixtures/restrictions/rangeFixtures.ts index c90dc14..4876e3b 100644 --- a/packages/validation/test/fixtures/restrictions/rangeFixtures.ts +++ b/packages/validation/test/fixtures/restrictions/rangeFixtures.ts @@ -1,4 +1,4 @@ -import type { RestrictionRange } from 'dictionary'; +import type { RestrictionRange } from '@overture-stack/lectern-dictionary'; export const rangePercent: RestrictionRange = { max: 100, diff --git a/packages/validation/test/fixtures/restrictions/regexFixtures.ts b/packages/validation/test/fixtures/restrictions/regexFixtures.ts index c896d34..61b9178 100644 --- a/packages/validation/test/fixtures/restrictions/regexFixtures.ts +++ b/packages/validation/test/fixtures/restrictions/regexFixtures.ts @@ -1,4 +1,4 @@ -import type { RestrictionRegex } from 'dictionary'; +import type { RestrictionRegex } from '@overture-stack/lectern-dictionary'; export const regexYearMonthDay: RestrictionRegex = '^([12]\\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12]\\d|3[01]))$'; // Example: "1999-01-31" export const regexMTGMana: RestrictionRegex = diff --git a/packages/validation/test/fixtures/schema/schemaAllDataMixedRestrictions.ts b/packages/validation/test/fixtures/schema/schemaAllDataMixedRestrictions.ts index 4f67646..731a06d 100644 --- a/packages/validation/test/fixtures/schema/schemaAllDataMixedRestrictions.ts +++ b/packages/validation/test/fixtures/schema/schemaAllDataMixedRestrictions.ts @@ -1,4 +1,4 @@ -import { Schema } from 'dictionary'; +import { Schema } from '@overture-stack/lectern-dictionary'; import { fieldStringManyRestrictions } from '../fields/multipleRestrictions/fieldStringManyRestrictions'; import { fieldBooleanNoRestriction } from '../fields/noRestrictions/fieldBooleanNoRestriction'; import { fieldIntegerRequired } from '../fields/simpleRestrictions/integer/fieldIntegerRequired'; diff --git a/packages/validation/test/fixtures/schema/schemaAllDataTypes.ts b/packages/validation/test/fixtures/schema/schemaAllDataTypes.ts index 98e3e73..51ca40b 100644 --- a/packages/validation/test/fixtures/schema/schemaAllDataTypes.ts +++ b/packages/validation/test/fixtures/schema/schemaAllDataTypes.ts @@ -1,4 +1,4 @@ -import { Schema } from 'dictionary'; +import { Schema } from '@overture-stack/lectern-dictionary'; import { fieldBooleanNoRestriction } from '../fields/noRestrictions/fieldBooleanNoRestriction'; import { fieldIntegerNoRestriction } from '../fields/noRestrictions/fieldIntegerNoRestriction'; import { fieldNumberNoRestriction } from '../fields/noRestrictions/fieldNumberNoRestriction'; diff --git a/packages/validation/test/fixtures/schema/schemaAllDataTypesRequired.ts b/packages/validation/test/fixtures/schema/schemaAllDataTypesRequired.ts index b29d08f..272cc94 100644 --- a/packages/validation/test/fixtures/schema/schemaAllDataTypesRequired.ts +++ b/packages/validation/test/fixtures/schema/schemaAllDataTypesRequired.ts @@ -1,4 +1,4 @@ -import { Schema } from 'dictionary'; +import { Schema } from '@overture-stack/lectern-dictionary'; import { fieldBooleanRequired } from '../fields/simpleRestrictions/boolean/fieldBooleanRequired'; import { fieldIntegerRequired } from '../fields/simpleRestrictions/integer/fieldIntegerRequired'; import { fieldNumberRequired } from '../fields/simpleRestrictions/number/fieldNumberRequired'; diff --git a/packages/validation/test/fixtures/schema/schemaSingleString.ts b/packages/validation/test/fixtures/schema/schemaSingleString.ts index 9f5a645..be90689 100644 --- a/packages/validation/test/fixtures/schema/schemaSingleString.ts +++ b/packages/validation/test/fixtures/schema/schemaSingleString.ts @@ -1,4 +1,4 @@ -import { Schema } from 'dictionary'; +import { Schema } from '@overture-stack/lectern-dictionary'; import { fieldStringNoRestriction } from '../fields/noRestrictions/fieldStringNoRestriction'; import { validateFixture } from '../../testUtils/validateFixture'; diff --git a/packages/validation/test/fixtures/schema/schemaSingleStringRequired.ts b/packages/validation/test/fixtures/schema/schemaSingleStringRequired.ts index 66ff645..b167f38 100644 --- a/packages/validation/test/fixtures/schema/schemaSingleStringRequired.ts +++ b/packages/validation/test/fixtures/schema/schemaSingleStringRequired.ts @@ -1,4 +1,4 @@ -import { Schema } from 'dictionary'; +import { Schema } from '@overture-stack/lectern-dictionary'; import { fieldStringRequired } from '../fields/simpleRestrictions/string/fieldStringRequired'; import { validateFixture } from '../../testUtils/validateFixture'; diff --git a/packages/validation/test/fixtures/schema/schemaUniqueKey.ts b/packages/validation/test/fixtures/schema/schemaUniqueKey.ts index d9185ad..aa2991d 100644 --- a/packages/validation/test/fixtures/schema/schemaUniqueKey.ts +++ b/packages/validation/test/fixtures/schema/schemaUniqueKey.ts @@ -1,4 +1,4 @@ -import { Schema } from 'dictionary'; +import { Schema } from '@overture-stack/lectern-dictionary'; import { validateFixture } from '../../testUtils/validateFixture'; import { fieldBooleanNoRestriction } from '../fields/noRestrictions/fieldBooleanNoRestriction'; import { fieldIntegerNoRestriction } from '../fields/noRestrictions/fieldIntegerNoRestriction'; diff --git a/packages/validation/test/fixtures/schema/schemaUniqueKeyWithArray.ts b/packages/validation/test/fixtures/schema/schemaUniqueKeyWithArray.ts index 6c84fa9..21c74e3 100644 --- a/packages/validation/test/fixtures/schema/schemaUniqueKeyWithArray.ts +++ b/packages/validation/test/fixtures/schema/schemaUniqueKeyWithArray.ts @@ -1,4 +1,4 @@ -import { Schema } from 'dictionary'; +import { Schema } from '@overture-stack/lectern-dictionary'; import { validateFixture } from '../../testUtils/validateFixture'; import { fieldBooleanNoRestriction } from '../fields/noRestrictions/fieldBooleanNoRestriction'; import { fieldIntegerNoRestriction } from '../fields/noRestrictions/fieldIntegerNoRestriction'; diff --git a/packages/validation/test/fixtures/schema/schemaUniqueString.ts b/packages/validation/test/fixtures/schema/schemaUniqueString.ts index 40aa14f..2ac95c5 100644 --- a/packages/validation/test/fixtures/schema/schemaUniqueString.ts +++ b/packages/validation/test/fixtures/schema/schemaUniqueString.ts @@ -1,4 +1,4 @@ -import { Schema } from 'dictionary'; +import { Schema } from '@overture-stack/lectern-dictionary'; import { fieldStringUnique } from '../fields/schemaRestrictions/fieldStringUnique'; import assert from 'node:assert'; import { validateFixture } from '../../testUtils/validateFixture'; diff --git a/packages/validation/test/fixtures/schema/schemaUniqueStringArray.ts b/packages/validation/test/fixtures/schema/schemaUniqueStringArray.ts index 002f98d..9f735a2 100644 --- a/packages/validation/test/fixtures/schema/schemaUniqueStringArray.ts +++ b/packages/validation/test/fixtures/schema/schemaUniqueStringArray.ts @@ -1,4 +1,4 @@ -import { Schema } from 'dictionary'; +import { Schema } from '@overture-stack/lectern-dictionary'; import { fieldStringUnique } from '../fields/schemaRestrictions/fieldStringUnique'; import assert from 'node:assert'; import { validateFixture } from '../../testUtils/validateFixture'; diff --git a/packages/validation/test/utils/rangeTest.spec.ts b/packages/validation/test/utils/rangeTest.spec.ts index 38a7973..b5ec0d6 100644 --- a/packages/validation/test/utils/rangeTest.spec.ts +++ b/packages/validation/test/utils/rangeTest.spec.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import type { RestrictionRange } from 'dictionary'; +import type { RestrictionRange } from '@overture-stack/lectern-dictionary'; import { isWithinRange } from '../../src/utils/isWithinRange'; import { expect } from 'chai'; diff --git a/packages/validation/test/validateField/validateField.spec.ts b/packages/validation/test/validateField/validateField.spec.ts index a08a2cd..ecaafa9 100644 --- a/packages/validation/test/validateField/validateField.spec.ts +++ b/packages/validation/test/validateField/validateField.spec.ts @@ -18,7 +18,7 @@ */ import { expect } from 'chai'; -import type { SchemaStringField } from 'dictionary'; +import type { SchemaStringField } from '@overture-stack/lectern-dictionary'; import assert from 'node:assert'; import { validateField } from '../../src'; import { fieldStringManyRestrictions } from '../fixtures/fields/multipleRestrictions/fieldStringManyRestrictions'; diff --git a/packages/validation/test/validateSchema/validateSchema.spec.ts b/packages/validation/test/validateSchema/validateSchema.spec.ts index 70789f1..0729d9b 100644 --- a/packages/validation/test/validateSchema/validateSchema.spec.ts +++ b/packages/validation/test/validateSchema/validateSchema.spec.ts @@ -1,5 +1,5 @@ import { expect } from 'chai'; -import type { DataRecord } from 'dictionary'; +import type { DataRecord } from '@overture-stack/lectern-dictionary'; import assert from 'node:assert'; import { validateSchema } from '../../src'; import { schemaSingleStringRequired } from '../fixtures/schema/schemaSingleStringRequired'; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0dc57af..e0f749e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -15,26 +15,26 @@ importers: specifier: ^7.1.8 version: 7.1.8 '@types/mocha': - specifier: ^10.0.6 - version: 10.0.6 + specifier: ^10.0.7 + version: 10.0.7 '@types/node': - specifier: ^20.14.7 - version: 20.14.7 + specifier: ^20.14.13 + version: 20.14.13 '@types/sinon': specifier: ^10.0.20 version: 10.0.20 chai: - specifier: ^4.4.1 - version: 4.4.1 + specifier: ^4.5.0 + version: 4.5.0 chai-as-promised: specifier: ^7.1.2 - version: 7.1.2(chai@4.4.1) + version: 7.1.2(chai@4.5.0) chai-http: specifier: ^4.4.0 version: 4.4.0 mocha: - specifier: ^10.4.0 - version: 10.4.0 + specifier: ^10.7.0 + version: 10.7.0 nx: specifier: ^16.10.0 version: 16.10.0 @@ -42,47 +42,44 @@ importers: specifier: ^15.1.0 version: 15.1.0 prettier: - specifier: ^3.3.2 - version: 3.3.2 + specifier: ^3.3.3 + version: 3.3.3 sinon: specifier: ^15.2.0 version: 15.2.0 ts-node: specifier: ^10.9.2 - version: 10.9.2(@types/node@20.14.7)(typescript@5.5.2) + version: 10.9.2(@types/node@20.14.13)(typescript@5.5.4) typescript: - specifier: ^5.5.2 - version: 5.5.2 + specifier: ^5.5.4 + version: 5.5.4 apps/server: dependencies: + '@overture-stack/lectern-dictionary': + specifier: workspace:^ + version: link:../../packages/dictionary ajv: - specifier: ^8.12.0 - version: 8.12.0 + specifier: ^8.17.1 + version: 8.17.1 axios: - specifier: ^1.4.0 - version: 1.4.0 + specifier: ^1.7.2 + version: 1.7.2 body-parser: specifier: ^1.20.2 version: 1.20.2 - common: - specifier: workspace:^ - version: link:../../packages/common - dictionary: - specifier: workspace:^ - version: link:../../packages/dictionary dotenv: - specifier: ^16.3.1 - version: 16.3.1 + specifier: ^16.4.5 + version: 16.4.5 errorhandler: specifier: ^1.5.1 version: 1.5.1 express: - specifier: ^4.18.2 - version: 4.18.2 + specifier: ^4.19.2 + version: 4.19.2 immer: - specifier: ^10.0.2 - version: 10.0.2 + specifier: ^10.1.1 + version: 10.1.1 jsonwebtoken: specifier: ^8.5.1 version: 8.5.1 @@ -90,11 +87,11 @@ importers: specifier: ^4.17.21 version: 4.17.21 memoizee: - specifier: ^0.4.15 - version: 0.4.15 + specifier: ^0.4.17 + version: 0.4.17 mongoose: - specifier: ^7.3.2 - version: 7.3.2 + specifier: ^7.8.0 + version: 7.8.0 ms: specifier: ^2.1.3 version: 2.1.3 @@ -103,38 +100,38 @@ importers: version: 0.9.22 swagger-ui-express: specifier: ^4.6.3 - version: 4.6.3(express@4.18.2) + version: 4.6.3(express@4.19.2) winston: - specifier: ^3.9.0 - version: 3.9.0 + specifier: ^3.13.1 + version: 3.13.1 zod: - specifier: ^3.21.4 - version: 3.21.4 + specifier: ^3.23.8 + version: 3.23.8 devDependencies: '@types/body-parser': - specifier: ^1.19.2 - version: 1.19.2 + specifier: ^1.19.5 + version: 1.19.5 '@types/errorhandler': specifier: 0.0.32 version: 0.0.32 '@types/express': - specifier: ^4.17.17 - version: 4.17.17 + specifier: ^4.17.21 + version: 4.17.21 '@types/jsonwebtoken': specifier: ^8.5.9 version: 8.5.9 '@types/lodash': - specifier: ^4.14.195 - version: 4.14.195 + specifier: ^4.17.7 + version: 4.17.7 '@types/memoizee': - specifier: ^0.4.8 - version: 0.4.8 + specifier: ^0.4.11 + version: 0.4.11 '@types/ms': - specifier: ^0.7.31 - version: 0.7.31 + specifier: ^0.7.34 + version: 0.7.34 '@types/superagent': - specifier: ^4.1.18 - version: 4.1.18 + specifier: ^4.1.24 + version: 4.1.24 '@types/swagger-ui-express': specifier: ^3.0.1 version: 3.0.1 @@ -148,29 +145,29 @@ importers: specifier: ^2.0.22 version: 2.0.22 prettier: - specifier: ^3 - version: 3.0.0 + specifier: ^3.3.3 + version: 3.3.3 testcontainers: specifier: ^1.3.1 version: 1.3.1 + typescript: + specifier: ^5.5.4 + version: 5.5.4 zod-to-json-schema: - specifier: ^3.21.3 - version: 3.21.3(zod@3.21.4) + specifier: ^3.23.2 + version: 3.23.2(zod@3.23.8) packages/client: dependencies: + '@overture-stack/lectern-dictionary': + specifier: workspace:^ + version: link:../dictionary '@overture-stack/lectern-validation': specifier: workspace:^ version: link:../validation cd: specifier: ^0.3.3 version: 0.3.3 - common: - specifier: workspace:^ - version: link:../common - dictionary: - specifier: workspace:^ - version: link:../dictionary lodash: specifier: ^4.17.21 version: 4.17.21 @@ -181,15 +178,15 @@ importers: specifier: ^2.1.0 version: 2.1.0 winston: - specifier: ^3.13.0 - version: 3.13.0 + specifier: ^3.13.1 + version: 3.13.1 devDependencies: '@types/chai': specifier: ^4.3.16 version: 4.3.16 '@types/lodash': - specifier: ^4.17.5 - version: 4.17.5 + specifier: ^4.17.7 + version: 4.17.7 '@types/mocha': specifier: ^8.2.3 version: 8.2.3 @@ -200,8 +197,8 @@ importers: specifier: ^2.6.11 version: 2.6.11 chai: - specifier: ^4.4.1 - version: 4.4.1 + specifier: ^4.5.0 + version: 4.5.0 husky: specifier: ^6.0.0 version: 6.0.0 @@ -215,79 +212,70 @@ importers: specifier: ^3.3.1 version: 3.3.1(prettier@2.8.8) rimraf: - specifier: ^5.0.7 - version: 5.0.7 + specifier: ^5.0.9 + version: 5.0.9 ts-node: specifier: ^9.1.1 - version: 9.1.1(typescript@5.5.2) + version: 9.1.1(typescript@5.5.4) tslint: specifier: ^6.1.3 - version: 6.1.3(typescript@5.5.2) + version: 6.1.3(typescript@5.5.4) typedoc: specifier: ^0.17.8 - version: 0.17.8(typescript@5.5.2) + version: 0.17.8(typescript@5.5.4) typescript: - specifier: ^5.5.2 - version: 5.5.2 - - packages/common: - devDependencies: - rimraf: - specifier: ^5.0.0 - version: 5.0.7 + specifier: ^5.5.4 + version: 5.5.4 packages/dictionary: dependencies: - common: - specifier: workspace:^ - version: link:../common immer: - specifier: ^10.0.2 - version: 10.0.2 + specifier: ^10.1.1 + version: 10.1.1 lodash: specifier: ^4.17.21 version: 4.17.21 zod: - specifier: ^3.21.4 - version: 3.21.4 + specifier: ^3.23.8 + version: 3.23.8 devDependencies: '@types/lodash': - specifier: ^4.14.195 - version: 4.14.195 + specifier: ^4.17.7 + version: 4.17.7 rimraf: - specifier: ^5.0.0 - version: 5.0.7 + specifier: ^5.0.9 + version: 5.0.9 packages/validation: dependencies: - common: - specifier: workspace:^ - version: link:../common - dictionary: + '@overture-stack/lectern-dictionary': specifier: workspace:^ version: link:../dictionary lodash: specifier: ^4.17.21 version: 4.17.21 zod: - specifier: ^3.21.4 - version: 3.21.4 + specifier: ^3.23.8 + version: 3.23.8 devDependencies: '@types/lodash': - specifier: ^4.14.195 - version: 4.14.195 + specifier: ^4.17.7 + version: 4.17.7 rimraf: - specifier: ^5.0.0 - version: 5.0.7 + specifier: ^5.0.9 + version: 5.0.9 - scripts/generate: + scripts: dependencies: - dictionary: + '@overture-stack/lectern-dictionary': specifier: workspace:^ - version: link:../../packages/dictionary + version: link:../packages/dictionary + zod: + specifier: ^3.23.8 + version: 3.23.8 zod-to-json-schema: - specifier: ^3.21.3 - version: 3.21.3(zod@3.21.4) + specifier: ^3.23.2 + version: 3.23.2(zod@3.23.8) packages: @@ -299,13 +287,6 @@ packages: '@jridgewell/trace-mapping': 0.3.25 dev: true - /@babel/code-frame@7.22.5: - resolution: {integrity: sha512-Xmwn266vad+6DAqEB2A6V/CcZVp62BbwVmcOJc2RPuwih1kw02TjQvWVWlcKGbBPd+8/0V5DEkOcizRGYsspYQ==} - engines: {node: '>=6.9.0'} - dependencies: - '@babel/highlight': 7.22.5 - dev: true - /@babel/code-frame@7.24.7: resolution: {integrity: sha512-BcYH1CVJBO9tvyIZ2jVeXgSIMvGZ2FDRvDdOIVQyuklNKSsx+eppDEBq/g47Ayw+RqNFE+URvOShmf+f/qwAlA==} engines: {node: '>=6.9.0'} @@ -314,27 +295,27 @@ packages: picocolors: 1.0.1 dev: true - /@babel/compat-data@7.24.7: - resolution: {integrity: sha512-qJzAIcv03PyaWqxRgO4mSU3lihncDT296vnyuE2O8uA4w3UHWI4S3hgeZd1L8W1Bft40w9JxJ2b412iDUFFRhw==} + /@babel/compat-data@7.25.2: + resolution: {integrity: sha512-bYcppcpKBvX4znYaPEeFau03bp89ShqNMLs+rmdptMw+heSZh9+z84d2YG+K7cYLbWwzdjtDoW/uqZmPjulClQ==} engines: {node: '>=6.9.0'} dev: true - /@babel/core@7.24.7: - resolution: {integrity: sha512-nykK+LEK86ahTkX/3TgauT0ikKoNCfKHEaZYTUVupJdTLzGNvrblu4u6fa7DhZONAltdf8e662t/abY8idrd/g==} + /@babel/core@7.25.2: + resolution: {integrity: sha512-BBt3opiCOxUr9euZ5/ro/Xv8/V7yJ5bjYMqG/C1YAo8MIKAnumZalCN+msbci3Pigy4lIQfPUpfMM27HMGaYEA==} engines: {node: '>=6.9.0'} dependencies: '@ampproject/remapping': 2.3.0 '@babel/code-frame': 7.24.7 - '@babel/generator': 7.24.7 - '@babel/helper-compilation-targets': 7.24.7 - '@babel/helper-module-transforms': 7.24.7(@babel/core@7.24.7) - '@babel/helpers': 7.24.7 - '@babel/parser': 7.24.7 - '@babel/template': 7.24.7 - '@babel/traverse': 7.24.7 - '@babel/types': 7.24.7 + '@babel/generator': 7.25.0 + '@babel/helper-compilation-targets': 7.25.2 + '@babel/helper-module-transforms': 7.25.2(@babel/core@7.25.2) + '@babel/helpers': 7.25.0 + '@babel/parser': 7.25.3 + '@babel/template': 7.25.0 + '@babel/traverse': 7.25.3 + '@babel/types': 7.25.2 convert-source-map: 2.0.0 - debug: 4.3.5 + debug: 4.3.6(supports-color@8.1.1) gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 @@ -342,71 +323,48 @@ packages: - supports-color dev: true - /@babel/generator@7.24.7: - resolution: {integrity: sha512-oipXieGC3i45Y1A41t4tAqpnEZWgB/lC6Ehh6+rOviR5XWpTtMmLN+fGjz9vOiNRt0p6RtO6DtD0pdU3vpqdSA==} + /@babel/generator@7.25.0: + resolution: {integrity: sha512-3LEEcj3PVW8pW2R1SR1M89g/qrYk/m/mB/tLqn7dn4sbBUQyTqnlod+II2U4dqiGtUmkcnAmkMDralTFZttRiw==} engines: {node: '>=6.9.0'} dependencies: - '@babel/types': 7.24.7 + '@babel/types': 7.25.2 '@jridgewell/gen-mapping': 0.3.5 '@jridgewell/trace-mapping': 0.3.25 jsesc: 2.5.2 dev: true - /@babel/helper-compilation-targets@7.24.7: - resolution: {integrity: sha512-ctSdRHBi20qWOfy27RUb4Fhp07KSJ3sXcuSvTrXrc4aG8NSYDo1ici3Vhg9bg69y5bj0Mr1lh0aeEgTvc12rMg==} + /@babel/helper-compilation-targets@7.25.2: + resolution: {integrity: sha512-U2U5LsSaZ7TAt3cfaymQ8WHh0pxvdHoEk6HVpaexxixjyEquMh0L0YNJNM6CTGKMXV1iksi0iZkGw4AcFkPaaw==} engines: {node: '>=6.9.0'} dependencies: - '@babel/compat-data': 7.24.7 - '@babel/helper-validator-option': 7.24.7 - browserslist: 4.23.1 + '@babel/compat-data': 7.25.2 + '@babel/helper-validator-option': 7.24.8 + browserslist: 4.23.2 lru-cache: 5.1.1 semver: 6.3.1 dev: true - /@babel/helper-environment-visitor@7.24.7: - resolution: {integrity: sha512-DoiN84+4Gnd0ncbBOM9AZENV4a5ZiL39HYMyZJGZ/AZEykHYdJw0wW3kdcsh9/Kn+BRXHLkkklZ51ecPKmI1CQ==} - engines: {node: '>=6.9.0'} - dependencies: - '@babel/types': 7.24.7 - dev: true - - /@babel/helper-function-name@7.24.7: - resolution: {integrity: sha512-FyoJTsj/PEUWu1/TYRiXTIHc8lbw+TDYkZuoE43opPS5TrI7MyONBE1oNvfguEXAD9yhQRrVBnXdXzSLQl9XnA==} - engines: {node: '>=6.9.0'} - dependencies: - '@babel/template': 7.24.7 - '@babel/types': 7.24.7 - dev: true - - /@babel/helper-hoist-variables@7.24.7: - resolution: {integrity: sha512-MJJwhkoGy5c4ehfoRyrJ/owKeMl19U54h27YYftT0o2teQ3FJ3nQUf/I3LlJsX4l3qlw7WRXUmiyajvHXoTubQ==} - engines: {node: '>=6.9.0'} - dependencies: - '@babel/types': 7.24.7 - dev: true - /@babel/helper-module-imports@7.24.7: resolution: {integrity: sha512-8AyH3C+74cgCVVXow/myrynrAGv+nTVg5vKu2nZph9x7RcRwzmh0VFallJuFTZ9mx6u4eSdXZfcOzSqTUm0HCA==} engines: {node: '>=6.9.0'} dependencies: - '@babel/traverse': 7.24.7 - '@babel/types': 7.24.7 + '@babel/traverse': 7.25.3 + '@babel/types': 7.25.2 transitivePeerDependencies: - supports-color dev: true - /@babel/helper-module-transforms@7.24.7(@babel/core@7.24.7): - resolution: {integrity: sha512-1fuJEwIrp+97rM4RWdO+qrRsZlAeL1lQJoPqtCYWv0NL115XM93hIH4CSRln2w52SqvmY5hqdtauB6QFCDiZNQ==} + /@babel/helper-module-transforms@7.25.2(@babel/core@7.25.2): + resolution: {integrity: sha512-BjyRAbix6j/wv83ftcVJmBt72QtHI56C7JXZoG2xATiLpmoC7dpd8WnkikExHDVPpi/3qCmO6WY1EaXOluiecQ==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0 dependencies: - '@babel/core': 7.24.7 - '@babel/helper-environment-visitor': 7.24.7 + '@babel/core': 7.25.2 '@babel/helper-module-imports': 7.24.7 '@babel/helper-simple-access': 7.24.7 - '@babel/helper-split-export-declaration': 7.24.7 '@babel/helper-validator-identifier': 7.24.7 + '@babel/traverse': 7.25.3 transitivePeerDependencies: - supports-color dev: true @@ -415,26 +373,14 @@ packages: resolution: {integrity: sha512-zBAIvbCMh5Ts+b86r/CjU+4XGYIs+R1j951gxI3KmmxBMhCg4oQMsv6ZXQ64XOm/cvzfU1FmoCyt6+owc5QMYg==} engines: {node: '>=6.9.0'} dependencies: - '@babel/traverse': 7.24.7 - '@babel/types': 7.24.7 + '@babel/traverse': 7.25.3 + '@babel/types': 7.25.2 transitivePeerDependencies: - supports-color dev: true - /@babel/helper-split-export-declaration@7.24.7: - resolution: {integrity: sha512-oy5V7pD+UvfkEATUKvIjvIAH/xCzfsFVw7ygW2SI6NClZzquT+mwdTfgfdbUiceh6iQO0CHtCPsyze/MZ2YbAA==} - engines: {node: '>=6.9.0'} - dependencies: - '@babel/types': 7.24.7 - dev: true - - /@babel/helper-string-parser@7.24.7: - resolution: {integrity: sha512-7MbVt6xrwFQbunH2DNQsAP5sTGxfqQtErvBIvIMi6EQnbgUOuVYanvREcmFrOPhoXBrTtjhhP+lW+o5UfK+tDg==} - engines: {node: '>=6.9.0'} - dev: true - - /@babel/helper-validator-identifier@7.22.5: - resolution: {integrity: sha512-aJXu+6lErq8ltp+JhkJUfk1MTGyuA4v7f3pA+BJ5HLfNC6nAQ0Cpi9uOquUj8Hehg0aUiHzWQbOVJGao6ztBAQ==} + /@babel/helper-string-parser@7.24.8: + resolution: {integrity: sha512-pO9KhhRcuUyGnJWwyEgnRJTSIZHiT+vMD0kPeD+so0l7mxkMT19g3pjY9GTnHySck/hDzq+dtW/4VgnMkippsQ==} engines: {node: '>=6.9.0'} dev: true @@ -443,26 +389,17 @@ packages: engines: {node: '>=6.9.0'} dev: true - /@babel/helper-validator-option@7.24.7: - resolution: {integrity: sha512-yy1/KvjhV/ZCL+SM7hBrvnZJ3ZuT9OuZgIJAGpPEToANvc3iM6iDvBnRjtElWibHU6n8/LPR/EjX9EtIEYO3pw==} - engines: {node: '>=6.9.0'} - dev: true - - /@babel/helpers@7.24.7: - resolution: {integrity: sha512-NlmJJtvcw72yRJRcnCmGvSi+3jDEg8qFu3z0AFoymmzLx5ERVWyzd9kVXr7Th9/8yIJi2Zc6av4Tqz3wFs8QWg==} + /@babel/helper-validator-option@7.24.8: + resolution: {integrity: sha512-xb8t9tD1MHLungh/AIoWYN+gVHaB9kwlu8gffXGSt3FFEIT7RjS+xWbc2vUD1UTZdIpKj/ab3rdqJ7ufngyi2Q==} engines: {node: '>=6.9.0'} - dependencies: - '@babel/template': 7.24.7 - '@babel/types': 7.24.7 dev: true - /@babel/highlight@7.22.5: - resolution: {integrity: sha512-BSKlD1hgnedS5XRnGOljZawtag7H1yPfQp0tdNJCHoH6AZ+Pcm9VvkrK59/Yy593Ypg0zMxH2BxD1VPYUQ7UIw==} + /@babel/helpers@7.25.0: + resolution: {integrity: sha512-MjgLZ42aCm0oGjJj8CtSM3DB8NOOf8h2l7DCTePJs29u+v7yO/RBX9nShlKMgFnRks/Q4tBAe7Hxnov9VkGwLw==} engines: {node: '>=6.9.0'} dependencies: - '@babel/helper-validator-identifier': 7.22.5 - chalk: 2.4.2 - js-tokens: 4.0.0 + '@babel/template': 7.25.0 + '@babel/types': 7.25.2 dev: true /@babel/highlight@7.24.7: @@ -475,62 +412,54 @@ packages: picocolors: 1.0.1 dev: true - /@babel/parser@7.24.7: - resolution: {integrity: sha512-9uUYRm6OqQrCqQdG1iCBwBPZgN8ciDBro2nIOFaiRz1/BCxaI7CNvQbDHvsArAC7Tw9Hda/B3U+6ui9u4HWXPw==} + /@babel/parser@7.25.3: + resolution: {integrity: sha512-iLTJKDbJ4hMvFPgQwwsVoxtHyWpKKPBrxkANrSYewDPaPpT5py5yeVkgPIJ7XYXhndxJpaA3PyALSXQ7u8e/Dw==} engines: {node: '>=6.0.0'} hasBin: true dependencies: - '@babel/types': 7.24.7 + '@babel/types': 7.25.2 dev: true - /@babel/runtime@7.22.6: - resolution: {integrity: sha512-wDb5pWm4WDdF6LFUde3Jl8WzPA+3ZbxYqkC6xAXuD3irdEHN1k0NfTRrJD8ZD378SJ61miMLCqIOXYhd8x+AJQ==} + /@babel/runtime@7.25.0: + resolution: {integrity: sha512-7dRy4DwXwtzBrPbZflqxnvfxLF8kdZXPkhymtDeFoFqE6ldzjQFgYTtYIFARcLEYDrqfBfYcZt1WqFxRoyC9Rw==} engines: {node: '>=6.9.0'} dependencies: - regenerator-runtime: 0.13.11 + regenerator-runtime: 0.14.1 dev: true - /@babel/template@7.24.7: - resolution: {integrity: sha512-jYqfPrU9JTF0PmPy1tLYHW4Mp4KlgxJD9l2nP9fD6yT/ICi554DmrWBAEYpIelzjHf1msDP3PxJIRt/nFNfBig==} + /@babel/template@7.25.0: + resolution: {integrity: sha512-aOOgh1/5XzKvg1jvVz7AVrx2piJ2XBi227DHmbY6y+bM9H2FlN+IfecYu4Xl0cNiiVejlsCri89LUsbj8vJD9Q==} engines: {node: '>=6.9.0'} dependencies: '@babel/code-frame': 7.24.7 - '@babel/parser': 7.24.7 - '@babel/types': 7.24.7 + '@babel/parser': 7.25.3 + '@babel/types': 7.25.2 dev: true - /@babel/traverse@7.24.7: - resolution: {integrity: sha512-yb65Ed5S/QAcewNPh0nZczy9JdYXkkAbIsEo+P7BE7yO3txAY30Y/oPa3QkQ5It3xVG2kpKMg9MsdxZaO31uKA==} + /@babel/traverse@7.25.3: + resolution: {integrity: sha512-HefgyP1x754oGCsKmV5reSmtV7IXj/kpaE1XYY+D9G5PvKKoFfSbiS4M77MdjuwlZKDIKFCffq9rPU+H/s3ZdQ==} engines: {node: '>=6.9.0'} dependencies: '@babel/code-frame': 7.24.7 - '@babel/generator': 7.24.7 - '@babel/helper-environment-visitor': 7.24.7 - '@babel/helper-function-name': 7.24.7 - '@babel/helper-hoist-variables': 7.24.7 - '@babel/helper-split-export-declaration': 7.24.7 - '@babel/parser': 7.24.7 - '@babel/types': 7.24.7 - debug: 4.3.5 + '@babel/generator': 7.25.0 + '@babel/parser': 7.25.3 + '@babel/template': 7.25.0 + '@babel/types': 7.25.2 + debug: 4.3.6(supports-color@8.1.1) globals: 11.12.0 transitivePeerDependencies: - supports-color dev: true - /@babel/types@7.24.7: - resolution: {integrity: sha512-XEFXSlxiG5td2EJRe8vOmRbaXVgfcBlszKujvVmWIK/UpywWljQCfzAv3RQCGujWQ1RD4YYWEAqDXfuJiy8f5Q==} + /@babel/types@7.25.2: + resolution: {integrity: sha512-YTnYtra7W9e6/oAZEHj0bJehPRUlLH9/fbpT5LfB0NhQXyALCRkRs3zH9v07IYhkgpqX6Z78FnuccZr/l4Fs4Q==} engines: {node: '>=6.9.0'} dependencies: - '@babel/helper-string-parser': 7.24.7 + '@babel/helper-string-parser': 7.24.8 '@babel/helper-validator-identifier': 7.24.7 to-fast-properties: 2.0.0 dev: true - /@colors/colors@1.5.0: - resolution: {integrity: sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==} - engines: {node: '>=0.1.90'} - dev: false - /@colors/colors@1.6.0: resolution: {integrity: sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==} engines: {node: '>=0.1.90'} @@ -591,7 +520,7 @@ packages: engines: {node: '>=6.0.0'} dependencies: '@jridgewell/set-array': 1.2.1 - '@jridgewell/sourcemap-codec': 1.4.15 + '@jridgewell/sourcemap-codec': 1.5.0 '@jridgewell/trace-mapping': 0.3.25 dev: true @@ -605,24 +534,32 @@ packages: engines: {node: '>=6.0.0'} dev: true - /@jridgewell/sourcemap-codec@1.4.15: - resolution: {integrity: sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==} + /@jridgewell/sourcemap-codec@1.5.0: + resolution: {integrity: sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==} dev: true /@jridgewell/trace-mapping@0.3.25: resolution: {integrity: sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==} dependencies: '@jridgewell/resolve-uri': 3.1.2 - '@jridgewell/sourcemap-codec': 1.4.15 + '@jridgewell/sourcemap-codec': 1.5.0 dev: true /@jridgewell/trace-mapping@0.3.9: resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==} dependencies: '@jridgewell/resolve-uri': 3.1.2 - '@jridgewell/sourcemap-codec': 1.4.15 + '@jridgewell/sourcemap-codec': 1.5.0 dev: true + /@mongodb-js/saslprep@1.1.8: + resolution: {integrity: sha512-qKwC/M/nNNaKUBMQ0nuzm47b7ZYWQHN3pcXq4IIcoSBc2hOIrflAxJduIvvqmhoz3gR2TacTAs8vlsCVPkiEdQ==} + requiresBuild: true + dependencies: + sparse-bitfield: 3.0.3 + dev: false + optional: true + /@nrwl/tao@16.10.0: resolution: {integrity: sha512-QNAanpINbr+Pod6e1xNgFbzK1x5wmZl+jMocgiEFXZ67KHvmbD6MAQQr0MMz+GPhIu7EE4QCTLTyCEMlAG+K5Q==} hasBin: true @@ -774,7 +711,7 @@ packages: dependencies: '@sinonjs/commons': 2.0.0 lodash.get: 4.4.2 - type-detect: 4.0.8 + type-detect: 4.1.0 dev: true /@sinonjs/text-encoding@0.7.2: @@ -797,11 +734,11 @@ packages: resolution: {integrity: sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==} dev: true - /@types/body-parser@1.19.2: - resolution: {integrity: sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g==} + /@types/body-parser@1.19.5: + resolution: {integrity: sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==} dependencies: - '@types/connect': 3.4.35 - '@types/node': 20.14.7 + '@types/connect': 3.4.38 + '@types/node': 22.0.0 dev: true /@types/chai-as-promised@7.1.8: @@ -814,14 +751,10 @@ packages: resolution: {integrity: sha512-PatH4iOdyh3MyWtmHVFXLWCCIhUbopaltqddG9BzB+gMIzee2MJrvd+jouii9Z3wzQJruGWAm7WOMjgfG8hQlQ==} dev: true - /@types/connect@3.4.35: - resolution: {integrity: sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==} + /@types/connect@3.4.38: + resolution: {integrity: sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==} dependencies: - '@types/node': 20.14.7 - dev: true - - /@types/cookiejar@2.1.2: - resolution: {integrity: sha512-t73xJJrvdTjXrn4jLS9VSGRbz0nUY3cl2DMGDU48lKl+HR9dbbjW2A9r3g40VA++mQpy6uuHg33gy7du2BKpog==} + '@types/node': 22.0.0 dev: true /@types/cookiejar@2.1.5: @@ -831,67 +764,59 @@ packages: /@types/errorhandler@0.0.32: resolution: {integrity: sha512-wC9CfwPMIzklPd5lEYC8HnQdlMC1PswlohWmEDMWlw+E/rMYuz5eSqKBc72Earb29KptKJrRl77qVRJzrZndww==} dependencies: - '@types/express': 4.17.17 + '@types/express': 4.17.21 dev: true - /@types/express-serve-static-core@4.17.35: - resolution: {integrity: sha512-wALWQwrgiB2AWTT91CB62b6Yt0sNHpznUXeZEcnPU3DRdlDIz74x8Qg1UUYKSVFi+va5vKOLYRBI1bRKiLLKIg==} + /@types/express-serve-static-core@4.19.5: + resolution: {integrity: sha512-y6W03tvrACO72aijJ5uF02FRq5cgDR9lUxddQ8vyF+GvmjJQqbzDcJngEjURc+ZsG31VI3hODNZJ2URj86pzmg==} dependencies: - '@types/node': 20.14.7 - '@types/qs': 6.9.7 - '@types/range-parser': 1.2.4 - '@types/send': 0.17.1 + '@types/node': 22.0.0 + '@types/qs': 6.9.15 + '@types/range-parser': 1.2.7 + '@types/send': 0.17.4 dev: true - /@types/express@4.17.17: - resolution: {integrity: sha512-Q4FmmuLGBG58btUnfS1c1r/NQdlp3DMfGDGig8WhfpA2YRUtEkxAjkZb0yvplJGYdF1fsQ81iMDcH24sSCNC/Q==} + /@types/express@4.17.21: + resolution: {integrity: sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ==} dependencies: - '@types/body-parser': 1.19.2 - '@types/express-serve-static-core': 4.17.35 - '@types/qs': 6.9.7 - '@types/serve-static': 1.15.2 + '@types/body-parser': 1.19.5 + '@types/express-serve-static-core': 4.19.5 + '@types/qs': 6.9.15 + '@types/serve-static': 1.15.7 dev: true - /@types/http-errors@2.0.1: - resolution: {integrity: sha512-/K3ds8TRAfBvi5vfjuz8y6+GiAYBZ0x4tXv1Av6CWBWn0IlADc+ZX9pMq7oU0fNQPnBwIZl3rmeLp6SBApbxSQ==} + /@types/http-errors@2.0.4: + resolution: {integrity: sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA==} dev: true /@types/jsonwebtoken@8.5.9: resolution: {integrity: sha512-272FMnFGzAVMGtu9tkr29hRL6bZj4Zs1KZNeHLnKqAvp06tAIcarTMwOh8/8bz4FmKRcMxZhZNeUAQsNLoiPhg==} dependencies: - '@types/node': 20.14.7 - dev: true - - /@types/lodash@4.14.195: - resolution: {integrity: sha512-Hwx9EUgdwf2GLarOjQp5ZH8ZmblzcbTBC2wtQWNKARBSxM9ezRIAUpeDTgoQRAFB0+8CNWXVA9+MaSOzOF3nPg==} - dev: true - - /@types/lodash@4.17.5: - resolution: {integrity: sha512-MBIOHVZqVqgfro1euRDWX7OO0fBVUUMrN6Pwm8LQsz8cWhEpihlvR70ENj3f40j58TNxZaWv2ndSkInykNBBJw==} + '@types/node': 22.0.0 dev: true - /@types/memoizee@0.4.8: - resolution: {integrity: sha512-qDpXKGgwKywnQt/64fH1O0LiPA++QGIYeykEUiZ51HymKVRLnUSGcRuF60IfpPeeXiuRwiR/W4y7S5VzbrgLCA==} + /@types/lodash@4.17.7: + resolution: {integrity: sha512-8wTvZawATi/lsmNu10/j2hk1KEP0IvjubqPE3cu1Xz7xfXXt5oCq3SNUz4fMIP4XGF9Ky+Ue2tBA3hcS7LSBlA==} dev: true - /@types/mime@1.3.2: - resolution: {integrity: sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==} + /@types/memoizee@0.4.11: + resolution: {integrity: sha512-2gyorIBZu8GoDr9pYjROkxWWcFtHCquF7TVbN2I+/OvgZhnIGQS0vX5KJz4lXNKb8XOSfxFOSG5OLru1ESqLUg==} dev: true - /@types/mime@3.0.1: - resolution: {integrity: sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA==} + /@types/mime@1.3.5: + resolution: {integrity: sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==} dev: true - /@types/mocha@10.0.6: - resolution: {integrity: sha512-dJvrYWxP/UcXm36Qn36fxhUKu8A/xMRXVT2cliFF1Z7UA9liG5Psj3ezNSZw+5puH2czDXRLcXQxf8JbJt0ejg==} + /@types/mocha@10.0.7: + resolution: {integrity: sha512-GN8yJ1mNTcFcah/wKEFIJckJx9iJLoMSzWcfRRuxz/Jk+U6KQNnml+etbtxFK8lPjzOw3zp4Ha/kjSst9fsHYw==} dev: true /@types/mocha@8.2.3: resolution: {integrity: sha512-ekGvFhFgrc2zYQoX4JeZPmVzZxw6Dtllga7iGHzfbYIYkAMUx/sAFP2GdFpLff+vdHXu5fl7WX9AT+TtqYcsyw==} dev: true - /@types/ms@0.7.31: - resolution: {integrity: sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==} + /@types/ms@0.7.34: + resolution: {integrity: sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==} dev: true /@types/node-fetch@2.6.11: @@ -905,36 +830,42 @@ packages: resolution: {integrity: sha512-J8xLz7q2OFulZ2cyGTLE1TbbZcjpno7FaN6zdJNrgAdrJ+DZzh/uFR6YrTb4C+nXakvud8Q4+rbhoIWlYQbUFQ==} dev: true - /@types/node@20.14.7: - resolution: {integrity: sha512-uTr2m2IbJJucF3KUxgnGOZvYbN0QgkGyWxG6973HCpMYFy2KfcgYuIwkJQMQkt1VbBMlvWRbpshFTLxnxCZjKQ==} + /@types/node@20.14.13: + resolution: {integrity: sha512-+bHoGiZb8UiQ0+WEtmph2IWQCjIqg8MDZMAV+ppRRhUZnquF5mQkP/9vpSwJClEiSM/C7fZZExPzfU0vJTyp8w==} dependencies: undici-types: 5.26.5 + dev: true - /@types/normalize-package-data@2.4.1: - resolution: {integrity: sha512-Gj7cI7z+98M282Tqmp2K5EIsoouUEzbBJhQQzDE3jSIRk6r9gsz0oUokqIUR4u1R3dMHo0pDHM7sNOHyhulypw==} + /@types/node@22.0.0: + resolution: {integrity: sha512-VT7KSYudcPOzP5Q0wfbowyNLaVR8QWUdw+088uFWwfvpY6uCWaXpqV6ieLAu9WBcnTa7H4Z5RLK8I5t2FuOcqw==} + dependencies: + undici-types: 6.11.1 + + /@types/normalize-package-data@2.4.4: + resolution: {integrity: sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA==} dev: true - /@types/qs@6.9.7: - resolution: {integrity: sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==} + /@types/qs@6.9.15: + resolution: {integrity: sha512-uXHQKES6DQKKCLh441Xv/dwxOq1TVS3JPUMlEqoEglvlhR6Mxnlew/Xq/LRVHpLyk7iK3zODe1qYHIMltO7XGg==} dev: true - /@types/range-parser@1.2.4: - resolution: {integrity: sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==} + /@types/range-parser@1.2.7: + resolution: {integrity: sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==} dev: true - /@types/send@0.17.1: - resolution: {integrity: sha512-Cwo8LE/0rnvX7kIIa3QHCkcuF21c05Ayb0ZfxPiv0W8VRiZiNW/WuRupHKpqqGVGf7SUA44QSOUKaEd9lIrd/Q==} + /@types/send@0.17.4: + resolution: {integrity: sha512-x2EM6TJOybec7c52BX0ZspPodMsQUd5L6PRwOunVyVUhXiBSKf3AezDL8Dgvgt5o0UfKNfuA0eMLr2wLT4AiBA==} dependencies: - '@types/mime': 1.3.2 - '@types/node': 20.14.7 + '@types/mime': 1.3.5 + '@types/node': 22.0.0 dev: true - /@types/serve-static@1.15.2: - resolution: {integrity: sha512-J2LqtvFYCzaj8pVYKw8klQXrLLk7TBZmQ4ShlcdkELFKGwGMfevMLneMMRkMgZxotOD9wg497LpC7O8PcvAmfw==} + /@types/serve-static@1.15.7: + resolution: {integrity: sha512-W8Ym+h8nhuRwaKPaDw34QUkwsGi6Rc4yYqvKFo5rm2FUEhCFbzVWrxXUxuKK8TASjWsysJY0nsmNCGhCOIsrOw==} dependencies: - '@types/http-errors': 2.0.1 - '@types/mime': 3.0.1 - '@types/node': 20.14.7 + '@types/http-errors': 2.0.4 + '@types/node': 22.0.0 + '@types/send': 0.17.4 dev: true /@types/sinon@10.0.20: @@ -951,40 +882,36 @@ packages: resolution: {integrity: sha512-YIGelp3ZyMiH0/A09PMAORO0EBGlF5xIKfDpK74wdYvWUs2o96b5CItJcWPdH409b7SAXIIG6p8NdU/4U2Maww==} dependencies: '@types/cookiejar': 2.1.5 - '@types/node': 20.14.7 + '@types/node': 20.14.13 dev: true - /@types/superagent@4.1.18: - resolution: {integrity: sha512-LOWgpacIV8GHhrsQU+QMZuomfqXiqzz3ILLkCtKx3Us6AmomFViuzKT9D693QTKgyut2oCytMG8/efOop+DB+w==} + /@types/superagent@4.1.24: + resolution: {integrity: sha512-mEafCgyKiMFin24SDzWN7yAADt4gt6YawFiNMp0QS5ZPboORfyxFt0s3VzJKhTaKg9py/4FUmrHLTNfJKt9Rbw==} dependencies: - '@types/cookiejar': 2.1.2 - '@types/node': 20.14.7 + '@types/cookiejar': 2.1.5 + '@types/node': 22.0.0 dev: true /@types/swagger-ui-express@3.0.1: resolution: {integrity: sha512-AHIIs9tUDimRAKvmHHBXxNfavjcvZuGgu/iXaSVpBk+C6EO/H1PUBIq4l5opFXLbV+hBuVbYnpycbobhSoMjTA==} dependencies: - '@types/express': 4.17.17 - '@types/serve-static': 1.15.2 + '@types/express': 4.17.21 + '@types/serve-static': 1.15.7 dev: true - /@types/triple-beam@1.3.2: - resolution: {integrity: sha512-txGIh+0eDFzKGC25zORnswy+br1Ha7hj5cMVwKIU7+s0U2AxxJru/jZSMU6OC9MJWP6+pc/hc6ZjyZShpsyY2g==} - dev: false - /@types/triple-beam@1.3.5: resolution: {integrity: sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==} dev: false - /@types/webidl-conversions@7.0.0: - resolution: {integrity: sha512-xTE1E+YF4aWPJJeUzaZI5DRntlkY3+BCVJi0axFptnjGmAoWxkyREIh/XMrfxVLejwQxMCfDXdICo0VLxThrog==} + /@types/webidl-conversions@7.0.3: + resolution: {integrity: sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA==} dev: false /@types/whatwg-url@8.2.2: resolution: {integrity: sha512-FtQu10RWgn3D9U4aazdwIE2yzphmTJREDqNdODHrbrZmmMqI0vMheC/6NE/J1Yveaj8H+ela+YwWTjq5PGmuhA==} dependencies: - '@types/node': 20.14.7 - '@types/webidl-conversions': 7.0.0 + '@types/node': 22.0.0 + '@types/webidl-conversions': 7.0.3 dev: false /@ungap/promise-all-settled@1.1.2: @@ -1018,10 +945,6 @@ packages: through: 2.3.8 dev: true - /abbrev@1.1.1: - resolution: {integrity: sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==} - dev: true - /accepts@1.3.8: resolution: {integrity: sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==} engines: {node: '>= 0.6'} @@ -1034,11 +957,11 @@ packages: resolution: {integrity: sha512-MxXdReSRhGO7VlFe1bRG/oI7/mdLV9B9JJT0N8vZOhF7gFRR5l3M8W9G8JxmKV+JC5mGqJ0QvqfSOLsCPa4nUw==} engines: {node: '>=0.4.0'} dependencies: - acorn: 8.12.0 + acorn: 8.12.1 dev: true - /acorn@8.12.0: - resolution: {integrity: sha512-RTvkC4w+KNXrM39/lWCUaG0IbRkWdCv7W/IOW9oU6SawyxulvkQy5HQPVTKxEjczcUvapcrw3cFx/60VN/NRNw==} + /acorn@8.12.1: + resolution: {integrity: sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg==} engines: {node: '>=0.4.0'} hasBin: true dev: true @@ -1060,13 +983,13 @@ packages: uri-js: 4.4.1 dev: false - /ajv@8.12.0: - resolution: {integrity: sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==} + /ajv@8.17.1: + resolution: {integrity: sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==} dependencies: fast-deep-equal: 3.1.3 + fast-uri: 3.0.1 json-schema-traverse: 1.0.0 require-from-string: 2.0.2 - uri-js: 4.4.1 dev: false /ansi-colors@4.1.1: @@ -1161,7 +1084,7 @@ packages: dev: true /array-flatten@1.1.1: - resolution: {integrity: sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=} + resolution: {integrity: sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==} dev: false /asap@2.0.6: @@ -1183,10 +1106,6 @@ packages: resolution: {integrity: sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==} dev: true - /async@3.2.4: - resolution: {integrity: sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==} - dev: false - /async@3.2.5: resolution: {integrity: sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg==} dev: false @@ -1198,18 +1117,8 @@ packages: resolution: {integrity: sha512-08kcGqnYf/YmjoRhfxyu+CLxBjUtHLXLXX/vUfx9l2LYzG3c1m61nrpyFUZI6zeS+Li/wWMMidD9KgrqtGq3mA==} dev: false - /aws4@1.12.0: - resolution: {integrity: sha512-NmWvPnx0F1SfrQbYwOi7OeaNGokp9XhzNioJ/CSBs8Qa4vxug81mhJEAVZwxXuBmYB5KDRfMq/F3RR0BIU7sWg==} - dev: false - - /axios@1.4.0: - resolution: {integrity: sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==} - dependencies: - follow-redirects: 1.15.2 - form-data: 4.0.0 - proxy-from-env: 1.1.0 - transitivePeerDependencies: - - debug + /aws4@1.13.0: + resolution: {integrity: sha512-3AungXC4I8kKsS9PuS4JH2nc+0bVY/mjgrephHTIi8fpEeGsTHBUJeosp0Wc1myYMElmD0B3Oc4XL/HVJ4PV2g==} dev: false /axios@1.7.2: @@ -1220,7 +1129,6 @@ packages: proxy-from-env: 1.1.0 transitivePeerDependencies: - debug - dev: true /balanced-match@1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} @@ -1256,26 +1164,6 @@ packages: readable-stream: 3.6.2 dev: true - /body-parser@1.20.1: - resolution: {integrity: sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==} - engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} - dependencies: - bytes: 3.1.2 - content-type: 1.0.5 - debug: 2.6.9 - depd: 2.0.0 - destroy: 1.2.0 - http-errors: 2.0.0 - iconv-lite: 0.4.24 - on-finished: 2.4.1 - qs: 6.11.0 - raw-body: 2.5.1 - type-is: 1.6.18 - unpipe: 1.0.0 - transitivePeerDependencies: - - supports-color - dev: false - /body-parser@1.20.2: resolution: {integrity: sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA==} engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} @@ -1320,19 +1208,19 @@ packages: resolution: {integrity: sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==} dev: true - /browserslist@4.23.1: - resolution: {integrity: sha512-TUfofFo/KsK/bWZ9TWQ5O26tsWW4Uhmt8IYklbnUa70udB6P2wA7w7o4PY4muaEPBQaAX+CEnmmIA41NVHtPVw==} + /browserslist@4.23.2: + resolution: {integrity: sha512-qkqSyistMYdxAcw+CzbZwlBy8AGmS/eEWs+sEV5TnLRGDOL+C5M2EnH6tlZyg0YoAxGJAFKh61En9BR941GnHA==} engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} hasBin: true dependencies: - caniuse-lite: 1.0.30001636 - electron-to-chromium: 1.4.808 - node-releases: 2.0.14 - update-browserslist-db: 1.0.16(browserslist@4.23.1) + caniuse-lite: 1.0.30001645 + electron-to-chromium: 1.5.4 + node-releases: 2.0.18 + update-browserslist-db: 1.1.0(browserslist@4.23.2) dev: true - /bson@5.4.0: - resolution: {integrity: sha512-WRZ5SQI5GfUuKnPTNmAYPiKIof3ORXAF4IRU5UcgmivNIon01rWQlw5RUH954dpu8yGL8T59YShVddIPaU/gFA==} + /bson@5.5.1: + resolution: {integrity: sha512-ix0EwukN2EpC0SRWIj/7B5+A6uQMQy6KMREI9qQqvgpkV2frH63T0UDVd1SYedL6dNCmDBYB3QtXi4ISk9YT+g==} engines: {node: '>=14.20.1'} dev: false @@ -1391,13 +1279,6 @@ packages: write-file-atomic: 3.0.3 dev: true - /call-bind@1.0.2: - resolution: {integrity: sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==} - dependencies: - function-bind: 1.1.1 - get-intrinsic: 1.2.1 - dev: false - /call-bind@1.0.7: resolution: {integrity: sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==} engines: {node: '>= 0.4'} @@ -1407,7 +1288,6 @@ packages: function-bind: 1.1.2 get-intrinsic: 1.2.4 set-function-length: 1.2.2 - dev: true /caller-callsite@2.0.0: resolution: {integrity: sha512-JuG3qI4QOftFsZyOn1qq87fq5grLIyk1JYd5lJmdA+fG7aQ9pA/i3JIJGcO3q0MrRcHlOt1U+ZeHW8Dq9axALQ==} @@ -1438,8 +1318,8 @@ packages: engines: {node: '>=10'} dev: true - /caniuse-lite@1.0.30001636: - resolution: {integrity: sha512-bMg2vmr8XBsbL6Lr0UHXy/21m84FTxDLWn2FSqMd5PrlbMxwJlQnC2YWYxVgp66PZE+BBNF2jYQUBKCo1FDeZg==} + /caniuse-lite@1.0.30001645: + resolution: {integrity: sha512-GFtY2+qt91kzyMk6j48dJcwJVq5uTkk71XxE3RtScx7XWRLsO7bU44LOFkOZYR8w9YMS0UhPSYpN/6rAMImmLw==} dev: true /caseless@0.12.0: @@ -1450,12 +1330,12 @@ packages: resolution: {integrity: sha512-X2y0Ssu48ucdkrNgCdg6k3EZWjWVy/dsEywUUTeZEIW31f3bQfq65Svm+TzU1Hz+qqhdmyCdjGhUvRsSKHl/mw==} dev: false - /chai-as-promised@7.1.2(chai@4.4.1): + /chai-as-promised@7.1.2(chai@4.5.0): resolution: {integrity: sha512-aBDHZxRzYnUYuIAIPBH2s511DjlKPzXNlXSGFC8CwmroWQLfrW0LtE1nK3MAwwNhJPa9raEjNCmRoFpG0Hurdw==} peerDependencies: chai: '>= 2.1.2 < 6' dependencies: - chai: 4.4.1 + chai: 4.5.0 check-error: 1.0.3 dev: true @@ -1469,14 +1349,14 @@ packages: cookiejar: 2.1.4 is-ip: 2.0.0 methods: 1.1.2 - qs: 6.12.1 + qs: 6.12.3 superagent: 8.1.2 transitivePeerDependencies: - supports-color dev: true - /chai@4.4.1: - resolution: {integrity: sha512-13sOfMv2+DWduEU+/xbun3LScLoqN17nBeTLUsmDfKdoiC1fr0n9PU4guu4AhRcOVFk/sW8LyZWHuhWtQZiF+g==} + /chai@4.5.0: + resolution: {integrity: sha512-RITGBfijLkBddZvnn8jdqoTypxvqbOLYQkGGxXzeFjVHvudaPw0HNFD9x928/eUwYWd2dPCugVqspGALTZZQKw==} engines: {node: '>=4'} dependencies: assertion-error: 1.1.0 @@ -1485,7 +1365,7 @@ packages: get-func-name: 2.0.2 loupe: 2.3.7 pathval: 1.1.1 - type-detect: 4.0.8 + type-detect: 4.1.0 dev: true /chalk@2.4.2: @@ -1531,8 +1411,8 @@ packages: fsevents: 2.3.3 dev: true - /chokidar@3.5.3: - resolution: {integrity: sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==} + /chokidar@3.6.0: + resolution: {integrity: sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==} engines: {node: '>= 8.10.0'} dependencies: anymatch: 3.1.3 @@ -1685,7 +1565,7 @@ packages: lodash: 4.17.21 read-pkg: 4.0.1 rxjs: 6.6.7 - spawn-command: 0.0.2-1 + spawn-command: 0.0.2 supports-color: 6.1.0 tree-kill: 1.2.2 yargs: 13.3.2 @@ -1712,11 +1592,11 @@ packages: dev: true /cookie-signature@1.0.6: - resolution: {integrity: sha1-4wOogrNCzD7oylE6eZmXNNqzriw=} + resolution: {integrity: sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==} dev: false - /cookie@0.5.0: - resolution: {integrity: sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==} + /cookie@0.6.0: + resolution: {integrity: sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==} engines: {node: '>= 0.6'} dev: false @@ -1752,7 +1632,7 @@ packages: dependencies: nice-try: 1.0.5 path-key: 2.0.1 - semver: 5.7.1 + semver: 5.7.2 shebang-command: 1.2.0 which: 1.3.1 dev: true @@ -1766,11 +1646,12 @@ packages: which: 2.0.2 dev: true - /d@1.0.1: - resolution: {integrity: sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA==} + /d@1.0.2: + resolution: {integrity: sha512-MOqHvMWF9/9MX6nza0KgvFH4HpMU0EF5uUDXqX/BtxtU8NfB0QzRtJ8Oe/6SuS4kbhyzVJwjd97EA4PKrzJ8bw==} + engines: {node: '>=0.12'} dependencies: - es5-ext: 0.10.62 - type: 1.2.0 + es5-ext: 0.10.64 + type: 2.7.3 dev: false /dashdash@1.14.1: @@ -1784,7 +1665,7 @@ packages: resolution: {integrity: sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==} engines: {node: '>=0.11'} dependencies: - '@babel/runtime': 7.22.6 + '@babel/runtime': 7.25.0 dev: true /debug@2.6.9: @@ -1834,8 +1715,8 @@ packages: supports-color: 8.1.1 dev: true - /debug@4.3.4(supports-color@8.1.1): - resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==} + /debug@4.3.6(supports-color@8.1.1): + resolution: {integrity: sha512-O/09Bd4Z1fBrU4VzkhFqVgpPzaGbw6Sm9FEkBT1A/YBXQFGuuSxa1dN2nxgxS34JmKXqYx8CZAwEVoJFImUXIg==} engines: {node: '>=6.0'} peerDependencies: supports-color: '*' @@ -1846,18 +1727,6 @@ packages: ms: 2.1.2 supports-color: 8.1.1 - /debug@4.3.5: - resolution: {integrity: sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==} - engines: {node: '>=6.0'} - peerDependencies: - supports-color: '*' - peerDependenciesMeta: - supports-color: - optional: true - dependencies: - ms: 2.1.2 - dev: true - /decamelize@1.2.0: resolution: {integrity: sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==} engines: {node: '>=0.10.0'} @@ -1872,7 +1741,7 @@ packages: resolution: {integrity: sha512-SUwdGfqdKOwxCPeVYjwSyRpJ7Z+fhpwIAtmCUdZIWZ/YP5R9WAsyuSgpLVDi9bjWoN2LXHNss/dk3urXtdQxGg==} engines: {node: '>=6'} dependencies: - type-detect: 4.0.8 + type-detect: 4.1.0 dev: true /default-require-extensions@3.0.1: @@ -1889,7 +1758,6 @@ packages: es-define-property: 1.0.0 es-errors: 1.3.0 gopd: 1.0.1 - dev: true /define-lazy-prop@2.0.0: resolution: {integrity: sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og==} @@ -1965,16 +1833,16 @@ packages: engines: {node: '>=12'} dev: true - /dotenv@16.3.1: - resolution: {integrity: sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ==} - engines: {node: '>=12'} - dev: false - /dotenv@16.3.2: resolution: {integrity: sha512-HTlk5nmhkm8F6JcdXvHIzaorzCoziNQT9mGxLPVXW8wJF1TiGSL60ZGB4gHWabHOaMmWmhvk2/lPHfnBiT78AQ==} engines: {node: '>=12'} dev: true + /dotenv@16.4.5: + resolution: {integrity: sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==} + engines: {node: '>=12'} + dev: false + /duplexer@0.1.2: resolution: {integrity: sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==} dev: true @@ -1997,11 +1865,11 @@ packages: dev: false /ee-first@1.1.1: - resolution: {integrity: sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=} + resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} dev: false - /electron-to-chromium@1.4.808: - resolution: {integrity: sha512-0ItWyhPYnww2VOuCGF4s1LTfbrdAV2ajy/TN+ZTuhR23AHI6rWHCrBXJ/uxoXOvRRqw8qjYVrG81HFI7x/2wdQ==} + /electron-to-chromium@1.5.4: + resolution: {integrity: sha512-orzA81VqLyIGUEA77YkVA1D+N+nNfl2isJVjjmOyrlxuooZ19ynb+dOlaDTqd/idKRS9lDCSBmtzM+kyCsMnkA==} dev: true /emoji-regex@7.0.3: @@ -2057,20 +1925,19 @@ packages: engines: {node: '>= 0.4'} dependencies: get-intrinsic: 1.2.4 - dev: true /es-errors@1.3.0: resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==} engines: {node: '>= 0.4'} - dev: true - /es5-ext@0.10.62: - resolution: {integrity: sha512-BHLqn0klhEpnOKSrzn/Xsz2UIW8j+cGmo9JLzr8BiUapV8hPL9+FliFqjwr9ngW7jWdnxv6eO+/LqyhJVqgrjA==} + /es5-ext@0.10.64: + resolution: {integrity: sha512-p2snDhiLaXe6dahss1LddxqEm+SkuDvV8dnIQG0MWjyHpcMNfXKPE+/Cc0y+PhxJX3A4xGNeFCj5oc0BUh6deg==} engines: {node: '>=0.10'} requiresBuild: true dependencies: es6-iterator: 2.0.3 - es6-symbol: 3.1.3 + es6-symbol: 3.1.4 + esniff: 2.0.1 next-tick: 1.1.0 dev: false @@ -2081,25 +1948,26 @@ packages: /es6-iterator@2.0.3: resolution: {integrity: sha512-zw4SRzoUkd+cl+ZoE15A9o1oQd920Bb0iOJMQkQhl3jNc03YqVjAhG7scf9C5KWRU/R13Orf588uCC6525o02g==} dependencies: - d: 1.0.1 - es5-ext: 0.10.62 - es6-symbol: 3.1.3 + d: 1.0.2 + es5-ext: 0.10.64 + es6-symbol: 3.1.4 dev: false - /es6-symbol@3.1.3: - resolution: {integrity: sha512-NJ6Yn3FuDinBaBRWl/q5X/s4koRHBrgKAu+yGI6JCBeiu3qrcbJhwT2GeR/EXVfylRk8dpQVJoLEFhK+Mu31NA==} + /es6-symbol@3.1.4: + resolution: {integrity: sha512-U9bFFjX8tFiATgtkJ1zg25+KviIXpgRvRHS8sau3GfhVzThRQrOeksPeT0BWW2MNZs1OEWJ1DPXOQMn0KKRkvg==} + engines: {node: '>=0.12'} dependencies: - d: 1.0.1 + d: 1.0.2 ext: 1.7.0 dev: false /es6-weak-map@2.0.3: resolution: {integrity: sha512-p5um32HOTO1kP+w7PRnB+5lQ43Z6muuMuIMffvDN8ZB4GcnjLBV6zGStpbASIMk4DCAvEaamhe2zhyCb/QXXsA==} dependencies: - d: 1.0.1 - es5-ext: 0.10.62 + d: 1.0.2 + es5-ext: 0.10.64 es6-iterator: 2.0.3 - es6-symbol: 3.1.3 + es6-symbol: 3.1.4 dev: false /escalade@3.1.2: @@ -2121,6 +1989,16 @@ packages: engines: {node: '>=10'} dev: true + /esniff@2.0.1: + resolution: {integrity: sha512-kTUIGKQ/mDPFoJ0oVfcmyJn4iBDRptjNVIzwIFR7tqWXdVI9xfA2RMwY/gbSpJG3lkdWNEjLap/NqVHZiJsdfg==} + engines: {node: '>=0.10'} + dependencies: + d: 1.0.2 + es5-ext: 0.10.64 + event-emitter: 0.3.5 + type: 2.7.3 + dev: false + /esprima@4.0.1: resolution: {integrity: sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==} engines: {node: '>=4'} @@ -2135,8 +2013,8 @@ packages: /event-emitter@0.3.5: resolution: {integrity: sha512-D9rRn9y7kLPnJ+hMq7S/nhvoKwwvVJahBi2BPmx3bvbsEdK3W9ii8cBSGjP+72/LnM4n6fo3+dkCX5FeTQruXA==} dependencies: - d: 1.0.1 - es5-ext: 0.10.62 + d: 1.0.2 + es5-ext: 0.10.64 dev: false /execa@1.0.0: @@ -2167,16 +2045,16 @@ packages: strip-final-newline: 2.0.0 dev: true - /express@4.18.2: - resolution: {integrity: sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==} + /express@4.19.2: + resolution: {integrity: sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==} engines: {node: '>= 0.10.0'} dependencies: accepts: 1.3.8 array-flatten: 1.1.1 - body-parser: 1.20.1 + body-parser: 1.20.2 content-disposition: 0.5.4 content-type: 1.0.5 - cookie: 0.5.0 + cookie: 0.6.0 cookie-signature: 1.0.6 debug: 2.6.9 depd: 2.0.0 @@ -2209,7 +2087,7 @@ packages: /ext@1.7.0: resolution: {integrity: sha512-6hxeJYaL110a9b5TEJSj0gojyHQAmA2ch5Os+ySCiA1QGdS697XWY1pzsrSjqA9LDEEgdB/KypIlR59RcLuHYw==} dependencies: - type: 2.7.2 + type: 2.7.3 dev: false /extend@3.0.2: @@ -2233,6 +2111,10 @@ packages: resolution: {integrity: sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==} dev: true + /fast-uri@3.0.1: + resolution: {integrity: sha512-MWipKbbYiYI0UC7cl8m/i/IWTqfC8YXsqjzybjddLsFjStroQzsHXkc73JutMvBiXmOvapk+axIl79ig5t55Bw==} + dev: false + /fecha@4.2.3: resolution: {integrity: sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==} dev: false @@ -2307,16 +2189,6 @@ packages: resolution: {integrity: sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==} dev: false - /follow-redirects@1.15.2: - resolution: {integrity: sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==} - engines: {node: '>=4.0'} - peerDependencies: - debug: '*' - peerDependenciesMeta: - debug: - optional: true - dev: false - /follow-redirects@1.15.6: resolution: {integrity: sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==} engines: {node: '>=4.0'} @@ -2325,7 +2197,6 @@ packages: peerDependenciesMeta: debug: optional: true - dev: true /foreground-child@2.0.0: resolution: {integrity: sha512-dCIq9FpEcyQyXKCkyzmlPTFNgrCzPudOe+mhvJU5zAtlBnGVy2yKxtfsxK2tQBThwq225jcvBjpw1Gr40uzZCA==} @@ -2370,7 +2241,7 @@ packages: dezalgo: 1.0.4 hexoid: 1.0.0 once: 1.4.0 - qs: 6.12.1 + qs: 6.12.3 dev: true /forwarded@0.2.0: @@ -2379,7 +2250,7 @@ packages: dev: false /fresh@0.5.2: - resolution: {integrity: sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=} + resolution: {integrity: sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==} engines: {node: '>= 0.6'} dev: false @@ -2421,12 +2292,8 @@ packages: dev: true optional: true - /function-bind@1.1.1: - resolution: {integrity: sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==} - /function-bind@1.1.2: resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} - dev: true /gensync@1.0.0-beta.2: resolution: {integrity: sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==} @@ -2442,15 +2309,6 @@ packages: resolution: {integrity: sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==} dev: true - /get-intrinsic@1.2.1: - resolution: {integrity: sha512-2DcsyfABl+gVHEfCOaTrWgyt+tb6MSEGmKq+kI5HwLbIYgjgmMcV8KQ41uaKz1xxUcn9tJtgFbQUEVcEbd0FYw==} - dependencies: - function-bind: 1.1.1 - has: 1.0.3 - has-proto: 1.0.1 - has-symbols: 1.0.3 - dev: false - /get-intrinsic@1.2.4: resolution: {integrity: sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==} engines: {node: '>= 0.4'} @@ -2460,7 +2318,6 @@ packages: has-proto: 1.0.3 has-symbols: 1.0.3 hasown: 2.0.2 - dev: true /get-package-type@0.1.0: resolution: {integrity: sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==} @@ -2504,14 +2361,13 @@ packages: is-glob: 4.0.3 dev: true - /glob@10.4.2: - resolution: {integrity: sha512-GwMlUF6PkPo3Gk21UxkCohOv0PLcIXVtKyLlpEI28R/cO/4eNOdmLk3CMW1wROV/WR/EsZOWAfBbBOqYvs88/w==} - engines: {node: '>=16 || 14 >=14.18'} + /glob@10.4.5: + resolution: {integrity: sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==} hasBin: true dependencies: foreground-child: 3.2.1 - jackspeak: 3.4.0 - minimatch: 9.0.4 + jackspeak: 3.4.3 + minimatch: 9.0.5 minipass: 7.1.2 package-json-from-dist: 1.0.0 path-scurry: 1.11.1 @@ -2561,7 +2417,7 @@ packages: fs.realpath: 1.0.0 inflight: 1.0.6 inherits: 2.0.4 - minimatch: 5.0.1 + minimatch: 5.1.6 once: 1.4.0 dev: true @@ -2574,7 +2430,6 @@ packages: resolution: {integrity: sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==} dependencies: get-intrinsic: 1.2.4 - dev: true /graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} @@ -2595,7 +2450,7 @@ packages: source-map: 0.6.1 wordwrap: 1.0.0 optionalDependencies: - uglify-js: 3.18.0 + uglify-js: 3.19.1 dev: true /har-schema@2.0.0: @@ -2625,28 +2480,15 @@ packages: resolution: {integrity: sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==} dependencies: es-define-property: 1.0.0 - dev: true - - /has-proto@1.0.1: - resolution: {integrity: sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==} - engines: {node: '>= 0.4'} - dev: false /has-proto@1.0.3: resolution: {integrity: sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==} engines: {node: '>= 0.4'} - dev: true /has-symbols@1.0.3: resolution: {integrity: sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==} engines: {node: '>= 0.4'} - /has@1.0.3: - resolution: {integrity: sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==} - engines: {node: '>= 0.4.0'} - dependencies: - function-bind: 1.1.1 - /hasha@5.2.2: resolution: {integrity: sha512-Hrp5vIK/xr5SkeN2onO32H0MgNZ0f17HRNH39WfL0SYUNOTZ5Lz1TJ8Pajo/87dYGEFlLMm7mIc/k/s6Bvz9HQ==} engines: {node: '>=8'} @@ -2660,7 +2502,6 @@ packages: engines: {node: '>= 0.4'} dependencies: function-bind: 1.1.2 - dev: true /he@1.2.0: resolution: {integrity: sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==} @@ -2701,7 +2542,7 @@ packages: dependencies: assert-plus: 1.0.0 jsprim: 1.4.2 - sshpk: 1.17.0 + sshpk: 1.18.0 dev: false /human-signals@1.1.1: @@ -2753,8 +2594,8 @@ packages: engines: {node: '>= 4'} dev: true - /immer@10.0.2: - resolution: {integrity: sha512-Rx3CqeqQ19sxUtYV9CU911Vhy8/721wRFnJv3REVGWUmoAcIwzifTsdmJte/MV+0/XpM35LZdQMBGkRIoLPwQA==} + /immer@10.1.1: + resolution: {integrity: sha512-s2MPrmjovJcoMaHtx6K11Ra7oD05NT97w1IC5zpMkT6Atjr7H8LjaDd81iIxUYpMKSRRNMJE703M1Fhr/TctHw==} dev: false /import-fresh@2.0.0: @@ -2791,15 +2632,19 @@ packages: engines: {node: '>= 0.10'} dev: true + /ip-address@9.0.5: + resolution: {integrity: sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==} + engines: {node: '>= 12'} + dependencies: + jsbn: 1.1.0 + sprintf-js: 1.1.3 + dev: false + /ip-regex@2.1.0: resolution: {integrity: sha512-58yWmlHpp7VYfcdTwMTvwMmqx/Elfxjd9RXTDyMsbL7lLWmhMylLEqiYVLKuLzOZqVgiWXD9MfR62Vv89VRxkw==} engines: {node: '>=4'} dev: true - /ip@2.0.0: - resolution: {integrity: sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==} - dev: false - /ipaddr.js@1.9.1: resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==} engines: {node: '>= 0.10'} @@ -2820,14 +2665,8 @@ packages: binary-extensions: 2.3.0 dev: true - /is-core-module@2.12.1: - resolution: {integrity: sha512-Q4ZuBAe2FUsKtyQJoQHlvP8OvBERxO3jEmy1I7hcRXcJBGGHFh/aJBswbXuS9sgrDH2QUO8ilkwNPHvHMd8clg==} - dependencies: - has: 1.0.3 - dev: true - - /is-core-module@2.14.0: - resolution: {integrity: sha512-a5dFJih5ZLYlRtDc0dZWP7RiKr6xIKzmn/oAYCDvdLThadVgyJwlaoQPmRtMSpz+rk0OGAgIu+TcM9HUF0fk1A==} + /is-core-module@2.15.0: + resolution: {integrity: sha512-Dd+Lb2/zvk9SKy1TGCt1wFJFo/MWBPMX5x7KcvLajWTGuomczdQX61PvY5yK6SVACwpoexWo81IfFyoKY2QnTA==} engines: {node: '>= 0.4'} dependencies: hasown: 2.0.2 @@ -2948,7 +2787,7 @@ packages: resolution: {integrity: sha512-BXgQl9kf4WTCPCCpmFGoJkz/+uhvm7h7PFKUYxh7qarQd3ER33vHG//qaE8eN25l07YqZPpHXU9I09l/RD5aGQ==} engines: {node: '>=8'} dependencies: - '@babel/core': 7.24.7 + '@babel/core': 7.25.2 '@istanbuljs/schema': 0.1.3 istanbul-lib-coverage: 3.2.2 semver: 6.3.1 @@ -2981,7 +2820,7 @@ packages: resolution: {integrity: sha512-n3s8EwkdFIJCG3BPKBYvskgXGoy88ARzvegkitk60NxRdwltLOTaH7CUiMRXvwYorl0Q712iEjcWB+fK/MrWVw==} engines: {node: '>=10'} dependencies: - debug: 4.3.5 + debug: 4.3.6(supports-color@8.1.1) istanbul-lib-coverage: 3.2.2 source-map: 0.6.1 transitivePeerDependencies: @@ -2996,9 +2835,8 @@ packages: istanbul-lib-report: 3.0.1 dev: true - /jackspeak@3.4.0: - resolution: {integrity: sha512-JVYhQnN59LVPFCEcVa2C3CrEKYacvjRfqIQl+h8oi91aLYQVWRYbxjPcv1bUiUy/kLmQaANrYfNMCO3kuEDHfw==} - engines: {node: '>=14'} + /jackspeak@3.4.3: + resolution: {integrity: sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==} dependencies: '@isaacs/cliui': 8.0.2 optionalDependencies: @@ -3050,6 +2888,10 @@ packages: resolution: {integrity: sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg==} dev: false + /jsbn@1.1.0: + resolution: {integrity: sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==} + dev: false + /jsesc@2.5.2: resolution: {integrity: sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==} engines: {node: '>=4'} @@ -3122,7 +2964,7 @@ packages: lodash.isstring: 4.0.1 lodash.once: 4.1.1 ms: 2.1.3 - semver: 5.7.1 + semver: 5.7.2 dev: false /jsprim@1.4.2: @@ -3248,19 +3090,8 @@ packages: is-unicode-supported: 0.1.0 dev: true - /logform@2.5.1: - resolution: {integrity: sha512-9FyqAm9o9NKKfiAKfZoYo9bGXXuwMkxQiQttkT4YjjVtQVIQtK6LmVtlxmCaFswo6N4AfEkHqZTV0taDtPotNg==} - dependencies: - '@colors/colors': 1.5.0 - '@types/triple-beam': 1.3.2 - fecha: 4.2.3 - ms: 2.1.3 - safe-stable-stringify: 2.4.3 - triple-beam: 1.3.0 - dev: false - - /logform@2.6.0: - resolution: {integrity: sha512-1ulHeNPp6k/LD8H91o7VYFBng5i1BDE7HoKxVbZiGFidS1Rj65qcywLxX+pVfAPoQJEjRdvKcusKwOupHCVOVQ==} + /logform@2.6.1: + resolution: {integrity: sha512-CdaO738xRapbKIMVn2m4F6KTj4j7ooJ8POVnebSgKo3KBz5axNXRAL7ZdRjIV6NOr2Uf4vjtRkxrFETOioCqSA==} engines: {node: '>= 12.0.0'} dependencies: '@colors/colors': 1.6.0 @@ -3277,9 +3108,8 @@ packages: get-func-name: 2.0.2 dev: true - /lru-cache@10.2.2: - resolution: {integrity: sha512-9hp3Vp2/hFQUiIwKo8XCeFVnrg8Pk3TYNPIR7tJADKi5YfcF7vEaK7avFHTlSy3kOKYaJQaalfEo6YuXdceBOQ==} - engines: {node: 14 || >=16.14} + /lru-cache@10.4.3: + resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} dev: true /lru-cache@5.1.1: @@ -3298,7 +3128,7 @@ packages: /lru-queue@0.1.0: resolution: {integrity: sha512-BpdYkt9EvGl8OfWHDQPISVpcl5xZthb+XPsbELj5AQXxIC8IriDZIQYjBJPEm5rS420sjZ0TLEzRcq5KdBhYrQ==} dependencies: - es5-ext: 0.10.62 + es5-ext: 0.10.64 dev: false /lunr@2.3.9: @@ -3316,7 +3146,7 @@ packages: resolution: {integrity: sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==} engines: {node: '>=10'} dependencies: - semver: 7.6.2 + semver: 7.6.3 dev: true /make-error@1.3.6: @@ -3330,21 +3160,22 @@ packages: dev: true /media-typer@0.3.0: - resolution: {integrity: sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=} + resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==} engines: {node: '>= 0.6'} dev: false - /memoizee@0.4.15: - resolution: {integrity: sha512-UBWmJpLZd5STPm7PMUlOw/TSy972M+z8gcyQ5veOnSDRREz/0bmpyTfKt3/51DhEBqCZQn1udM/5flcSPYhkdQ==} + /memoizee@0.4.17: + resolution: {integrity: sha512-DGqD7Hjpi/1or4F/aYAspXKNm5Yili0QDAFAY4QYvpqpgiY6+1jOfqpmByzjxbWd/T9mChbCArXAbDAsTm5oXA==} + engines: {node: '>=0.12'} dependencies: - d: 1.0.1 - es5-ext: 0.10.62 + d: 1.0.2 + es5-ext: 0.10.64 es6-weak-map: 2.0.3 event-emitter: 0.3.5 is-promise: 2.2.2 lru-queue: 0.1.0 next-tick: 1.1.0 - timers-ext: 0.1.7 + timers-ext: 0.1.8 dev: false /memory-pager@1.5.0: @@ -3354,7 +3185,7 @@ packages: optional: true /merge-descriptors@1.0.1: - resolution: {integrity: sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=} + resolution: {integrity: sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==} dev: false /merge-stream@2.0.0: @@ -3410,15 +3241,15 @@ packages: brace-expansion: 1.1.11 dev: true - /minimatch@5.0.1: - resolution: {integrity: sha512-nLDxIFRyhDblz3qMuq+SoRZED4+miJ/G+tdDrjkkkRnjAsBexeGpgjLEQ0blJy7rHhR2b93rhQY4SvyWu9v03g==} + /minimatch@5.1.6: + resolution: {integrity: sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==} engines: {node: '>=10'} dependencies: brace-expansion: 2.0.1 dev: true - /minimatch@9.0.4: - resolution: {integrity: sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==} + /minimatch@9.0.5: + resolution: {integrity: sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==} engines: {node: '>=16 || 14 >=14.17'} dependencies: brace-expansion: 2.0.1 @@ -3444,30 +3275,30 @@ packages: minimist: 1.2.8 dev: true - /mocha@10.4.0: - resolution: {integrity: sha512-eqhGB8JKapEYcC4ytX/xrzKforgEc3j1pGlAXVy3eRwrtAy5/nIfT1SvgGzfN0XZZxeLq0aQWkOUAmqIJiv+bA==} + /mocha@10.7.0: + resolution: {integrity: sha512-v8/rBWr2VO5YkspYINnvu81inSz2y3ODJrhO175/Exzor1RcEZZkizgE2A+w/CAXXoESS8Kys5E62dOHGHzULA==} engines: {node: '>= 14.0.0'} hasBin: true dependencies: - ansi-colors: 4.1.1 + ansi-colors: 4.1.3 browser-stdout: 1.3.1 - chokidar: 3.5.3 - debug: 4.3.4(supports-color@8.1.1) - diff: 5.0.0 + chokidar: 3.6.0 + debug: 4.3.6(supports-color@8.1.1) + diff: 5.2.0 escape-string-regexp: 4.0.0 find-up: 5.0.0 glob: 8.1.0 he: 1.2.0 js-yaml: 4.1.0 log-symbols: 4.1.0 - minimatch: 5.0.1 + minimatch: 5.1.6 ms: 2.1.3 - serialize-javascript: 6.0.0 + serialize-javascript: 6.0.2 strip-json-comments: 3.1.1 supports-color: 8.1.1 - workerpool: 6.2.1 + workerpool: 6.5.1 yargs: 16.2.0 - yargs-parser: 20.2.4 + yargs-parser: 20.2.9 yargs-unparser: 2.0.0 dev: true @@ -3510,41 +3341,49 @@ packages: whatwg-url: 11.0.0 dev: false - /mongodb@5.6.0: - resolution: {integrity: sha512-z8qVs9NfobHJm6uzK56XBZF8XwM9H294iRnB7wNjF0SnY93si5HPziIJn+qqvUR5QOff/4L0gCD6SShdR/GtVQ==} + /mongodb@5.9.2: + resolution: {integrity: sha512-H60HecKO4Bc+7dhOv4sJlgvenK4fQNqqUIlXxZYQNbfEWSALGAwGoyJd/0Qwk4TttFXUOHJ2ZJQe/52ScaUwtQ==} engines: {node: '>=14.20.1'} peerDependencies: - '@aws-sdk/credential-providers': ^3.201.0 + '@aws-sdk/credential-providers': ^3.188.0 + '@mongodb-js/zstd': ^1.0.0 + kerberos: ^1.0.0 || ^2.0.0 mongodb-client-encryption: '>=2.3.0 <3' snappy: ^7.2.2 peerDependenciesMeta: '@aws-sdk/credential-providers': optional: true + '@mongodb-js/zstd': + optional: true + kerberos: + optional: true mongodb-client-encryption: optional: true snappy: optional: true dependencies: - bson: 5.4.0 + bson: 5.5.1 mongodb-connection-string-url: 2.6.0 - socks: 2.7.1 + socks: 2.8.3 optionalDependencies: - saslprep: 1.0.3 + '@mongodb-js/saslprep': 1.1.8 dev: false - /mongoose@7.3.2: - resolution: {integrity: sha512-Z86m5ASwYYFyT++wPQTtuTl5Jh052w6G1IM8LxPu/6iuqxQo6nUOaEoGZfMy0ovw3Dyw3415Jue3pYXkRqPkfA==} + /mongoose@7.8.0: + resolution: {integrity: sha512-wLAP7xYz+tEnzy4VsZyMJ1mfaSIwfaeoSQ55ZVovFkdh1FVta6VNSVFCpJMzEinMJsRzTbZTcD4pND9J5aDiyA==} engines: {node: '>=14.20.1'} dependencies: - bson: 5.4.0 + bson: 5.5.1 kareem: 2.5.1 - mongodb: 5.6.0 + mongodb: 5.9.2 mpath: 0.9.0 mquery: 5.0.0 ms: 2.1.3 sift: 16.0.1 transitivePeerDependencies: - '@aws-sdk/credential-providers' + - '@mongodb-js/zstd' + - kerberos - mongodb-client-encryption - snappy - supports-color @@ -3559,7 +3398,7 @@ packages: resolution: {integrity: sha512-iQMncpmEK8R8ncT8HJGsGc9Dsp8xcgYMVSbs5jgnm1lFHTZqMJTUWTDx1LBO8+mK3tPNZWFLBghQEIOULSTHZg==} engines: {node: '>=14.0.0'} dependencies: - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.6(supports-color@8.1.1) transitivePeerDependencies: - supports-color dev: false @@ -3654,8 +3493,8 @@ packages: process-on-spawn: 1.0.0 dev: true - /node-releases@2.0.14: - resolution: {integrity: sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw==} + /node-releases@2.0.18: + resolution: {integrity: sha512-d9VeXT4SJ7ZeOqGX6R5EM022wpL+eWPooLI+5UpWn2jCT1aosUQEhQP214x33Wkwx3JQMvIm+tIoVOdodFS40g==} dev: true /node-vault@0.9.22: @@ -3675,31 +3514,24 @@ packages: engines: {node: '>=8.10.0'} hasBin: true dependencies: - chokidar: 3.5.3 + chokidar: 3.6.0 debug: 3.2.7(supports-color@5.5.0) ignore-by-default: 1.0.1 minimatch: 3.1.2 pstree.remy: 1.1.8 - semver: 5.7.1 + semver: 5.7.2 simple-update-notifier: 1.1.0 supports-color: 5.5.0 - touch: 3.1.0 + touch: 3.1.1 undefsafe: 2.0.5 dev: true - /nopt@1.0.10: - resolution: {integrity: sha512-NWmpvLSqUrgrAC9HCuxEvb+PSloHpqVu+FqcO4eeF2h5qYRhA7ev6KvelyQAKtegUbC6RypJnlEOhd8vloNKYg==} - hasBin: true - dependencies: - abbrev: 1.1.1 - dev: true - /normalize-package-data@2.5.0: resolution: {integrity: sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA==} dependencies: hosted-git-info: 2.8.9 - resolve: 1.22.2 - semver: 5.7.1 + resolve: 1.22.8 + semver: 5.7.2 validate-npm-package-license: 3.0.4 dev: true @@ -3826,13 +3658,9 @@ packages: resolution: {integrity: sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==} dev: false - /object-inspect@1.12.3: - resolution: {integrity: sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g==} - dev: false - - /object-inspect@1.13.1: - resolution: {integrity: sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==} - dev: true + /object-inspect@1.13.2: + resolution: {integrity: sha512-IRZSRuzJiynemAXPYtPe5BoI/RESNYR7TYm50MC5Mqbd3Jmw5y790sErYw3V6SryFJD64b74qQQs9wn5Bg/k3g==} + engines: {node: '>= 0.4'} /on-finished@2.4.1: resolution: {integrity: sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==} @@ -3952,7 +3780,7 @@ packages: resolution: {integrity: sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==} engines: {node: '>=8'} dependencies: - '@babel/code-frame': 7.22.5 + '@babel/code-frame': 7.24.7 error-ex: 1.3.2 json-parse-even-better-errors: 2.3.1 lines-and-columns: 1.2.4 @@ -3996,7 +3824,7 @@ packages: resolution: {integrity: sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==} engines: {node: '>=16 || 14 >=14.18'} dependencies: - lru-cache: 10.2.2 + lru-cache: 10.4.3 minipass: 7.1.2 dev: true @@ -4054,14 +3882,8 @@ packages: hasBin: true dev: true - /prettier@3.0.0: - resolution: {integrity: sha512-zBf5eHpwHOGPC47h0zrPyNn+eAEIdEzfywMoYn2XPi0P44Zp0tSq64rq0xAREh4auw2cJZHo9QUob+NqCQky4g==} - engines: {node: '>=14'} - hasBin: true - dev: true - - /prettier@3.3.2: - resolution: {integrity: sha512-rAVeHYMcv8ATV5d508CFdn+8/pHPpXeIid1DdrPwXnaAdH7cqjVbpJaT5eq4yRAFU/lsbwYwSF/n5iNrdJHPQA==} + /prettier@3.3.3: + resolution: {integrity: sha512-i2tDNA0O5IrMO757lfrdQZCc2jPNDVntV0m/+4whiDfWaTKfMNgR7Qz0NAeGz/nRqF4m5/6CLzbP4/liHt12Ew==} engines: {node: '>=14'} hasBin: true dev: true @@ -4149,8 +3971,8 @@ packages: resolution: {integrity: sha512-jmYNElW7yvO7TV33CjSmvSiE2yco3bV2czu/OzDKdMNVZQWfxCblURLhf+47syQRBntjfLdd/H0egrzIG+oaFQ==} dev: false - /punycode@2.3.0: - resolution: {integrity: sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==} + /punycode@2.3.1: + resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} engines: {node: '>=6'} dev: false @@ -4158,11 +3980,11 @@ packages: resolution: {integrity: sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==} engines: {node: '>=0.6'} dependencies: - side-channel: 1.0.4 + side-channel: 1.0.6 dev: false - /qs@6.12.1: - resolution: {integrity: sha512-zWmv4RSuB9r2mYQw3zxQuHWeU+42aKi1wWig/j4ele4ygELZ7PEO6MM7rim9oAQH2A5MWfsAVf/jPvTPgCbvUQ==} + /qs@6.12.3: + resolution: {integrity: sha512-AWJm14H1vVaO/iNZ4/hO+HyaTehuy9nRqVdkTqlJt0HWvBiBIEXFmb4C0DGeYo3Xes9rrEW+TxHsaigCbN5ICQ==} engines: {node: '>=0.6'} dependencies: side-channel: 1.0.6 @@ -4184,16 +4006,6 @@ packages: engines: {node: '>= 0.6'} dev: false - /raw-body@2.5.1: - resolution: {integrity: sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==} - engines: {node: '>= 0.8'} - dependencies: - bytes: 3.1.2 - http-errors: 2.0.0 - iconv-lite: 0.4.24 - unpipe: 1.0.0 - dev: false - /raw-body@2.5.2: resolution: {integrity: sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==} engines: {node: '>= 0.8'} @@ -4221,7 +4033,7 @@ packages: resolution: {integrity: sha512-Ug69mNOpfvKDAc2Q8DRpMjjzdtrnv9HcSMX+4VsZxD1aZ6ZzrIE7rlzXBtWTyhULSMKg076AW6WR5iZpD0JiOg==} engines: {node: '>=8'} dependencies: - '@types/normalize-package-data': 2.4.1 + '@types/normalize-package-data': 2.4.4 normalize-package-data: 2.5.0 parse-json: 5.2.0 type-fest: 0.6.0 @@ -4277,8 +4089,8 @@ packages: resolve: 1.22.8 dev: true - /regenerator-runtime@0.13.11: - resolution: {integrity: sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==} + /regenerator-runtime@0.14.1: + resolution: {integrity: sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==} dev: true /release-zalgo@1.0.0: @@ -4317,7 +4129,7 @@ packages: deprecated: request has been deprecated, see https://github.com/request/request/issues/3142 dependencies: aws-sign2: 0.7.0 - aws4: 1.12.0 + aws4: 1.13.0 caseless: 0.12.0 combined-stream: 1.0.8 extend: 3.0.2 @@ -4362,20 +4174,11 @@ packages: engines: {node: '>=8'} dev: true - /resolve@1.22.2: - resolution: {integrity: sha512-Sb+mjNHOULsBv818T40qSPeRiuWLyaGMa5ewydRLFimneixmVy2zdivRl+AF6jaYPC8ERxGDmFSiqui6SfPd+g==} - hasBin: true - dependencies: - is-core-module: 2.12.1 - path-parse: 1.0.7 - supports-preserve-symlinks-flag: 1.0.0 - dev: true - /resolve@1.22.8: resolution: {integrity: sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==} hasBin: true dependencies: - is-core-module: 2.14.0 + is-core-module: 2.15.0 path-parse: 1.0.7 supports-preserve-symlinks-flag: 1.0.0 dev: true @@ -4396,12 +4199,12 @@ packages: glob: 7.2.3 dev: true - /rimraf@5.0.7: - resolution: {integrity: sha512-nV6YcJo5wbLW77m+8KjH8aB/7/rxQy9SZ0HY5shnwULfS+9nmTtVXAJET5NdZmCzA4fPI/Hm1wo/Po/4mopOdg==} - engines: {node: '>=14.18'} + /rimraf@5.0.9: + resolution: {integrity: sha512-3i7b8OcswU6CpU8Ej89quJD4O98id7TtVM5U4Mybh84zQXdrFmDLouWBEEaD/QfO3gDDfH+AGFCGsR7kngzQnA==} + engines: {node: 14 >=14.20 || 16 >=16.20 || >=18} hasBin: true dependencies: - glob: 10.4.2 + glob: 10.4.5 dev: true /run-node@1.0.0: @@ -4433,27 +4236,13 @@ packages: resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} dev: false - /saslprep@1.0.3: - resolution: {integrity: sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag==} - engines: {node: '>=6'} - requiresBuild: true - dependencies: - sparse-bitfield: 3.0.3 - dev: false - optional: true - /semver-compare@1.0.0: resolution: {integrity: sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==} dev: true - /semver@5.7.1: - resolution: {integrity: sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==} - hasBin: true - /semver@5.7.2: resolution: {integrity: sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==} hasBin: true - dev: true /semver@6.3.1: resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==} @@ -4473,8 +4262,8 @@ packages: lru-cache: 6.0.0 dev: true - /semver@7.6.2: - resolution: {integrity: sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==} + /semver@7.6.3: + resolution: {integrity: sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==} engines: {node: '>=10'} hasBin: true dev: true @@ -4506,8 +4295,8 @@ packages: randombytes: 2.1.0 dev: true - /serialize-javascript@6.0.0: - resolution: {integrity: sha512-Qr3TosvguFt8ePWqsvRfrKyQXIiW+nGbYpy8XK24NQHE83caxWt+mIymTT19DGFbNWNLfEwsrkSmN64lVWB9ag==} + /serialize-javascript@6.0.2: + resolution: {integrity: sha512-Saa1xPByTTq2gdeFZYLLo+RFE35NHZkAbqZeWNd3BpzppeVisAqpDjcp8dyf6uIvEqJRd46jemmyA4iFIeVk8g==} dependencies: randombytes: 2.1.0 dev: true @@ -4538,7 +4327,6 @@ packages: get-intrinsic: 1.2.4 gopd: 1.0.1 has-property-descriptors: 1.0.2 - dev: true /setprototypeof@1.2.0: resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} @@ -4578,14 +4366,6 @@ packages: rechoir: 0.6.2 dev: true - /side-channel@1.0.4: - resolution: {integrity: sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==} - dependencies: - call-bind: 1.0.2 - get-intrinsic: 1.2.1 - object-inspect: 1.12.3 - dev: false - /side-channel@1.0.6: resolution: {integrity: sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==} engines: {node: '>= 0.4'} @@ -4593,8 +4373,7 @@ packages: call-bind: 1.0.7 es-errors: 1.3.0 get-intrinsic: 1.2.4 - object-inspect: 1.13.1 - dev: true + object-inspect: 1.13.2 /sift@16.0.1: resolution: {integrity: sha512-Wv6BjQ5zbhW7VFefWusVP33T/EM0vYikCaQ2qR8yULbsilAT8/wQaXvuQ3ptGLpoKx+lihJE3y2UTgKDyyNHZQ==} @@ -4644,11 +4423,11 @@ packages: engines: {node: '>= 6.0.0', npm: '>= 3.0.0'} dev: false - /socks@2.7.1: - resolution: {integrity: sha512-7maUZy1N7uo6+WVEX6psASxtNlKaNVMlGQKkG/63nEDdLOWNbiUMoLK7X4uYoLhQstau72mLgfEWcXcwsaHbYQ==} - engines: {node: '>= 10.13.0', npm: '>= 3.0.0'} + /socks@2.8.3: + resolution: {integrity: sha512-l5x7VUUWbjVFbafGLxPWkYsHIhEvmF85tbIeFZWc8ZPtoMyybuEhL7Jye/ooC4/d48FgOjSJXgsF/AJPYCW8Zw==} + engines: {node: '>= 10.0.0', npm: '>= 3.0.0'} dependencies: - ip: 2.0.0 + ip-address: 9.0.5 smart-buffer: 4.2.0 dev: false @@ -4672,8 +4451,8 @@ packages: dev: false optional: true - /spawn-command@0.0.2-1: - resolution: {integrity: sha512-n98l9E2RMSJ9ON1AKisHzz7V42VDiBQGY6PB1BwRglz99wpVsSuGzQ+jOi6lFXBGVTCrRpltvjm+/XA+tpeJrg==} + /spawn-command@0.0.2: + resolution: {integrity: sha512-zC8zGoGkmc8J9ndvml8Xksr1Amk9qBujgbF0JAIWO7kXr43w0h/0GJNM/Vustixu+YE8N/MTrQ7N31FvHUACxQ==} dev: true /spawn-wrap@2.0.0: @@ -4692,22 +4471,22 @@ packages: resolution: {integrity: sha512-kN9dJbvnySHULIluDHy32WHRUu3Og7B9sbY7tsFLctQkIqnMh3hErYgdMjTYuqmcXX+lK5T1lnUt3G7zNswmZA==} dependencies: spdx-expression-parse: 3.0.1 - spdx-license-ids: 3.0.13 + spdx-license-ids: 3.0.18 dev: true - /spdx-exceptions@2.3.0: - resolution: {integrity: sha512-/tTrYOC7PPI1nUAgx34hUpqXuyJG+DTHJTnIULG4rDygi4xu/tfgmq1e1cIRwRzwZgo4NLySi+ricLkZkw4i5A==} + /spdx-exceptions@2.5.0: + resolution: {integrity: sha512-PiU42r+xO4UbUS1buo3LPJkjlO7430Xn5SVAhdpzzsPHsjbYVflnnFdATgabnLude+Cqu25p6N+g2lw/PFsa4w==} dev: true /spdx-expression-parse@3.0.1: resolution: {integrity: sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q==} dependencies: - spdx-exceptions: 2.3.0 - spdx-license-ids: 3.0.13 + spdx-exceptions: 2.5.0 + spdx-license-ids: 3.0.18 dev: true - /spdx-license-ids@3.0.13: - resolution: {integrity: sha512-XkD+zwiqXHikFZm4AX/7JSCXA98U5Db4AFd5XUg/+9UNtnH75+Z9KxtpYiJZx36mUDVOwH83pl7yvCer6ewM3w==} + /spdx-license-ids@3.0.18: + resolution: {integrity: sha512-xxRs31BqRYHwiMzudOrpSiHtZ8i/GeionCBDSilhYRj+9gIcI8wCZTlXZKu9vZIVqViP3dcp9qE5G6AlIaD+TQ==} dev: true /split-ca@1.0.1: @@ -4718,8 +4497,12 @@ packages: resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} dev: true - /sshpk@1.17.0: - resolution: {integrity: sha512-/9HIEs1ZXGhSPE8X6Ccm7Nam1z8KcoCqPdI7ecm1N33EzAetWahvQWVqLZtaZQ+IDKX4IyA2o0gBzqIMkAagHQ==} + /sprintf-js@1.1.3: + resolution: {integrity: sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==} + dev: false + + /sshpk@1.18.0: + resolution: {integrity: sha512-2p2KJZTSqQ/I3+HX42EpYOa2l3f8Erv8MWKsy2I9uf4wA7yFIkXRffYdsx86y6z4vHtV8u7g+pPlr8/4ouAxsQ==} engines: {node: '>=0.10.0'} hasBin: true dependencies: @@ -4874,14 +4657,14 @@ packages: dependencies: component-emitter: 1.3.1 cookiejar: 2.1.4 - debug: 4.3.5 + debug: 4.3.6(supports-color@8.1.1) fast-safe-stringify: 2.1.1 form-data: 4.0.0 formidable: 2.1.2 methods: 1.1.2 mime: 2.6.0 - qs: 6.12.1 - semver: 7.6.2 + qs: 6.12.3 + semver: 7.6.3 transitivePeerDependencies: - supports-color dev: true @@ -4918,18 +4701,18 @@ packages: engines: {node: '>= 0.4'} dev: true - /swagger-ui-dist@5.1.0: - resolution: {integrity: sha512-c1KmAjuVODxw+vwkNLALQZrgdlBAuBbr2xSPfYrJgseEi7gFKcTvShysPmyuDI4kcUa1+5rFpjWvXdusKY74mg==} + /swagger-ui-dist@5.17.14: + resolution: {integrity: sha512-CVbSfaLpstV65OnSjbXfVd6Sta3q3F7Cj/yYuvHMp1P90LztOLs6PfUnKEVAeiIVQt9u2SaPwv0LiH/OyMjHRw==} dev: false - /swagger-ui-express@4.6.3(express@4.18.2): + /swagger-ui-express@4.6.3(express@4.19.2): resolution: {integrity: sha512-CDje4PndhTD2HkgyKH3pab+LKspDeB/NhPN2OF1j+piYIamQqBYwAXWESOT1Yju2xFg51bRW9sUng2WxDjzArw==} engines: {node: '>= v0.10.32'} peerDependencies: express: '>=4.0.0 || >=5.0.0-beta' dependencies: - express: 4.18.2 - swagger-ui-dist: 5.1.0 + express: 4.19.2 + swagger-ui-dist: 5.17.14 dev: false /tar-fs@1.16.3: @@ -4987,7 +4770,7 @@ packages: resolution: {integrity: sha512-cfy7GYd0uanjkgVFlcDLtjWUVQLmTHke5pa1djTBW/ppbv5HfHkoDt6cI1JtEj8NKjNE4BoJx+au2/eFQOq4HQ==} dependencies: byline: 5.0.0 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.6(supports-color@8.1.1) dockerode: 2.5.8 get-port: 4.2.0 node-duration: 1.0.4 @@ -5005,10 +4788,11 @@ packages: resolution: {integrity: sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==} dev: true - /timers-ext@0.1.7: - resolution: {integrity: sha512-b85NUNzTSdodShTIbky6ZF02e8STtVVfD+fu4aXXShEELpozH+bCpJLYMPZbsABN2wDH7fJpqIoXxJpzbf0NqQ==} + /timers-ext@0.1.8: + resolution: {integrity: sha512-wFH7+SEAcKfJpfLPkrgMPvvwnEtj8W4IurvEyrKsDleXnKLCDw71w8jltvfLa8Rm4qQxxT4jmDBYbJG/z7qoww==} + engines: {node: '>=0.12'} dependencies: - es5-ext: 0.10.62 + es5-ext: 0.10.64 next-tick: 1.1.0 dev: false @@ -5038,11 +4822,9 @@ packages: engines: {node: '>=0.6'} dev: false - /touch@3.1.0: - resolution: {integrity: sha512-WBx8Uy5TLtOSRtIq+M03/sKDrXCLHxwDcquSP2c43Le03/9serjQBIztjRz6FkJez9D/hleyAXTBGLwwZUw9lA==} + /touch@3.1.1: + resolution: {integrity: sha512-r0eojU4bI8MnHr8c5bNo7lJDdI2qXlWWJk6a9EAFG7vbhTjElYhBVS3/miuE0uOuoLdb8Mc/rVfsmm6eo5o9GA==} hasBin: true - dependencies: - nopt: 1.0.10 dev: true /tough-cookie@2.4.3: @@ -5058,7 +4840,7 @@ packages: engines: {node: '>=0.8'} dependencies: psl: 1.9.0 - punycode: 2.3.0 + punycode: 2.3.1 dev: false /tr46@0.0.3: @@ -5069,7 +4851,7 @@ packages: resolution: {integrity: sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==} engines: {node: '>=12'} dependencies: - punycode: 2.3.0 + punycode: 2.3.1 dev: false /tree-kill@1.2.2: @@ -5077,16 +4859,12 @@ packages: hasBin: true dev: true - /triple-beam@1.3.0: - resolution: {integrity: sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw==} - dev: false - /triple-beam@1.4.1: resolution: {integrity: sha512-aZbgViZrg1QNcG+LULa7nhZpJTZSLm/mXnHXnbAbjmN5aSa0y7V+wvv6+4WaBtpISJzThKy+PIPxc1Nq1EJ9mg==} engines: {node: '>= 14.0.0'} dev: false - /ts-node@10.9.2(@types/node@20.14.7)(typescript@5.5.2): + /ts-node@10.9.2(@types/node@20.14.13)(typescript@5.5.4): resolution: {integrity: sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==} hasBin: true peerDependencies: @@ -5105,19 +4883,19 @@ packages: '@tsconfig/node12': 1.0.11 '@tsconfig/node14': 1.0.3 '@tsconfig/node16': 1.0.4 - '@types/node': 20.14.7 - acorn: 8.12.0 + '@types/node': 20.14.13 + acorn: 8.12.1 acorn-walk: 8.3.3 arg: 4.1.3 create-require: 1.1.1 diff: 4.0.2 make-error: 1.3.6 - typescript: 5.5.2 + typescript: 5.5.4 v8-compile-cache-lib: 3.0.1 yn: 3.1.1 dev: true - /ts-node@9.1.1(typescript@5.5.2): + /ts-node@9.1.1(typescript@5.5.4): resolution: {integrity: sha512-hPlt7ZACERQGf03M253ytLY3dHbGNGrAq9qIHWUY9XHYl1z7wYngSr3OQ5xmui8o2AaxsONxIzjafLUiWBo1Fg==} engines: {node: '>=10.0.0'} hasBin: true @@ -5129,7 +4907,7 @@ packages: diff: 4.0.2 make-error: 1.3.6 source-map-support: 0.5.21 - typescript: 5.5.2 + typescript: 5.5.4 yn: 3.1.1 dev: true @@ -5150,7 +4928,7 @@ packages: resolution: {integrity: sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==} dev: true - /tslint@6.1.3(typescript@5.5.2): + /tslint@6.1.3(typescript@5.5.4): resolution: {integrity: sha512-IbR4nkT96EQOvKE2PW/djGz8iGNeJ4rF2mBfiYaR/nvUWYKJhLwimoJKgjIFEIDibBtOevj7BqCRL4oHeWWUCg==} engines: {node: '>=4.8.0'} deprecated: TSLint has been deprecated in favor of ESLint. Please see https://github.com/palantir/tslint/issues/4534 for more information. @@ -5170,17 +4948,17 @@ packages: resolve: 1.22.8 semver: 5.7.2 tslib: 1.14.1 - tsutils: 2.29.0(typescript@5.5.2) - typescript: 5.5.2 + tsutils: 2.29.0(typescript@5.5.4) + typescript: 5.5.4 dev: true - /tsutils@2.29.0(typescript@5.5.2): + /tsutils@2.29.0(typescript@5.5.4): resolution: {integrity: sha512-g5JVHCIJwzfISaXpXE1qvNalca5Jwob6FjI4AoPlqMusJ6ftFE7IkkFoMhVLRgK+4Kx3gkzb8UZK5t5yTTvEmA==} peerDependencies: typescript: '>=2.1.0 || >=2.1.0-dev || >=2.2.0-dev || >=2.3.0-dev || >=2.4.0-dev || >=2.5.0-dev || >=2.6.0-dev || >=2.7.0-dev || >=2.8.0-dev || >=2.9.0-dev || >= 3.0.0-dev || >= 3.1.0-dev' dependencies: tslib: 1.14.1 - typescript: 5.5.2 + typescript: 5.5.4 dev: true /tunnel-agent@0.6.0: @@ -5203,6 +4981,11 @@ packages: engines: {node: '>=4'} dev: true + /type-detect@4.1.0: + resolution: {integrity: sha512-Acylog8/luQ8L7il+geoSxhEkazvkslg7PSNKOX59mbB9cOveP5aq9h74Y7YU8yDpJwetzQQrfIwtf4Wp4LKcw==} + engines: {node: '>=4'} + dev: true + /type-fest@0.6.0: resolution: {integrity: sha512-q+MB8nYR1KDLrgr4G5yemftpMC7/QLqVndBmEEdqzmNj5dcFOO4Oo8qlwZE3ULT3+Zim1F8Kq4cBnikNhlCMlg==} engines: {node: '>=8'} @@ -5221,12 +5004,8 @@ packages: mime-types: 2.1.35 dev: false - /type@1.2.0: - resolution: {integrity: sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg==} - dev: false - - /type@2.7.2: - resolution: {integrity: sha512-dzlvlNlt6AXU7EBSfpAscydQ7gXB+pPGsPnfJnZpiNJBDj7IaJzQlBZYGdEi4R9HmPdBv2XmWJ6YUtoTa7lmCw==} + /type@2.7.3: + resolution: {integrity: sha512-8j+1QmAbPvLZow5Qpi6NCaN8FB60p/6x8/vfNqOk/hC+HuvFZhL4+WfekuhQLiqFZXOgQdrs3B+XxEmCc6b3FQ==} dev: false /typedarray-to-buffer@3.1.5: @@ -5246,7 +5025,7 @@ packages: lunr: 2.3.9 dev: true - /typedoc@0.17.8(typescript@5.5.2): + /typedoc@0.17.8(typescript@5.5.4): resolution: {integrity: sha512-/OyrHCJ8jtzu+QZ+771YaxQ9s4g5Z3XsQE3Ma7q+BL392xxBn4UMvvCdVnqKC2T/dz03/VXSLVKOP3lHmDdc/w==} engines: {node: '>= 8.0.0'} hasBin: true @@ -5263,17 +5042,17 @@ packages: progress: 2.0.3 shelljs: 0.8.5 typedoc-default-themes: 0.10.2 - typescript: 5.5.2 + typescript: 5.5.4 dev: true - /typescript@5.5.2: - resolution: {integrity: sha512-NcRtPEOsPFFWjobJEtfihkLCZCXZt/os3zf8nTxjVH3RvTSxjrCamJpbExGvYOF+tFHc3pA65qpdwPbzjohhew==} + /typescript@5.5.4: + resolution: {integrity: sha512-Mtq29sKDAEYP7aljRgtPOpTvOfbwRWlS6dPRzwjdE+C0R4brX/GUyhHSecbHMFLNBLcJIPt9nl9yG5TZ1weH+Q==} engines: {node: '>=14.17'} hasBin: true dev: true - /uglify-js@3.18.0: - resolution: {integrity: sha512-SyVVbcNBCk0dzr9XL/R/ySrmYf0s372K6/hFklzgcp2lBFyXtw4I7BOdDjlLhE1aVqaI/SHWXWmYdlZxuyF38A==} + /uglify-js@3.19.1: + resolution: {integrity: sha512-y/2wiW+ceTYR2TSSptAhfnEtpLaQ4Ups5zrjB2d3kuVxHj16j/QJwPl5PvuGy9uARb39J0+iKxcRPvtpsx4A4A==} engines: {node: '>=0.8.0'} hasBin: true requiresBuild: true @@ -5286,6 +5065,10 @@ packages: /undici-types@5.26.5: resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} + dev: true + + /undici-types@6.11.1: + resolution: {integrity: sha512-mIDEX2ek50x0OlRgxryxsenE5XaQD4on5U2inY7RApK3SOJpofyw7uW2AyfMKkhAxXIceo2DeWGVGwyvng1GNQ==} /universalify@0.1.2: resolution: {integrity: sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==} @@ -5302,13 +5085,13 @@ packages: engines: {node: '>= 0.8'} dev: false - /update-browserslist-db@1.0.16(browserslist@4.23.1): - resolution: {integrity: sha512-KVbTxlBYlckhF5wgfyZXTWnMn7MMZjMu9XG8bPlliUOP9ThaF4QnhP8qrjrH7DRzHfSk0oQv1wToW+iA5GajEQ==} + /update-browserslist-db@1.1.0(browserslist@4.23.2): + resolution: {integrity: sha512-EdRAaAyk2cUE1wOf2DkEhzxqOQvFOoRJFNS6NeyJ01Gp2beMRpBAINjM2iDXE3KCuKhwnvHIQCJm6ThL2Z+HzQ==} hasBin: true peerDependencies: browserslist: '>= 4.21.0' dependencies: - browserslist: 4.23.1 + browserslist: 4.23.2 escalade: 3.1.2 picocolors: 1.0.1 dev: true @@ -5316,14 +5099,14 @@ packages: /uri-js@4.4.1: resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} dependencies: - punycode: 2.3.0 + punycode: 2.3.1 dev: false /util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} /utils-merge@1.0.1: - resolution: {integrity: sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=} + resolution: {integrity: sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==} engines: {node: '>= 0.4.0'} dev: false @@ -5416,56 +5199,30 @@ packages: string-width: 2.1.1 dev: true - /winston-transport@4.5.0: - resolution: {integrity: sha512-YpZzcUzBedhlTAfJg6vJDlyEai/IFMIVcaEZZyl3UXIl4gmqRpU7AE89AHLkbzLUsv0NVmw7ts+iztqKxxPW1Q==} - engines: {node: '>= 6.4.0'} - dependencies: - logform: 2.5.1 - readable-stream: 3.6.2 - triple-beam: 1.3.0 - dev: false - - /winston-transport@4.7.0: - resolution: {integrity: sha512-ajBj65K5I7denzer2IYW6+2bNIVqLGDHqDw3Ow8Ohh+vdW+rv4MZ6eiDvHoKhfJFZ2auyN8byXieDDJ96ViONg==} + /winston-transport@4.7.1: + resolution: {integrity: sha512-wQCXXVgfv/wUPOfb2x0ruxzwkcZfxcktz6JIMUaPLmcNhO4bZTwA/WtDWK74xV3F2dKu8YadrFv0qhwYjVEwhA==} engines: {node: '>= 12.0.0'} dependencies: - logform: 2.6.0 + logform: 2.6.1 readable-stream: 3.6.2 triple-beam: 1.4.1 dev: false - /winston@3.13.0: - resolution: {integrity: sha512-rwidmA1w3SE4j0E5MuIufFhyJPBDG7Nu71RkZor1p2+qHvJSZ9GYDA81AyleQcZbh/+V6HjeBdfnTZJm9rSeQQ==} + /winston@3.13.1: + resolution: {integrity: sha512-SvZit7VFNvXRzbqGHsv5KSmgbEYR5EiQfDAL9gxYkRqa934Hnk++zze0wANKtMHcy/gI4W/3xmSDwlhf865WGw==} engines: {node: '>= 12.0.0'} dependencies: '@colors/colors': 1.6.0 '@dabh/diagnostics': 2.0.3 async: 3.2.5 is-stream: 2.0.1 - logform: 2.6.0 + logform: 2.6.1 one-time: 1.0.0 readable-stream: 3.6.2 safe-stable-stringify: 2.4.3 stack-trace: 0.0.10 triple-beam: 1.4.1 - winston-transport: 4.7.0 - dev: false - - /winston@3.9.0: - resolution: {integrity: sha512-jW51iW/X95BCW6MMtZWr2jKQBP4hV5bIDq9QrIjfDk6Q9QuxvTKEAlpUNAzP+HYHFFCeENhph16s0zEunu4uuQ==} - engines: {node: '>= 12.0.0'} - dependencies: - '@colors/colors': 1.5.0 - '@dabh/diagnostics': 2.0.3 - async: 3.2.4 - is-stream: 2.0.1 - logform: 2.5.1 - one-time: 1.0.0 - readable-stream: 3.6.2 - safe-stable-stringify: 2.4.3 - stack-trace: 0.0.10 - triple-beam: 1.3.0 - winston-transport: 4.5.0 + winston-transport: 4.7.1 dev: false /wordwrap@1.0.0: @@ -5476,8 +5233,8 @@ packages: resolution: {integrity: sha512-toV7q9rWNYha963Pl/qyeZ6wG+3nnsyvolaNUS8+R5Wtw6qJPTxIlOP1ZSvcGhEJw+l3HMMmtiNo9Gl61G4GVg==} dev: true - /workerpool@6.2.1: - resolution: {integrity: sha512-ILEIE97kDZvF9Wb9f6h5aXK4swSlKGUcOEGiIYb2OOu/IrDU9iwj0fD//SsA6E5ibwJxpEvhullJY4Sl4GcpAw==} + /workerpool@6.5.1: + resolution: {integrity: sha512-Fs4dNYcsdpYSAfVxhnl1L5zTksjvOJxtC5hzMNl+1t9B8hTJTdKDyZ5ju7ztgPy+ft9tBFXoOlDNiOT9WUXZlA==} dev: true /wrap-ansi@5.1.0: @@ -5571,6 +5328,11 @@ packages: engines: {node: '>=10'} dev: true + /yargs-parser@20.2.9: + resolution: {integrity: sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==} + engines: {node: '>=10'} + dev: true + /yargs-parser@21.1.1: resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} engines: {node: '>=12'} @@ -5628,7 +5390,7 @@ packages: require-directory: 2.1.1 string-width: 4.2.3 y18n: 5.0.8 - yargs-parser: 20.2.4 + yargs-parser: 20.2.9 dev: true /yargs@17.7.2: @@ -5654,12 +5416,12 @@ packages: engines: {node: '>=10'} dev: true - /zod-to-json-schema@3.21.3(zod@3.21.4): - resolution: {integrity: sha512-09W/9oyxeF1/wWnzCb6MursW+lOzgKi91QwE7eTBbC+t/qgfuLsUVDai3lHemSQnQu/UONAcT/fv3ZnDvbTeKg==} + /zod-to-json-schema@3.23.2(zod@3.23.8): + resolution: {integrity: sha512-uSt90Gzc/tUfyNqxnjlfBs8W6WSGpNBv0rVsNxP/BVSMHMKGdthPYff4xtCHYloJGM0CFxFsb3NbC0eqPhfImw==} peerDependencies: - zod: ^3.21.4 + zod: ^3.23.3 dependencies: - zod: 3.21.4 + zod: 3.23.8 - /zod@3.21.4: - resolution: {integrity: sha512-m46AKbrzKVzOzs/DZgVnG5H55N1sv1M8qZU3A8RIKbs3mrACDNeIOeilDymVb2HdmP8uwshOCF4uJ8uM9rCqJw==} + /zod@3.23.8: + resolution: {integrity: sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 82c9034..c9e3cb4 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,4 +1,4 @@ packages: - 'apps/*' - 'packages/*' - - 'scripts/*' \ No newline at end of file + - 'scripts' \ No newline at end of file diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..36d4a2f --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,12 @@ +# Lectern Scripts + +## Generate + +Create a JSON Schema file representation of the Lectern Dictionary meta schema. This file will be created at the path [`generated/DictionaryMetaSchema.json`](../../generated/DictionaryMetaSchema.json) and is available through the github repository for reference by any users that plan to use JSON schema to validate a data dictionary schema as a valid Lectern Dictionary. + +Entry point: `src/generateMetaSchema.ts` +Workspace Command: `pnpm -w generate` +Local Command: `pnpm generate` + + + diff --git a/scripts/generate/package.json b/scripts/generate/package.json deleted file mode 100644 index 43147da..0000000 --- a/scripts/generate/package.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "scripts", - "version": "1.0.0", - "private": true, - "description": "", - "main": "index.js", - "scripts": { - "generate": "npm run generate:metaschema", - "generate:metaschema": "node -r ts-node/register ./src/generateMetaSchema.ts" - }, - "keywords": [], - "author": "", - "license": "AGPL-3.0", - "dependencies": { - "dictionary": "workspace:^", - "zod-to-json-schema": "^3.21.3" - } -} diff --git a/scripts/package.json b/scripts/package.json new file mode 100644 index 0000000..9fb0e75 --- /dev/null +++ b/scripts/package.json @@ -0,0 +1,16 @@ +{ + "name": "lectern-scripts", + "version": "1.0.0", + "private": true, + "description": "Scripts for managing the Lectern monorepo.", + "scripts": { + "generate": "node -r ts-node/register ./src/generateMetaSchema.ts" + }, + "keywords": [], + "license": "AGPL-3.0", + "dependencies": { + "@overture-stack/lectern-dictionary": "workspace:^", + "zod": "^3.23.8", + "zod-to-json-schema": "^3.23.2" + } +} diff --git a/scripts/generate/src/generateMetaSchema.ts b/scripts/src/generateMetaSchema.ts similarity index 90% rename from scripts/generate/src/generateMetaSchema.ts rename to scripts/src/generateMetaSchema.ts index 7ab6f12..c419a3e 100644 --- a/scripts/generate/src/generateMetaSchema.ts +++ b/scripts/src/generateMetaSchema.ts @@ -4,14 +4,15 @@ */ import { Dictionary, + DictionaryBase, DictionaryMeta, - NameString, + NameValue, ReferenceArray, ReferenceTag, References, Schema, SchemaField, -} from 'dictionary'; +} from '@overture-stack/lectern-dictionary'; import fs from 'fs'; import { zodToJsonSchema } from 'zod-to-json-schema'; @@ -25,7 +26,7 @@ const jsonSchema = zodToJsonSchema(Dictionary, { ReferenceArray, References, Meta: DictionaryMeta, - Name: NameString, + Name: NameValue, Schema, SchemaField, }, diff --git a/packages/common/tsconfig.json b/scripts/tsconfig.json similarity index 66% rename from packages/common/tsconfig.json rename to scripts/tsconfig.json index 90dcbbd..57c9c2b 100644 --- a/packages/common/tsconfig.json +++ b/scripts/tsconfig.json @@ -9,12 +9,9 @@ "noImplicitAny": true, "noUnusedParameters": true, "esModuleInterop": true, - "declaration": true, - "sourceMap": true, - "inlineSources": true, - "outDir": "dist/", - "baseUrl": "./src", + "noEmit": true, + "outDir": "dist", "skipLibCheck": true }, - "include": ["./src/**/*.ts"] + "include": ["./src/**/*.ts", "./test/**/*.ts"] } From 49c8f00e2adc0994fb48006d256a463e8440c10e Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Mon, 19 Aug 2024 10:32:59 -0400 Subject: [PATCH 25/27] Conditional Restrictions - Add conditional restriction logic to field meta schemas and implement validation logic (#223) * Remove script restrictions; Move unique property out of restrictions These are breaking changes to the meta schema that need to be published as a major release. A document should be added to the repo describing these changes. * Field restrictions can be an array of restriction objects * Add tests for field restrictions of all forms * Test validation library uses all restrictions in array * Add document to detail major version changes * Adds conditional restrictions to dictionary restriction schema - performs code list and regex reference replacement recursively through conditional restrictions - performs reference replacement recursively through meta objects - applies conditional restriction checks when resolving restriction rules for each field - WIP: still requires many tests * Tests for references in meta and regex arrays * ConditionalRestrictionTest has optional case with proper default in validation * Single match object per condition, instead of array * Fix recursive conditional restriction parsing and add tests * Named container and volume in compose file * Fix import path * Empty field restriction validation tests * Remove development test file * Remove script restriction from test dictionary * Restriction schemas directly written for each field type - Generic conditional restriction function was removed because it could not be interpretted by the json schema generator - Although there is repeated code, directly writing the typed conditional restriction type schemas is easier to parse and hopefully maintain. They have their types enforced by a generic type even if the schema itself is not generated through a function. * Adds tests for conditional restriction match rules * Generated JSON Schema with Conditional Restrictions * Remove TODO statements that are not needed * Fix broken table in reference doc * Code cleanup by removing unused functions and comments --- apps/server/docker-compose.yaml | 10 +- apps/server/src/services/dictionaryService.ts | 22 +- apps/server/src/services/schemaService.ts | 40 +- .../test/fixtures/schemas/references.ts | 7 - .../integration/fixtures/updateNewFile.json | 1 - docs/dictionary-reference.md | 37 +- docs/lectern-2.0-changes.md | 48 + generated/DictionaryMetaSchema.json | 1064 ++++++++++++----- .../src/metaSchema/dictionarySchemas.ts | 137 ++- .../src/metaSchema/referenceSchemas.ts | 2 +- .../src/metaSchema/restrictionsSchemas.ts | 110 +- packages/dictionary/src/references.ts | 339 ++++-- .../src/utils/resolveRestrictions.ts | 51 - packages/dictionary/src/utils/schemaUtils.ts | 21 +- packages/dictionary/src/utils/typeUtils.ts | 77 ++ packages/dictionary/test/diff.spec.ts | 4 +- .../dictionary/test/fixtures/diff/initial.ts | 2 +- .../dictionary/test/fixtures/diff/updated.ts | 2 +- .../references/codeList_references/input.ts | 5 +- .../references/codeList_references/output.ts | 2 +- .../empty_references_section/input.ts | 2 +- .../empty_references_section/output.ts | 2 +- .../nested_meta_references/input.ts | 29 + .../nested_meta_references/output.ts | 25 + .../input.ts | 25 +- .../output.ts | 15 +- .../references/no_references_section/input.ts | 2 +- .../no_references_section/output.ts | 2 +- .../input.ts} | 2 +- .../regex_reference_with_array/output.ts | 21 + .../input.ts | 53 + .../output.ts | 43 + .../dictionarySchemas.spec.ts} | 193 +-- .../metaSchema/restrictionsSchemas.spec.ts | 190 +++ packages/dictionary/test/references.spec.ts | 97 +- .../booleanRegex.spec.ts} | 2 +- .../test/{ => utils}/versionUtils.spec.ts | 2 +- .../parseValues/matchCodeListFormatting.ts | 25 +- .../validation/src/parseValues/parseValues.ts | 4 +- .../validation/src/utils/isValidValueType.ts | 4 +- .../src/utils/resultForArrayTestCase.ts | 29 + packages/validation/src/utils/typeUtils.ts | 95 -- .../src/validateField/FieldRestrictionRule.ts | 16 +- .../src/validateField/conditions/index.ts | 25 + .../conditions/testConditionalRestriction.ts | 112 ++ .../conditions/testMatchCodeList.ts | 49 + .../conditions/testMatchCount.ts | 48 + .../conditions/testMatchExists.ts | 60 + .../conditions/testMatchRange.ts | 31 +- .../conditions/testMatchRegex.ts | 35 + .../conditions/testMatchValue.ts | 63 + .../validateField/resolveFieldRestrictions.ts | 98 -- .../src/validateField/restrictions/index.ts | 2 + .../restrictions/resolveFieldRestrictions.ts | 100 ++ .../validateField/restrictions/testEmpty.ts | 81 ++ .../validateField/restrictions/testRegex.ts | 19 +- .../src/validateField/validateField.ts | 18 +- .../src/validateSchema/validateSchema.ts | 2 +- .../fieldStringConditionalExists.ts | 29 + ...ieldStringConditionalMultipleConditions.ts | 48 + ...eldStringConditionalMultipleFieldsRegex.ts | 34 + .../fieldStringNestedConditional.ts | 64 + .../fieldStringRequiredConditionalRange.ts | 35 + .../fieldStringArrayMultipleRegex.ts | 22 + .../schemaRestrictions/fieldStringUnique.ts | 2 +- .../fieldStringUniqueArray.ts | 2 +- .../fixtures/restrictions/regexFixtures.ts | 5 +- .../conditions/testMatchCodeList.spec.ts | 45 + .../conditions/testMatchCount.spec.ts | 65 + .../conditions/testMatchExists.spec.ts | 73 ++ .../conditions/testMatchRange.spec.ts | 53 + .../conditions/testMatchRegex.spec.ts | 54 + .../conditions/testMatchValue.spec.ts | 86 ++ .../resolveFieldRestrictions.spec.ts | 419 +++++++ .../test/validateField/validateField.spec.ts | 16 + .../validateRecord/validateRecord.spec.ts | 1 - scripts/src/generateMetaSchema.ts | 25 +- 77 files changed, 3688 insertions(+), 992 deletions(-) create mode 100644 docs/lectern-2.0-changes.md delete mode 100644 packages/dictionary/src/utils/resolveRestrictions.ts create mode 100644 packages/dictionary/test/fixtures/references/nested_meta_references/input.ts create mode 100644 packages/dictionary/test/fixtures/references/nested_meta_references/output.ts rename packages/dictionary/test/fixtures/references/{script_references => no_referece_tags}/input.ts (53%) rename packages/dictionary/test/fixtures/references/{script_references => no_referece_tags}/output.ts (64%) rename packages/dictionary/test/fixtures/references/{regex_reference/input_with_array.ts => regex_reference_with_array/input.ts} (90%) create mode 100644 packages/dictionary/test/fixtures/references/regex_reference_with_array/output.ts create mode 100644 packages/dictionary/test/fixtures/references/restrictions_array_with_references/input.ts create mode 100644 packages/dictionary/test/fixtures/references/restrictions_array_with_references/output.ts rename packages/dictionary/test/{dictionaryTypes.spec.ts => metaSchema/dictionarySchemas.spec.ts} (77%) create mode 100644 packages/dictionary/test/metaSchema/restrictionsSchemas.spec.ts rename packages/dictionary/test/{dataTypes.spec.ts => types/booleanRegex.spec.ts} (98%) rename packages/dictionary/test/{ => utils}/versionUtils.spec.ts (98%) create mode 100644 packages/validation/src/utils/resultForArrayTestCase.ts delete mode 100644 packages/validation/src/utils/typeUtils.ts create mode 100644 packages/validation/src/validateField/conditions/index.ts create mode 100644 packages/validation/src/validateField/conditions/testConditionalRestriction.ts create mode 100644 packages/validation/src/validateField/conditions/testMatchCodeList.ts create mode 100644 packages/validation/src/validateField/conditions/testMatchCount.ts create mode 100644 packages/validation/src/validateField/conditions/testMatchExists.ts rename apps/server/test/functional/normalize.spec.ts => packages/validation/src/validateField/conditions/testMatchRange.ts (53%) create mode 100644 packages/validation/src/validateField/conditions/testMatchRegex.ts create mode 100644 packages/validation/src/validateField/conditions/testMatchValue.ts delete mode 100644 packages/validation/src/validateField/resolveFieldRestrictions.ts create mode 100644 packages/validation/src/validateField/restrictions/resolveFieldRestrictions.ts create mode 100644 packages/validation/src/validateField/restrictions/testEmpty.ts create mode 100644 packages/validation/test/fixtures/fields/conditionalRestrictions/fieldStringConditionalExists.ts create mode 100644 packages/validation/test/fixtures/fields/conditionalRestrictions/fieldStringConditionalMultipleConditions.ts create mode 100644 packages/validation/test/fixtures/fields/conditionalRestrictions/fieldStringConditionalMultipleFieldsRegex.ts create mode 100644 packages/validation/test/fixtures/fields/conditionalRestrictions/fieldStringNestedConditional.ts create mode 100644 packages/validation/test/fixtures/fields/conditionalRestrictions/fieldStringRequiredConditionalRange.ts create mode 100644 packages/validation/test/fixtures/fields/multipleRestrictions/fieldStringArrayMultipleRegex.ts create mode 100644 packages/validation/test/validateField/conditions/testMatchCodeList.spec.ts create mode 100644 packages/validation/test/validateField/conditions/testMatchCount.spec.ts create mode 100644 packages/validation/test/validateField/conditions/testMatchExists.spec.ts create mode 100644 packages/validation/test/validateField/conditions/testMatchRange.spec.ts create mode 100644 packages/validation/test/validateField/conditions/testMatchRegex.spec.ts create mode 100644 packages/validation/test/validateField/conditions/testMatchValue.spec.ts create mode 100644 packages/validation/test/validateField/resolveFieldRestrictions.spec.ts diff --git a/apps/server/docker-compose.yaml b/apps/server/docker-compose.yaml index e230673..73f62b7 100644 --- a/apps/server/docker-compose.yaml +++ b/apps/server/docker-compose.yaml @@ -1,12 +1,13 @@ -version: "2" +version: '2' services: lecternDb: - image: "bitnami/mongodb:4.0" + container_name: lectern-mongo + image: bitnami/mongodb:4.0 ports: - - "27017:27017" + - 27017:27017 volumes: - - "mongodb_data:/bitnami" + - mongodb_data:/bitnami environment: MONGODB_USERNAME: admin MONGODB_PASSWORD: password @@ -14,4 +15,5 @@ services: MONGODB_ROOT_PASSWORD: password123 volumes: mongodb_data: + name: lectern-mongo-data driver: local diff --git a/apps/server/src/services/dictionaryService.ts b/apps/server/src/services/dictionaryService.ts index 8b3d914..3c77d28 100644 --- a/apps/server/src/services/dictionaryService.ts +++ b/apps/server/src/services/dictionaryService.ts @@ -29,7 +29,7 @@ import * as immer from 'immer'; import { omit } from 'lodash'; import logger from '../config/logger'; import * as DictionaryRepo from '../db/dictionary'; -import { normalizeSchema, validate } from '../services/schemaService'; +import { validateDictionarySchema } from '../services/schemaService'; import type { DictionaryDocument, DictionaryDocumentSummary } from '../db/dbTypes'; /** @@ -127,17 +127,15 @@ export const create = async (newDict: Dictionary): Promise => { // Verify schemas match dictionary newDict.schemas.forEach((e) => { - const result = validate(e, newDict.references || {}); + const result = validateDictionarySchema(e, newDict.references || {}); if (!result.valid) throw new BadRequestError(JSON.stringify(result.errors)); }); - const normalizedSchemas = newDict.schemas.map((schema) => normalizeSchema(schema)); - // Save new dictionary version const result = await DictionaryRepo.addDictionary({ name: newDict.name, version: newDict.version, - schemas: normalizedSchemas, + schemas: newDict.schemas, references: newDict.references || {}, }); return result; @@ -159,18 +157,16 @@ export const addSchema = async (id: string, schema: Schema): Promise throw new BadRequestError('Dictionary that you are trying to update is not the latest version.'); } - const result = validate(schema, existingDictionary.references || {}); + const result = validateDictionarySchema(schema, existingDictionary.references || {}); if (!result.valid) throw new BadRequestError(JSON.stringify(result.errors)); if (existingDictionary.schemas.some((s) => s.name === schema.name)) { throw new ConflictError('Schema with this name already exists.'); } - const normalizedSchema = normalizeSchema(schema); - const updatedDictionary = immer.produce(existingDictionary, (draft) => { draft.version = VersionUtils.incrementMajor(draft.version); - draft.schemas = [...draft.schemas, normalizedSchema]; + draft.schemas = [...draft.schemas, schema]; }); // Save new dictionary version @@ -191,7 +187,7 @@ export const updateSchema = async (id: string, schema: Schema, major: boolean): await checkLatest(existingDictionary); - const result = validate(schema, existingDictionary.references || {}); + const result = validateDictionarySchema(schema, existingDictionary.references || {}); if (!result.valid) throw new BadRequestError(JSON.stringify(result.errors)); // Ensure it exists @@ -202,9 +198,7 @@ export const updateSchema = async (id: string, schema: Schema, major: boolean): // Filter out one to update const schemas = existingDictionary.schemas.filter((s) => !(s['name'] === schema['name'])); - const normalizedSchema = normalizeSchema(schema); - - schemas.push(normalizedSchema); + schemas.push(schema); // Increment Version const nextVersion = major @@ -212,7 +206,7 @@ export const updateSchema = async (id: string, schema: Schema, major: boolean): : VersionUtils.incrementMinor(existingDictionary.version); const updatedDictionary = immer.produce(existingDictionary, (draft) => { const filteredSchemas = draft.schemas.filter((s) => !(s['name'] === schema['name'])); - draft.schemas = [...filteredSchemas, normalizedSchema]; + draft.schemas = [...filteredSchemas, schema]; draft.version = nextVersion; }); diff --git a/apps/server/src/services/schemaService.ts b/apps/server/src/services/schemaService.ts index 2bbae87..59b3f5b 100644 --- a/apps/server/src/services/schemaService.ts +++ b/apps/server/src/services/schemaService.ts @@ -18,10 +18,12 @@ */ import { References, replaceSchemaReferences, Schema } from '@overture-stack/lectern-dictionary'; -import * as immer from 'immer'; import { ZodError } from 'zod'; -export function validate(schema: Schema, references: References): { valid: boolean; errors?: ZodError } { +export function validateDictionarySchema( + schema: Schema, + references: References, +): { valid: boolean; errors?: ZodError } { const schemaWithReplacements = replaceSchemaReferences(schema, references); // Ensure schema is still valid after reference replacement @@ -30,37 +32,3 @@ export function validate(schema: Schema, references: References): { valid: boole return parseResult.success ? { valid: true } : { valid: false, errors: parseResult.error }; } - -/** - * String formatting of values provided as scripts. This will normalize the formatting of newline characters, - * All instances of `/r/n` will be converted to `/n` - * @param script - * @returns - */ -function normalizeScript(input: string | string[]) { - const normalize = (script: string) => script.replace(/\r\n/g, '\n'); - - if (typeof input === 'string') { - return normalize(input); - } else { - return input.map(normalize); - } -} - -export function normalizeSchema(schema: Schema): Schema { - const normalizedFields = schema.fields.map((baseField) => - immer.produce(baseField, (field) => { - if ( - field.valueType !== 'boolean' && - field.restrictions !== undefined && - field.restrictions.script !== undefined - ) { - field.restrictions.script = normalizeScript(field.restrictions.script); - } - }), - ); - - return immer.produce(schema, (draft) => { - draft.fields = normalizedFields; - }); -} diff --git a/apps/server/test/fixtures/schemas/references.ts b/apps/server/test/fixtures/schemas/references.ts index abb4f01..b1f3485 100644 --- a/apps/server/test/fixtures/schemas/references.ts +++ b/apps/server/test/fixtures/schemas/references.ts @@ -35,12 +35,5 @@ export default { codeList: '#/listA', }, }, - { - name: 'script_as_reference', - valueType: 'number', - restrictions: { - script: '#/scriptA', - }, - }, ], } satisfies Schema; diff --git a/apps/server/test/integration/fixtures/updateNewFile.json b/apps/server/test/integration/fixtures/updateNewFile.json index ba9df6c..5aa1132 100644 --- a/apps/server/test/integration/fixtures/updateNewFile.json +++ b/apps/server/test/integration/fixtures/updateNewFile.json @@ -30,7 +30,6 @@ "units": "days" }, "restrictions": { - "script": ["validateWithMagic(check dependece on another field)"], "range": { "min": 0, "max": 99999 diff --git a/docs/dictionary-reference.md b/docs/dictionary-reference.md index 64bf017..e06bb0a 100644 --- a/docs/dictionary-reference.md +++ b/docs/dictionary-reference.md @@ -96,16 +96,16 @@ The restrictions property of a field can have a value that is either a single re The full list of available restrictions are: -| Restriction | Used with Field Types | Type | Description | Examples | -| ----------- | ----------------------------- | --------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `codeList` | `integer`, `number`, `string` | Array of type of the field | An array of values of the type matching this field. Data provided for this field must have one of the values in this list. | `["Weak", "Average", "Strong"]` | -| `compare` | all | [ComparedFieldsRule](#comparedfieldsrule-data-structure) object | Enforces that this field has a value based on the provided value in another field. Examples would be to ensure that the two values are not equal, or for numeric values ensure one is greater than the other. | `{ "fields": ["age_at_diagnosis"], "relation": "greaterThanOrEqual" }` Ensure that a field such as `age_at_death` is greater than the provided `age_at_diagnosis` | -| `count` | Array fields of all types | `integer` or [`RangeRule`](#rangerule-data-structure) object | Enfroces the number of entries in an array. Can specify an exact array size, or provide range rules that set maximum and minimum counts. | `7` or `{"min": 5, "max": 10}` | -| `empty` | all | | Requires that no value is provided. This is useful when used on a [conditional restriction](#conditional-restrictions) in order to prevent a value from being given when the condition is `true`. For an array field with this restriction, an empty array is a valid value for this restriction. | n/a | -| `range` | `integer`, `number` | | Uses a [RangeRule](#rangerule-data-structure) object to define minimum and/or maximum values for this field | `{"min": 5}`, `{"exclusiveMax": 50}`, `{"exclusiveMin": 5, "max": 50}` | -| `regex` | `string` | | A regular expression that all values must match. | `^[a-z0-9]+$` | -| `required` | all | | A value must be provided, missing/undefined values will fail validation. Empty strings will not be accepted, though `0` (for `number` and `int` fields) and `false` (for `boolean` fields) are accepted. An array field with this restriction must have at least one entry. | `true`, `false` | -| `unique` | all | | When a field has the `unique` restriction, each record must have a distinct value for this field. Uniqueness tests are case sensitive, so `Abc` and `abc` are both distinct values. This restriction is only applied when a collection of records are tested together, ensuring that no two records in that collection share a value. | `true`, `false` | +| Restriction | Used with Field Types | Type | Description | Examples | +| ----------- | ----------------------------- | --------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `codeList` | `integer`, `number`, `string` | Array of type of the field | An array of values of the type matching this field. Data provided for this field must have one of the values in this list. | `["Weak", "Average", "Strong"]` | +| `compare` | all | [ComparedFieldsRule](#comparedfieldsrule-data-structure) object | Enforces that this field has a value based on the provided value in another field. Examples would be to ensure that the two values are not equal, or for numeric values ensure one is greater than the other. | `{ "fields": ["age_at_diagnosis"], "relation": "greaterThanOrEqual" }` Ensure that a field such as `age_at_death` is greater than the provided `age_at_diagnosis` | +| `count` | Array fields of all types | `integer` or [`RangeRule`](#rangerule-data-structure) object | Enfroces the number of entries in an array. Can specify an exact array size, or provide range rules that set maximum and minimum counts. | `7` or `{"min": 5, "max": 10}` | +| `empty` | all | | Requires that no value is provided. This is useful when used on a [conditional restriction](#conditional-restrictions) in order to prevent a value from being given when the condition is `true`. For an array field with this restriction, an empty array is a valid value for this restriction. | n/a | +| `range` | `integer`, `number` | | Uses a [RangeRule](#rangerule-data-structure) object to define minimum and/or maximum values for this field | `{"min": 5}`, `{"exclusiveMax": 50}`, `{"exclusiveMin": 5, "max": 50}` | +| `regex` | `string` | | A regular expression that all values must match. | `^[a-z0-9]+$` | +| `required` | all | | A value must be provided, missing/undefined values will fail validation. Empty strings will not be accepted, though `0` (for `number` and `int` fields) and `false` (for `boolean` fields) are accepted. An array field with this restriction must have at least one entry. | `true`, `false` | +| `unique` | all | | When a field has the `unique` restriction, each record must have a distinct value for this field. Uniqueness tests are case insensitive, so `Abc` and `abc` are treated as the same value. This restriction is only applied when a collection of records are tested together, ensuring that no two records in that collection share a value. | `true`, `false` | #### Conditional Restrictions @@ -170,15 +170,14 @@ A requirement condition is defined by providing a field name or list of field na > ``` ##### Conditional Match Rules -| Property | Used with Field Types | Type | Description | Example | -| ---------- | --------------------- | -------------------------------------------------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | -| `codeList` | all | Array of type of specified fields | A list of values that the field could match. This rule passes when the specified field's value can be found in this list. | `["value_one", "value_two"]` | -| `compare` | all | [ComparedFieldsRule](#comparedfieldsrule-data-structure) | Compare the value of the specified fields to values of another field (or set of fields). This can be configured to check if the fields match, dont match, and for numeric fields can check if the field is Greater or Lesser than. | `{ "fields": ["compared_to_field"], "relation": "equal" }` | -| `count` | Array type fields | Integer, or [RangeRule](#rangerule-data-structure) | Matches the number of values in an array field. This condition can be provided as a number, in which case this condition matches if the array is that exact length. This condition can be provided as a Range object as well, in which case it will match if the number of elements in the array pass the minimum and maximum conditions provided in the condition. | `2` - Field must have exactly 2 elements.
`{ max: 10 }` - Field must have no more than 10 items. | -| `exists` | all | Boolean | This condition requires a field to either have a value or have no value. When the `exists` condition is set to `true`, the field must have a value. When `exists` is sdet to `false`, the field must have no value. For array fields, `exists=false` only matches when the array is completely empty, and `exists=true` passes if the array has 1 or more values - `arrayCase` has no interaction with the `exists` condition. | `true` | -| `range` | `number`, `integer` | [RangeRule](#rangerule-data-structure) | Maximum and minimum value conditions that a numeric field must pass. | `{ min: 5, exclusiveMax: 10 }` Represents an integer from 5-9. | -| `regex` | `string` | String (Regular Expression) | A regular expression pattern that the value must match. | `^NCIT:C\d+$` Value must match an NCI Thesaurus ID. | -| `value` | all | Type of specified fields | Field value exactly matches the value of the specified field. Strings are matched case sensitive. | `some_value` | +| Property | Used with Field Types | Type | Description | Example | +| ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------- | -------------------------------------------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | +| `codeList` | all | Array of type of specified fields | A list of values that the field could match. This rule passes when the specified field's value can be found in this list. | `["value_one", "value_two"]` | +| `count` | Array type fields | Integer, or [RangeRule](#rangerule-data-structure) | Matches the number of values in an array field. This condition can be provided as a number, in which case this condition matches if the array is that exact length. This condition can be provided as a Range object as well, in which case it will match if the number of elements in the array pass the minimum and maximum conditions provided in the condition. | `2` - Field must have exactly 2 elements.
`{ max: 10 }` - Field must have no more than 10 items. | +| `exists` | all | Boolean | This condition requires a field to either have a value or have no value. When the `exists` condition is set to `true`, the field must have a value. When `exists` is sdet to `false`, the field must have no value. For array fields, `exists=false` only matches when the array is completely empty, and `exists=true` passes if the array has 1 or more values - `arrayCase` has no interaction with the `exists` condition. | `true` | +| `range` | `number`, `integer` | [RangeRule](#rangerule-data-structure) | Maximum and minimum value conditions that a numeric field must pass. | `{ min: 5, exclusiveMax: 10 }` Represents an integer from 5-9. | +| `regex` | `string` | String (Regular Expression) | A regular expression pattern that the value must match. | `^NCIT:C\d+$` Value must match an NCI Thesaurus ID. | +| `value` | all | Type of specified fields | Field value matches the value of the specified field. Strings are matched case insensitive. When arrays are matched, the order of their elements is ignored - a field matches this condition if the elements in field are the same elements as in the value match rule. For example, the rule `['abc', 'def']` matches the value `['def', 'abc']` but does not match `['abc', 'def', 'ghi']`. | `some_value`, `[1, 2, 3]` | ### Meta Data Structure diff --git a/docs/lectern-2.0-changes.md b/docs/lectern-2.0-changes.md new file mode 100644 index 0000000..9f38953 --- /dev/null +++ b/docs/lectern-2.0-changes.md @@ -0,0 +1,48 @@ +# Lectern Version 2 Changes + +The release of Lectern 2.0 brings some important upgrades to the Lectern service, published tooling, and importantly the Lectern Dictionary specification. Most of these changes are backwards compatible but some breaking changes have been introduced. A section at the end describes a process for upgrading from Lectern version 1 to 2. + +## Summary of Changes + +### Meta-Schema Updates + +- Script restrictions have been removed. +- The `unique` restriction has been moved to be a property of the field, not a restriction. +- Conditional restrictions have been added. +- Fields now accept an array of restrictions, allowing multiple regex and codeList restrictions applied to a specific field. +- Regex restrictions can be a single string or array of strings, allowing multiple regular expressions to be applied to a field's value(s). + +### Lectern JS Client Updates + +- The client package has been moved into the same organization as other Overture software. + - Now published at `@overture-stack/lectern-client`. + - Old package located at `@overturebio-stack/lectern-client` is marked as deprecated. +- API Changes + - Processing functions renamed to match validation and parsing functions + - Updated interface for Lectern Server REST client + - Exposes dictionary meta-schema validation, data parsing, and data validation functions + + +### New Published Lectern TS Packages + +- [Lectern Dictionary](../packages/dictionary/) + - Meta-Schema for validating Lectern Dictionaries + - Functions to calculate diffs between dictionaries + - Functions to manage dictionary refernces +- [Lectern Validation](../packages/validation/) + - Parse raw string values into typed data to match a Lectern Dictionary + - Validate data using a Lectern Dictionary + +## Upgrading from Lectern 1 + +### Lectern Server Migration + +Placeholder + +### Updating Lectern Dictionaries + +Placeholder + +### Upgrading Lectern Client + +Placeholder diff --git a/generated/DictionaryMetaSchema.json b/generated/DictionaryMetaSchema.json index a43a52d..d0d29b7 100644 --- a/generated/DictionaryMetaSchema.json +++ b/generated/DictionaryMetaSchema.json @@ -1,39 +1,723 @@ { "$ref": "#/definitions/Dictionary", "definitions": { - "ReferenceTag": { - "type": "string", - "pattern": "^#(\\/[-_A-Za-z0-9]+)+$" + "SchemaBooleanField": { + "type": "object", + "properties": { + "name": { + "$ref": "#/definitions/Name" + }, + "description": { + "type": "string" + }, + "isArray": { + "type": "boolean" + }, + "meta": { + "$ref": "#/definitions/Meta" + }, + "unique": { + "type": "boolean" + }, + "valueType": { + "type": "string", + "const": "boolean" + }, + "restrictions": { + "anyOf": [ + { + "anyOf": [ + { + "$ref": "#/definitions/BooleanFieldRestrictions" + }, + { + "type": "object", + "properties": { + "if": { + "$ref": "#/definitions/ConditionalRestrictionTest" + }, + "then": { + "anyOf": [ + { + "anyOf": [ + { + "$ref": "#/definitions/BooleanFieldRestrictions" + }, + { + "$ref": "#/definitions/SchemaBooleanField/properties/restrictions/anyOf/0/anyOf/1" + } + ] + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "$ref": "#/definitions/BooleanFieldRestrictions" + }, + { + "$ref": "#/definitions/SchemaBooleanField/properties/restrictions/anyOf/0/anyOf/1" + } + ] + } + } + ] + }, + "else": { + "anyOf": [ + { + "anyOf": [ + { + "$ref": "#/definitions/BooleanFieldRestrictions" + }, + { + "$ref": "#/definitions/SchemaBooleanField/properties/restrictions/anyOf/0/anyOf/1" + } + ] + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "$ref": "#/definitions/BooleanFieldRestrictions" + }, + { + "$ref": "#/definitions/SchemaBooleanField/properties/restrictions/anyOf/0/anyOf/1" + } + ] + } + } + ] + } + }, + "required": [ + "if" + ], + "additionalProperties": false + } + ] + }, + { + "type": "array", + "items": { + "$ref": "#/definitions/SchemaBooleanField/properties/restrictions/anyOf/0" + } + } + ] + } + }, + "required": [ + "name", + "valueType" + ], + "additionalProperties": false }, - "ReferenceArray": { - "type": "array", - "items": { - "anyOf": [ - { - "type": "string" - }, - { - "$ref": "#/definitions/ReferenceTag" - } - ] + "SchemaIntegerField": { + "type": "object", + "properties": { + "name": { + "$ref": "#/definitions/Name" + }, + "description": { + "$ref": "#/definitions/SchemaBooleanField/properties/description" + }, + "isArray": { + "$ref": "#/definitions/SchemaBooleanField/properties/isArray" + }, + "meta": { + "$ref": "#/definitions/SchemaBooleanField/properties/meta" + }, + "unique": { + "$ref": "#/definitions/SchemaBooleanField/properties/unique" + }, + "valueType": { + "type": "string", + "const": "integer" + }, + "restrictions": { + "anyOf": [ + { + "anyOf": [ + { + "$ref": "#/definitions/IntegerFieldRestrictions" + }, + { + "type": "object", + "properties": { + "if": { + "$ref": "#/definitions/ConditionalRestrictionTest" + }, + "then": { + "anyOf": [ + { + "anyOf": [ + { + "$ref": "#/definitions/IntegerFieldRestrictions" + }, + { + "$ref": "#/definitions/SchemaIntegerField/properties/restrictions/anyOf/0/anyOf/1" + } + ] + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "$ref": "#/definitions/IntegerFieldRestrictions" + }, + { + "$ref": "#/definitions/SchemaIntegerField/properties/restrictions/anyOf/0/anyOf/1" + } + ] + } + } + ] + }, + "else": { + "anyOf": [ + { + "anyOf": [ + { + "$ref": "#/definitions/IntegerFieldRestrictions" + }, + { + "$ref": "#/definitions/SchemaIntegerField/properties/restrictions/anyOf/0/anyOf/1" + } + ] + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "$ref": "#/definitions/IntegerFieldRestrictions" + }, + { + "$ref": "#/definitions/SchemaIntegerField/properties/restrictions/anyOf/0/anyOf/1" + } + ] + } + } + ] + } + }, + "required": [ + "if" + ], + "additionalProperties": false + } + ] + }, + { + "type": "array", + "items": { + "$ref": "#/definitions/SchemaIntegerField/properties/restrictions/anyOf/0" + } + } + ] + } }, - "minItems": 1 + "required": [ + "name", + "valueType" + ], + "additionalProperties": false }, - "References": { + "SchemaNumberField": { "type": "object", - "additionalProperties": { - "anyOf": [ - { - "$ref": "#/definitions/ReferenceArray/items/anyOf/0" + "properties": { + "name": { + "$ref": "#/definitions/Name" + }, + "description": { + "$ref": "#/definitions/SchemaBooleanField/properties/description" + }, + "isArray": { + "$ref": "#/definitions/SchemaBooleanField/properties/isArray" + }, + "meta": { + "$ref": "#/definitions/SchemaBooleanField/properties/meta" + }, + "unique": { + "$ref": "#/definitions/SchemaBooleanField/properties/unique" + }, + "valueType": { + "type": "string", + "const": "number" + }, + "restrictions": { + "anyOf": [ + { + "anyOf": [ + { + "$ref": "#/definitions/NumberFieldRestrictions" + }, + { + "type": "object", + "properties": { + "if": { + "$ref": "#/definitions/ConditionalRestrictionTest" + }, + "then": { + "anyOf": [ + { + "anyOf": [ + { + "$ref": "#/definitions/NumberFieldRestrictions" + }, + { + "$ref": "#/definitions/SchemaNumberField/properties/restrictions/anyOf/0/anyOf/1" + } + ] + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "$ref": "#/definitions/NumberFieldRestrictions" + }, + { + "$ref": "#/definitions/SchemaNumberField/properties/restrictions/anyOf/0/anyOf/1" + } + ] + } + } + ] + }, + "else": { + "anyOf": [ + { + "anyOf": [ + { + "$ref": "#/definitions/NumberFieldRestrictions" + }, + { + "$ref": "#/definitions/SchemaNumberField/properties/restrictions/anyOf/0/anyOf/1" + } + ] + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "$ref": "#/definitions/NumberFieldRestrictions" + }, + { + "$ref": "#/definitions/SchemaNumberField/properties/restrictions/anyOf/0/anyOf/1" + } + ] + } + } + ] + } + }, + "required": [ + "if" + ], + "additionalProperties": false + } + ] + }, + { + "type": "array", + "items": { + "$ref": "#/definitions/SchemaNumberField/properties/restrictions/anyOf/0" + } + } + ] + } + }, + "required": [ + "name", + "valueType" + ], + "additionalProperties": false + }, + "SchemaStringField": { + "type": "object", + "properties": { + "name": { + "$ref": "#/definitions/Name" + }, + "description": { + "$ref": "#/definitions/SchemaBooleanField/properties/description" + }, + "isArray": { + "$ref": "#/definitions/SchemaBooleanField/properties/isArray" + }, + "meta": { + "$ref": "#/definitions/SchemaBooleanField/properties/meta" + }, + "unique": { + "$ref": "#/definitions/SchemaBooleanField/properties/unique" + }, + "valueType": { + "type": "string", + "const": "string" + }, + "restrictions": { + "anyOf": [ + { + "anyOf": [ + { + "$ref": "#/definitions/StringFieldRestrictions" + }, + { + "type": "object", + "properties": { + "if": { + "$ref": "#/definitions/ConditionalRestrictionTest" + }, + "then": { + "anyOf": [ + { + "anyOf": [ + { + "$ref": "#/definitions/StringFieldRestrictions" + }, + { + "$ref": "#/definitions/SchemaStringField/properties/restrictions/anyOf/0/anyOf/1" + } + ] + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "$ref": "#/definitions/StringFieldRestrictions" + }, + { + "$ref": "#/definitions/SchemaStringField/properties/restrictions/anyOf/0/anyOf/1" + } + ] + } + } + ] + }, + "else": { + "anyOf": [ + { + "anyOf": [ + { + "$ref": "#/definitions/StringFieldRestrictions" + }, + { + "$ref": "#/definitions/SchemaStringField/properties/restrictions/anyOf/0/anyOf/1" + } + ] + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "$ref": "#/definitions/StringFieldRestrictions" + }, + { + "$ref": "#/definitions/SchemaStringField/properties/restrictions/anyOf/0/anyOf/1" + } + ] + } + } + ] + } + }, + "required": [ + "if" + ], + "additionalProperties": false + } + ] + }, + { + "type": "array", + "items": { + "$ref": "#/definitions/SchemaStringField/properties/restrictions/anyOf/0" + } + } + ] + } + }, + "required": [ + "name", + "valueType" + ], + "additionalProperties": false + }, + "BooleanFieldRestrictions": { + "type": "object", + "properties": { + "empty": { + "type": "boolean" + }, + "required": { + "type": "boolean" + } + }, + "additionalProperties": false + }, + "IntegerFieldRestrictions": { + "type": "object", + "properties": { + "codeList": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer" + }, + "minItems": 1 + }, + { + "$ref": "#/definitions/ReferenceTag" + } + ] + }, + "empty": { + "type": "boolean" + }, + "required": { + "type": "boolean" + }, + "range": { + "type": "object", + "properties": { + "exclusiveMax": { + "$ref": "#/definitions/IntegerFieldRestrictions/properties/codeList/anyOf/0/items" + }, + "exclusiveMin": { + "$ref": "#/definitions/IntegerFieldRestrictions/properties/codeList/anyOf/0/items" + }, + "max": { + "$ref": "#/definitions/IntegerFieldRestrictions/properties/codeList/anyOf/0/items" + }, + "min": { + "$ref": "#/definitions/IntegerFieldRestrictions/properties/codeList/anyOf/0/items" + } }, - { - "$ref": "#/definitions/ReferenceArray" + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "NumberFieldRestrictions": { + "type": "object", + "properties": { + "codeList": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number" + }, + "minItems": 1 + }, + { + "$ref": "#/definitions/ReferenceTag" + } + ] + }, + "empty": { + "type": "boolean" + }, + "required": { + "type": "boolean" + }, + "range": { + "type": "object", + "properties": { + "exclusiveMax": { + "type": "number" + }, + "exclusiveMin": { + "type": "number" + }, + "max": { + "type": "number" + }, + "min": { + "type": "number" + } }, - { - "$ref": "#/definitions/References" + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "StringFieldRestrictions": { + "type": "object", + "properties": { + "codeList": { + "anyOf": [ + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/definitions/ReferenceTag" + } + ] + }, + "minItems": 1 + }, + { + "$ref": "#/definitions/ReferenceTag" + } + ] + }, + "empty": { + "type": "boolean" + }, + "required": { + "type": "boolean" + }, + "regex": { + "anyOf": [ + { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "$ref": "#/definitions/StringFieldRestrictions/properties/regex/anyOf/0/anyOf/0" + } + } + ] + }, + { + "$ref": "#/definitions/ReferenceTag" + } + ] + } + }, + "additionalProperties": false + }, + "ConditionalRestrictionTest": { + "type": "object", + "properties": { + "conditions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "fields": { + "type": "array", + "items": { + "type": "string" + } + }, + "match": { + "type": "object", + "properties": { + "codeList": { + "anyOf": [ + { + "$ref": "#/definitions/StringFieldRestrictions/properties/codeList/anyOf/0" + }, + { + "$ref": "#/definitions/NumberFieldRestrictions/properties/codeList/anyOf/0" + }, + { + "$ref": "#/definitions/IntegerFieldRestrictions/properties/codeList/anyOf/0" + } + ] + }, + "count": { + "anyOf": [ + { + "type": "number" + }, + { + "$ref": "#/definitions/IntegerFieldRestrictions/properties/range" + } + ] + }, + "exists": { + "type": "boolean" + }, + "range": { + "$ref": "#/definitions/NumberFieldRestrictions/properties/range" + }, + "regex": { + "$ref": "#/definitions/StringFieldRestrictions/properties/regex/anyOf/0" + }, + "value": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "array", + "items": { + "$ref": "#/definitions/ConditionalRestrictionTest/properties/conditions/items/properties/match/properties/value/anyOf/0" + } + }, + { + "type": "integer" + }, + { + "type": "array", + "items": { + "$ref": "#/definitions/ConditionalRestrictionTest/properties/conditions/items/properties/match/properties/value/anyOf/2" + } + }, + { + "type": "number" + }, + { + "type": "array", + "items": { + "$ref": "#/definitions/ConditionalRestrictionTest/properties/conditions/items/properties/match/properties/value/anyOf/4" + } + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "$ref": "#/definitions/ConditionalRestrictionTest/properties/conditions/items/properties/match/properties/value/anyOf/6" + } + } + ] + } + }, + "additionalProperties": false + }, + "case": { + "type": "string", + "enum": [ + "all", + "any", + "none" + ] + } + }, + "required": [ + "fields", + "match" + ], + "additionalProperties": false } - ] - } + }, + "case": { + "$ref": "#/definitions/ConditionalRestrictionTest/properties/conditions/items/properties/case" + } + }, + "required": [ + "conditions" + ], + "additionalProperties": false }, "Meta": { "type": "object", @@ -75,6 +759,40 @@ "minLength": 1, "pattern": "^[^.]+$" }, + "ReferenceArray": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/definitions/ReferenceTag" + } + ] + }, + "minItems": 1 + }, + "ReferenceTag": { + "type": "string", + "pattern": "^#(\\/[-_A-Za-z0-9]+)+$" + }, + "References": { + "type": "object", + "additionalProperties": { + "anyOf": [ + { + "$ref": "#/definitions/ReferenceArray/items/anyOf/0" + }, + { + "$ref": "#/definitions/ReferenceArray" + }, + { + "$ref": "#/definitions/References" + } + ] + } + }, "Schema": { "type": "object", "properties": { @@ -153,302 +871,16 @@ "SchemaField": { "anyOf": [ { - "type": "object", - "properties": { - "name": { - "$ref": "#/definitions/Name" - }, - "description": { - "type": "string" - }, - "isArray": { - "type": "boolean" - }, - "meta": { - "$ref": "#/definitions/Meta" - }, - "valueType": { - "type": "string", - "const": "string" - }, - "restrictions": { - "type": "object", - "properties": { - "codeList": { - "anyOf": [ - { - "type": "array", - "items": { - "anyOf": [ - { - "type": "string" - }, - { - "$ref": "#/definitions/ReferenceTag" - } - ] - }, - "minItems": 1 - }, - { - "$ref": "#/definitions/ReferenceTag" - } - ] - }, - "required": { - "type": "boolean" - }, - "script": { - "anyOf": [ - { - "type": "array", - "items": { - "anyOf": [ - { - "type": "string" - }, - { - "$ref": "#/definitions/ReferenceTag" - } - ] - }, - "minItems": 1 - }, - { - "$ref": "#/definitions/ReferenceTag" - } - ] - }, - "regex": { - "anyOf": [ - { - "type": "string" - }, - { - "$ref": "#/definitions/ReferenceTag" - } - ] - }, - "unique": { - "type": "boolean" - } - }, - "additionalProperties": false - } - }, - "required": [ - "name", - "valueType" - ], - "additionalProperties": false + "$ref": "#/definitions/SchemaStringField" }, { - "type": "object", - "properties": { - "name": { - "$ref": "#/definitions/Name" - }, - "description": { - "$ref": "#/definitions/SchemaField/anyOf/0/properties/description" - }, - "isArray": { - "$ref": "#/definitions/SchemaField/anyOf/0/properties/isArray" - }, - "meta": { - "$ref": "#/definitions/SchemaField/anyOf/0/properties/meta" - }, - "valueType": { - "type": "string", - "const": "number" - }, - "restrictions": { - "type": "object", - "properties": { - "codeList": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "number" - }, - "minItems": 1 - }, - { - "$ref": "#/definitions/ReferenceTag" - } - ] - }, - "required": { - "type": "boolean" - }, - "script": { - "anyOf": [ - { - "$ref": "#/definitions/SchemaField/anyOf/0/properties/restrictions/properties/script/anyOf/0" - }, - { - "$ref": "#/definitions/ReferenceTag" - } - ] - }, - "range": { - "type": "object", - "properties": { - "exclusiveMax": { - "type": "number" - }, - "exclusiveMin": { - "type": "number" - }, - "max": { - "type": "number" - }, - "min": { - "type": "number" - } - }, - "additionalProperties": false - }, - "unique": { - "type": "boolean" - } - }, - "additionalProperties": false - } - }, - "required": [ - "name", - "valueType" - ], - "additionalProperties": false + "$ref": "#/definitions/SchemaNumberField" }, { - "type": "object", - "properties": { - "name": { - "$ref": "#/definitions/Name" - }, - "description": { - "$ref": "#/definitions/SchemaField/anyOf/0/properties/description" - }, - "isArray": { - "$ref": "#/definitions/SchemaField/anyOf/0/properties/isArray" - }, - "meta": { - "$ref": "#/definitions/SchemaField/anyOf/0/properties/meta" - }, - "valueType": { - "type": "string", - "const": "integer" - }, - "restrictions": { - "type": "object", - "properties": { - "codeList": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer" - }, - "minItems": 1 - }, - { - "$ref": "#/definitions/ReferenceTag" - } - ] - }, - "required": { - "type": "boolean" - }, - "script": { - "anyOf": [ - { - "$ref": "#/definitions/SchemaField/anyOf/0/properties/restrictions/properties/script/anyOf/0" - }, - { - "$ref": "#/definitions/ReferenceTag" - } - ] - }, - "range": { - "type": "object", - "properties": { - "exclusiveMax": { - "$ref": "#/definitions/SchemaField/anyOf/2/properties/restrictions/properties/codeList/anyOf/0/items" - }, - "exclusiveMin": { - "$ref": "#/definitions/SchemaField/anyOf/2/properties/restrictions/properties/codeList/anyOf/0/items" - }, - "max": { - "$ref": "#/definitions/SchemaField/anyOf/2/properties/restrictions/properties/codeList/anyOf/0/items" - }, - "min": { - "$ref": "#/definitions/SchemaField/anyOf/2/properties/restrictions/properties/codeList/anyOf/0/items" - } - }, - "additionalProperties": false - }, - "unique": { - "type": "boolean" - } - }, - "additionalProperties": false - } - }, - "required": [ - "name", - "valueType" - ], - "additionalProperties": false + "$ref": "#/definitions/SchemaIntegerField" }, { - "type": "object", - "properties": { - "name": { - "$ref": "#/definitions/Name" - }, - "description": { - "$ref": "#/definitions/SchemaField/anyOf/0/properties/description" - }, - "isArray": { - "$ref": "#/definitions/SchemaField/anyOf/0/properties/isArray" - }, - "meta": { - "$ref": "#/definitions/SchemaField/anyOf/0/properties/meta" - }, - "valueType": { - "type": "string", - "const": "boolean" - }, - "restrictions": { - "type": "object", - "properties": { - "required": { - "type": "boolean" - }, - "script": { - "anyOf": [ - { - "$ref": "#/definitions/SchemaField/anyOf/0/properties/restrictions/properties/script/anyOf/0" - }, - { - "$ref": "#/definitions/ReferenceTag" - } - ] - }, - "unique": { - "type": "boolean" - } - }, - "additionalProperties": false - } - }, - "required": [ - "name", - "valueType" - ], - "additionalProperties": false + "$ref": "#/definitions/SchemaBooleanField" } ] }, diff --git a/packages/dictionary/src/metaSchema/dictionarySchemas.ts b/packages/dictionary/src/metaSchema/dictionarySchemas.ts index 1e159b3..63bb073 100644 --- a/packages/dictionary/src/metaSchema/dictionarySchemas.ts +++ b/packages/dictionary/src/metaSchema/dictionarySchemas.ts @@ -17,17 +17,18 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { z as zod } from 'zod'; +import { z as zod, type ZodType } from 'zod'; import allUnique from '../utils/allUnique'; import { ReferenceTag, References } from './referenceSchemas'; import { + ConditionalRestriction, + ConditionalRestrictionTest, RestrictionCodeListInteger, RestrictionCodeListNumber, RestrictionCodeListString, RestrictionIntegerRange, RestrictionNumberRange, RestrictionRegex, - RestrictionScript, } from './restrictionsSchemas'; /** @@ -66,43 +67,112 @@ export type SchemaFieldValueType = zod.infer; /* ****************************** * * Field Type Restriction Objects * * ****************************** */ -export const StringFieldRestrictions = zod +export const BooleanFieldRestrictions = zod + .object({ empty: zod.boolean(), required: zod.boolean() }) + .partial() + .strict(); +export type BooleanFieldRestrictions = zod.infer; + +const BooleanFieldConditionalRestriction: ZodType> = zod .object({ - codeList: RestrictionCodeListString.or(ReferenceTag), + if: ConditionalRestrictionTest, + then: BooleanFieldRestrictions.or(zod.lazy(() => BooleanFieldConditionalRestriction)) + .or(zod.array(zod.union([BooleanFieldRestrictions, zod.lazy(() => BooleanFieldConditionalRestriction)]))) + .optional(), + else: BooleanFieldRestrictions.or(zod.lazy(() => BooleanFieldConditionalRestriction)) + .or(zod.array(zod.union([BooleanFieldRestrictions, zod.lazy(() => BooleanFieldConditionalRestriction)]))) + .optional(), + }) + .strict(); + +const BooleanFieldRestrictionsObject = BooleanFieldRestrictions.or(BooleanFieldConditionalRestriction); +export type BooleanFieldRestrictionsObject = zod.infer; + +export const IntegerFieldRestrictions = zod + .object({ + codeList: RestrictionCodeListInteger.or(ReferenceTag), + empty: zod.boolean(), required: zod.boolean(), - script: RestrictionScript.or(ReferenceTag), - regex: RestrictionRegex.or(ReferenceTag), - unique: zod.boolean(), + range: RestrictionIntegerRange, }) - .partial(); -export type StringFieldRestrictions = zod.infer; + .partial() + .strict(); +export type IntegerFieldRestrictions = zod.infer; + +const IntegerFieldConditionalRestriction: ZodType> = zod + .object({ + if: ConditionalRestrictionTest, + then: IntegerFieldRestrictions.or(zod.lazy(() => IntegerFieldConditionalRestriction)) + .or(zod.array(zod.union([IntegerFieldRestrictions, zod.lazy(() => IntegerFieldConditionalRestriction)]))) + .optional(), + else: IntegerFieldRestrictions.or(zod.lazy(() => IntegerFieldConditionalRestriction)) + .or(zod.array(zod.union([IntegerFieldRestrictions, zod.lazy(() => IntegerFieldConditionalRestriction)]))) + .optional(), + }) + .strict(); + +const IntegerFieldRestrictionsObject = IntegerFieldRestrictions.or(IntegerFieldConditionalRestriction); +export type IntegerFieldRestrictionsObject = zod.infer; export const NumberFieldRestrictions = zod .object({ codeList: RestrictionCodeListNumber.or(ReferenceTag), + empty: zod.boolean(), required: zod.boolean(), - script: RestrictionScript.or(ReferenceTag), range: RestrictionNumberRange, - unique: zod.boolean(), }) - .partial(); + .partial() + .strict(); export type NumberFieldRestrictions = zod.infer; -export const IntegerFieldRestrictions = zod +const NumberFieldConditionalRestriction: ZodType> = zod .object({ - codeList: RestrictionCodeListInteger.or(ReferenceTag), + if: ConditionalRestrictionTest, + then: NumberFieldRestrictions.or(zod.lazy(() => NumberFieldConditionalRestriction)) + .or(zod.array(zod.union([NumberFieldRestrictions, zod.lazy(() => NumberFieldConditionalRestriction)]))) + .optional(), + else: NumberFieldRestrictions.or(zod.lazy(() => NumberFieldConditionalRestriction)) + .or(zod.array(zod.union([NumberFieldRestrictions, zod.lazy(() => NumberFieldConditionalRestriction)]))) + .optional(), + }) + .strict(); + +const NumberFieldRestrictionsObject = NumberFieldRestrictions.or(NumberFieldConditionalRestriction); +export type NumberFieldRestrictionsObject = zod.infer; + +export const StringFieldRestrictions = zod + .object({ + codeList: RestrictionCodeListString.or(ReferenceTag), + empty: zod.boolean(), required: zod.boolean(), - script: RestrictionScript.or(ReferenceTag), - range: RestrictionIntegerRange, - unique: zod.boolean(), + regex: RestrictionRegex.or(ReferenceTag), }) - .partial(); -export type IntegerFieldRestrictions = zod.infer; + .partial() + .strict(); +export type StringFieldRestrictions = zod.infer; -export const BooleanFieldRestrictions = zod - .object({ required: zod.boolean(), script: RestrictionScript.or(ReferenceTag), unique: zod.boolean() }) - .partial(); -export type BooleanFieldRestrictions = zod.infer; +const StringFieldConditionalRestriction: ZodType> = zod + .object({ + if: ConditionalRestrictionTest, + then: StringFieldRestrictions.or(zod.lazy(() => StringFieldConditionalRestriction)) + .or(zod.array(zod.union([StringFieldRestrictions, zod.lazy(() => StringFieldConditionalRestriction)]))) + .optional(), + else: StringFieldRestrictions.or(zod.lazy(() => StringFieldConditionalRestriction)) + .or(zod.array(zod.union([StringFieldRestrictions, zod.lazy(() => StringFieldConditionalRestriction)]))) + .optional(), + }) + .strict(); + +const StringFieldRestrictionsObject = StringFieldRestrictions.or(StringFieldConditionalRestriction); +export type StringFieldRestrictionsObject = zod.infer; + +export const AnyFieldRestrictions = zod.union([ + BooleanFieldRestrictions, + IntegerFieldRestrictions, + NumberFieldRestrictions, + StringFieldRestrictions, +]); +export type AnyFieldRestrictions = zod.infer; /* ***************** * * Field Definitions * @@ -113,22 +183,23 @@ export const SchemaFieldBase = zod description: zod.string().optional(), isArray: zod.boolean().optional(), meta: DictionaryMeta.optional(), + unique: zod.boolean().optional(), }) .strict(); export type SchemaFieldBase = zod.infer; -export const SchemaStringField = SchemaFieldBase.merge( +export const SchemaBooleanField = SchemaFieldBase.merge( zod.object({ - valueType: zod.literal(SchemaFieldValueType.Values.string), - restrictions: StringFieldRestrictions.optional(), + valueType: zod.literal(SchemaFieldValueType.Values.boolean), + restrictions: BooleanFieldRestrictionsObject.or(BooleanFieldRestrictionsObject.array()).optional(), }), ).strict(); -export type SchemaStringField = zod.infer; +export type SchemaBooleanField = zod.infer; export const SchemaNumberField = SchemaFieldBase.merge( zod.object({ valueType: zod.literal(SchemaFieldValueType.Values.number), - restrictions: NumberFieldRestrictions.optional(), + restrictions: NumberFieldRestrictionsObject.or(NumberFieldRestrictionsObject.array()).optional(), }), ).strict(); export type SchemaNumberField = zod.infer; @@ -136,18 +207,18 @@ export type SchemaNumberField = zod.infer; export const SchemaIntegerField = SchemaFieldBase.merge( zod.object({ valueType: zod.literal(SchemaFieldValueType.Values.integer), - restrictions: IntegerFieldRestrictions.optional(), + restrictions: IntegerFieldRestrictionsObject.or(IntegerFieldRestrictionsObject.array()).optional(), }), ).strict(); export type SchemaIntegerField = zod.infer; -export const SchemaBooleanField = SchemaFieldBase.merge( +export const SchemaStringField = SchemaFieldBase.merge( zod.object({ - valueType: zod.literal(SchemaFieldValueType.Values.boolean), - restrictions: BooleanFieldRestrictions.optional(), + valueType: zod.literal(SchemaFieldValueType.Values.string), + restrictions: StringFieldRestrictionsObject.or(StringFieldRestrictionsObject.array()).optional(), }), ).strict(); -export type SchemaBooleanField = zod.infer; +export type SchemaStringField = zod.infer; export const SchemaField = zod.discriminatedUnion('valueType', [ SchemaStringField, diff --git a/packages/dictionary/src/metaSchema/referenceSchemas.ts b/packages/dictionary/src/metaSchema/referenceSchemas.ts index 9a8daf7..2d3e23b 100644 --- a/packages/dictionary/src/metaSchema/referenceSchemas.ts +++ b/packages/dictionary/src/metaSchema/referenceSchemas.ts @@ -23,7 +23,7 @@ export const ReferenceTag = zod .string() .regex( RegExp('^#(/[-_A-Za-z0-9]+)+$'), - 'Not formatted as a valid reference tag. References must be formatted like `#/path/to/reference', + 'Not formatted as a valid reference tag. References must be formatted like `#/path/to/reference`', ); export type ReferenceTag = zod.infer; diff --git a/packages/dictionary/src/metaSchema/restrictionsSchemas.ts b/packages/dictionary/src/metaSchema/restrictionsSchemas.ts index f9835e4..1497cc4 100644 --- a/packages/dictionary/src/metaSchema/restrictionsSchemas.ts +++ b/packages/dictionary/src/metaSchema/restrictionsSchemas.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { z as zod } from 'zod'; +import { z as zod, type ZodSchema } from 'zod'; import { ReferenceTag } from './referenceSchemas'; import type { Values } from '../types'; @@ -25,6 +25,7 @@ export const Integer = zod.number().int(); export const FieldRestrictionTypes = { codeList: 'codeList', + empty: 'empty', range: 'range', required: 'required', regex: 'regex', @@ -33,9 +34,6 @@ export const FieldRestrictionTypes = { } as const; export type FieldRestrictionType = Values; -export const RestrictionScript = zod.array(zod.string().or(ReferenceTag)).min(1); //TODO: script formatting validation -export type RestrictionScript = zod.infer; - export const RestrictionCodeListString = zod.union([zod.string(), ReferenceTag]).array().min(1); export type RestrictionCodeListString = zod.infer; @@ -45,7 +43,12 @@ export type RestrictionCodeListNumber = zod.infer; -export type RestrictionCodeList = RestrictionCodeListString | RestrictionCodeListNumber | RestrictionCodeListInteger; +export const RestrictionCodeList = zod.union([ + RestrictionCodeListString, + RestrictionCodeListNumber, + RestrictionCodeListInteger, +]); +export type RestrictionCodeList = zod.infer; export const RestrictionNumberRange = zod .object({ @@ -98,7 +101,7 @@ export const RestrictionIntegerRange = zod export const RestrictionRange = RestrictionNumberRange; export type RestrictionRange = zod.infer; -export const RestrictionRegex = zod.string().superRefine((data, context) => { +const RegexString = zod.string().superRefine((data, context) => { try { // Attempt to build regexp from the value RegExp(data); @@ -111,4 +114,99 @@ export const RestrictionRegex = zod.string().superRefine((data, context) => { }); } }); +export const RestrictionRegex = RegexString.or(RegexString.array()); export type RestrictionRegex = zod.infer; + +/* + * Conditions for ConditionalRestrictionTest + */ +export const StringFieldValue = zod.string(); +export const StringArrayFieldValue = StringFieldValue.array(); +export const NumberFieldValue = zod.number(); +export const NumberArrayFieldValue = NumberFieldValue.array(); +export const IntegerFieldValue = zod.number().int(); +export const IntegerArrayFieldValue = IntegerFieldValue.array(); +export const BooleanFieldValue = zod.boolean(); +export const BooleanArrayFieldValue = BooleanFieldValue.array(); + +export const FieldValue = zod.union([ + BooleanFieldValue, + BooleanArrayFieldValue, + IntegerFieldValue, + IntegerArrayFieldValue, + NumberFieldValue, + NumberArrayFieldValue, + StringFieldValue, + StringArrayFieldValue, +]); +export type FieldValue = zod.infer; + +export const ArrayTestCase = zod.enum(['all', 'any', 'none']); +export type ArrayTestCase = zod.infer; + +export const ARRAY_TEST_CASE_DEFAULT = ArrayTestCase.Values.all; + +export const MatchRuleCodeList = RestrictionCodeList; +export type MatchRuleCodeList = zod.infer; + +export const MatchRuleCount = zod.number().or(RestrictionIntegerRange); +export type MatchRuleCount = zod.infer; + +export const MatchRuleExists = zod.boolean(); +export type MatchRuleExists = zod.infer; + +export const MatchRuleRange = RestrictionRange; +export type MatchRuleRange = zod.infer; + +export const MatchRuleRegex = RestrictionRegex; +export type MatchRuleRegex = zod.infer; + +export const MatchRuleValue = FieldValue; +export type MatchRuleValue = zod.infer; + +export const MatchRule = zod.union([ + MatchRuleCodeList, + MatchRuleCount, + MatchRuleExists, + MatchRuleRange, + MatchRuleRegex, + MatchRuleValue, +]); +export type MatchRule = zod.infer; + +export const ConditionMatch = zod + .object({ + codeList: MatchRuleCodeList, + count: MatchRuleCount, + exists: MatchRuleExists, + range: MatchRuleRange, + regex: MatchRuleRegex, + value: MatchRuleValue, + }) + .partial(); +type ConditionMatch = zod.infer; + +export const RestrictionCondition = zod.object({ + fields: zod.string().array(), + match: ConditionMatch, + case: ArrayTestCase.optional(), +}); +export type RestrictionCondition = zod.infer; + +export const ConditionalRestrictionTest = zod.object({ + conditions: zod.array(RestrictionCondition), + case: ArrayTestCase.optional(), +}); +export type ConditionalRestrictionTest = zod.infer; + +export type ConditionalRestriction = { + if: ConditionalRestrictionTest; + then?: + | TRestrictionObject + | ConditionalRestriction + | (TRestrictionObject | ConditionalRestriction)[]; + else?: + | TRestrictionObject + | ConditionalRestriction + | (TRestrictionObject | ConditionalRestriction)[]; +}; diff --git a/packages/dictionary/src/references.ts b/packages/dictionary/src/references.ts index 8c3a9df..0419883 100644 --- a/packages/dictionary/src/references.ts +++ b/packages/dictionary/src/references.ts @@ -17,101 +17,45 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import * as immer from 'immer'; import { cloneDeep, get, isObject, omit } from 'lodash'; -import { Dictionary, ReferenceArray, ReferenceTag, ReferenceValue, References, Schema, TypeUtils } from '.'; +import { + Dictionary, + DictionaryMeta, + ReferenceArray, + ReferenceTag, + ReferenceValue, + References, + Schema, + TypeUtils, + type SchemaField, + type StringFieldRestrictionsObject, +} from '.'; import { InvalidReferenceError } from './errors'; +import { isNumberArray, isStringArray } from './utils/typeUtils'; // This is the union of all schema sections that could have reference values type OutputReferenceValues = ReferenceArray | ReferenceValue; type DiscoveredMap = Map; +const createDiscoveredMap = () => new Map(); + type VisitedSet = Set; +const createVisitedSet = () => new Set(); + +const isReferenceTag = (input: unknown): input is ReferenceTag => ReferenceTag.safeParse(input).success; /** - * Logic for replacing references in an individual schema. + * Convert a ReferenceTag value into a dot separated path that can be used by lodash _.get to find the value + * in the references object. * - * This is used by the replaceReferences method that replaces references in ALL schemas. By allowing the caller - * to provide the discovered/visited objects that are used in the recursive logic we can let the replaceReferences - * method reuse the same dictionaries across all schemas. - * @param schema - * @param references - * @param discovered - * @param visited - * @returns + * @example + * const referenceTag = `#/some/path`; + * + * const path = referenceTagToObjectPath(referenceTag); + * // some.path + * + * const referenceValue = _.get(references, path); */ -const internalReplaceSchemaReferences = ( - schema: Schema, - references: References, - discovered: DiscoveredMap, - visited: VisitedSet, -): Schema => { - const clone = cloneDeep(schema); - - clone.fields.forEach((field) => { - // Process Field Meta: - if (field.meta !== undefined) { - for (const key in field.meta) { - const value = field.meta[key]; - if (isReferenceTag(value)) { - const replacement = resolveReference(value, references, discovered, visited); - if (Array.isArray(replacement)) { - throw new InvalidReferenceError( - `Field '${field.name}' has meta field '${key}' with a reference '${value}' that resolves to an array. Meta fields must be string, number, or boolean.`, - ); - } - } - } - } - // Process Field Restrictions: - if (field.restrictions !== undefined) { - // reusable functions to simplify converting - const resolveRestriction = (value: string | string[]) => - resolveAllReferences(value, references, discovered, visited); - const resolveNoArrays = (value: string | string[], restrictionName: string) => { - const output = resolveRestriction(value); - if (Array.isArray(output)) { - throw new InvalidReferenceError( - `Field '${field.name}' has restriction '${restrictionName}' with a reference '${value}' that resolves to an array. This restriction must be a string.`, - ); - } - return output; - }; - switch (field.valueType) { - // Each field type has different allowed restriction types, we need to handle the reference replacement rules carefully - // to ensure the output schema adhers to the type rules. - // All the checking for undefined prevents us from adding properties with value undefined into the field's ouput JSON - case 'string': - if (field.restrictions.codeList !== undefined) { - field.restrictions.codeList = TypeUtils.asArray(resolveRestriction(field.restrictions.codeList)); - } - if (field.restrictions.regex !== undefined) { - field.restrictions.regex = resolveNoArrays(field.restrictions.regex, 'regex'); - } - if (field.restrictions.script !== undefined) { - field.restrictions.script = TypeUtils.asArray(resolveRestriction(field.restrictions.script)); - } - break; - case 'number': - if (field.restrictions.script !== undefined) { - field.restrictions.script = TypeUtils.asArray(resolveRestriction(field.restrictions.script)); - } - break; - case 'integer': - if (field.restrictions.script !== undefined) { - field.restrictions.script = TypeUtils.asArray(resolveRestriction(field.restrictions.script)); - } - break; - case 'boolean': - break; - } - } - }); - - return clone; -}; - -const isReferenceTag = (input: unknown): input is ReferenceTag => ReferenceTag.safeParse(input).success; const referenceTagToObjectPath = (value: ReferenceTag): string => { try { return value.split('/').slice(1).join('.'); @@ -120,6 +64,33 @@ const referenceTagToObjectPath = (value: ReferenceTag): string => { } }; +/** + * For an array of strings, replace all values that are ReferenceTags with the corresponding reference value. + */ +const resolveArrayReferences = ( + value: string[], + references: References, + discovered: DiscoveredMap, + visited: VisitedSet, +): string[] => + value.flatMap((item) => (isReferenceTag(item) ? resolveReference(item, references, discovered, visited) : item)); + +const transformOneOrMany = (data: TInput | TInput[], transform: (item: TInput) => TOutput) => { + if (Array.isArray(data)) { + return data.map(transform); + } else { + return transform(data); + } +}; + +/** + * + * @param value + * @param references + * @param discovered + * @param visited + * @returns + */ const resolveAllReferences = ( value: string | string[], references: References, @@ -127,9 +98,7 @@ const resolveAllReferences = ( visited: VisitedSet, ): string | string[] => { if (Array.isArray(value)) { - return value.flatMap((item) => - isReferenceTag(item) ? resolveReference(item, references, discovered, visited) : item, - ); + return resolveArrayReferences(value, references, discovered, visited); } if (isReferenceTag(value)) { return resolveReference(value, references, discovered, visited); @@ -170,9 +139,7 @@ const resolveReference = ( throw new InvalidReferenceError(`No reference found for provided tag '${tag}'.`); } if (Array.isArray(replacement)) { - const output = replacement.flatMap((item) => - isReferenceTag(item) ? resolveReference(item, references, discovered, visited) : item, - ); + const output = resolveArrayReferences(replacement, references, discovered, visited); discovered.set(tag, output); return output; } else if (isReferenceTag(replacement)) { @@ -186,6 +153,174 @@ const resolveReference = ( } }; +/** + * Warning: This mutates the meta argument object. This is meant for use within this module only and should not be exported. + */ +const recursiveReplaceMetaReferences = ( + meta: DictionaryMeta, + references: References, + discovered: DiscoveredMap, + visited: VisitedSet, +): DictionaryMeta => { + for (const [key, value] of Object.entries(meta)) { + if (isStringArray(value)) { + // value is an array of strings, we want to check if any of the values are reference tags and replace them if they are. + const replacement = resolveArrayReferences(value, references, discovered, visited); + meta[key] = replacement; + } else if (isReferenceTag(value)) { + // value is a reference tag, we replace it with the corresponding reference. + const replacement = resolveReference(value, references, discovered, visited); + meta[key] = replacement; + } else if (typeof value === 'object' && !Array.isArray(value)) { + // value is a nested meta object, send it into this function! recursion! + const replacement = recursiveReplaceMetaReferences(value, references, discovered, visited); + meta[key] = replacement; + } + } + + return meta; +}; + +/** + * Warning: This mutates the restrictionsObject argument object. This is meant for use within this module only and should not be exported. + */ +const replaceReferencesInStringRestrictionsObject = ( + restrictionsObject: StringFieldRestrictionsObject, + references: References, + discovered: DiscoveredMap, + visited: VisitedSet, +) => { + if ('if' in restrictionsObject) { + // Do replacements inside the if conditions + restrictionsObject.if.conditions = restrictionsObject.if.conditions.map((condition) => { + if (condition.match.codeList && !isNumberArray(condition.match.codeList)) { + condition.match.codeList = TypeUtils.asArray( + resolveAllReferences(condition.match.codeList, references, discovered, visited), + ); + } + if (typeof condition.match.value === 'string' || isStringArray(condition.match.value)) { + condition.match.value = resolveAllReferences(condition.match.value, references, discovered, visited); + } + if (condition.match.regex) { + condition.match.regex = TypeUtils.asArray( + resolveAllReferences(condition.match.regex, references, discovered, visited), + ); + } + + return condition; + }); + + const transform = (data: StringFieldRestrictionsObject) => + replaceReferencesInStringRestrictionsObject(data, references, discovered, visited); + if (restrictionsObject.then) { + restrictionsObject.then = transformOneOrMany(restrictionsObject.then, transform); + } + if (restrictionsObject.else) { + restrictionsObject.else = transformOneOrMany(restrictionsObject.else, transform); + } + } else { + if (restrictionsObject.codeList !== undefined) { + restrictionsObject.codeList = TypeUtils.asArray( + resolveAllReferences(restrictionsObject.codeList, references, discovered, visited), + ); + } + if (restrictionsObject.regex !== undefined) { + const updatedRegex = resolveAllReferences(restrictionsObject.regex, references, discovered, visited); + restrictionsObject.regex = updatedRegex; + } + } + return restrictionsObject; +}; + +/** + * Warning: This mutates the field argument object. This is meant for use within this module only and should not be exported. + */ +const internalReplaceFieldReferences = ( + field: SchemaField, + references: References, + discovered: DiscoveredMap, + visited: VisitedSet, +): SchemaField => { + // Process Field Meta: + if (field.meta !== undefined) { + field.meta = recursiveReplaceMetaReferences(field.meta, references, discovered, visited); + } + + // Process Field Restrictions: + if (field.restrictions !== undefined) { + // Each field type has different allowed restriction types, + // we need to handle the reference replacement rules carefully + // to ensure the output schema adheres to the type rules. + switch (field.valueType) { + case 'string': { + field.restrictions = transformOneOrMany(field.restrictions, (restriction) => + replaceReferencesInStringRestrictionsObject(restriction, references, discovered, visited), + ); + break; + } + case 'number': { + break; + } + case 'integer': { + break; + } + case 'boolean': { + break; + } + } + } + return field; +}; + +/** + * Logic for replacing references in a schema. + * + * This is used by the replaceReferences method that replaces references in ALL schemas. By allowing the caller + * to provide the discovered/visited objects that are used in the recursive logic we can let the replaceReferences + * method reuse the same dictionaries across all schemas. + * + * Warning: This mutates the schema argument object. This is meant for use within this module only and should not be exported. + * @param schema + * @param references + * @param discovered + * @param visited + * @returns + */ +const internalReplaceSchemaReferences = ( + schema: Schema, + references: References, + discovered: DiscoveredMap, + visited: VisitedSet, +): Schema => { + if (schema.meta) { + schema.meta = recursiveReplaceMetaReferences(schema.meta, references, discovered, visited); + } + + schema.fields.forEach((field) => { + internalReplaceFieldReferences(field, references, discovered, visited); + }); + + return schema; +}; + +/** + * Replace all ReferenceTags in in the restrictions and meta sections of the field definition object with + * values retrieved from the `references` object. + */ +export const replaceMetaReferences = (meta: DictionaryMeta, references: References): DictionaryMeta => { + const clone = cloneDeep(meta); + return recursiveReplaceMetaReferences(clone, references, createDiscoveredMap(), createVisitedSet()); +}; + +/** + * Replace all ReferenceTags in in the restrictions and meta sections of the field definition object with + * values retrieved from the `references` object. + */ +export const replaceFieldReferences = (field: SchemaField, references: References): SchemaField => { + const clone = cloneDeep(field); + return internalReplaceFieldReferences(clone, references, createDiscoveredMap(), createVisitedSet()); +}; + /** * Replace all Reference Tags in the restrictions and meta sections of the schema with values retrieved from * the `references` argument. @@ -193,13 +328,10 @@ const resolveReference = ( * @param references * @return schema clone with references replaced */ -export const replaceSchemaReferences = (schema: Schema, references: References) => - internalReplaceSchemaReferences( - schema, - references, - new Map(), - new Set(), - ); +export const replaceSchemaReferences = (schema: Schema, references: References) => { + const clone = cloneDeep(schema); + return internalReplaceSchemaReferences(clone, references, createDiscoveredMap(), createVisitedSet()); +}; /** * Replace all ReferenceTags found in dictionary schemas with the values retrieved from the dictionary's references. @@ -207,15 +339,14 @@ export const replaceSchemaReferences = (schema: Schema, references: References) * @returns Clone of dictionary with reference replacements */ export const replaceReferences = (dictionary: Dictionary): Dictionary => { + const clone = cloneDeep(omit(dictionary, 'references')); const references = dictionary.references || {}; - const discovered: DiscoveredMap = new Map(); - const visited: VisitedSet = new Set(); + const discovered = createDiscoveredMap(); + const visited = createVisitedSet(); - const updatedDictionary = immer.produce(dictionary, (draft) => { - draft.schemas = draft.schemas.map((schema) => - internalReplaceSchemaReferences(schema, references, discovered, visited), - ); - }); + clone.schemas = dictionary.schemas.map((schema) => + internalReplaceSchemaReferences(schema, references, discovered, visited), + ); - return omit(updatedDictionary, 'references'); + return clone; }; diff --git a/packages/dictionary/src/utils/resolveRestrictions.ts b/packages/dictionary/src/utils/resolveRestrictions.ts deleted file mode 100644 index 74460fd..0000000 --- a/packages/dictionary/src/utils/resolveRestrictions.ts +++ /dev/null @@ -1,51 +0,0 @@ -// import { -// BooleanFieldRestrictions, -// IntegerFieldRestrictions, -// NumberFieldRestrictions, -// SchemaBooleanField, -// SchemaField, -// SchemaIntegerField, -// SchemaNumberField, -// SchemaStringField, -// StringFieldRestrictions, -// } from 'types'; - -// type SingleElement = T extends readonly (infer Element)[] ? Element : T; - -// /** -// * Given a schema field, get the type of the restriction object -// */ -// export type RestrictionObject = T extends { restrictions?: infer RestrictionType } -// ? SingleElement -// : never; - -// type BR = RestrictionObject; -// const x: BR = {}; - -// export const resolveRestrictions = ( -// field: T, -// ): RestrictionObject => { -// const y: RestrictionObject = { required: true }; - -// if (!field.restrictions) { -// return {}; -// } -// if (field.valueType === 'integer') { -// const restrictions = field.restrictions; -// return {}; -// } -// return {}; -// // if(!field.restrictions) { -// // return {}; -// // } -// // if(!Array.isArray(field.)) -// }; - -// type A = { type: 'a'; thing: number }; -// type B = { type: 'b'; thing: string }; -// type DUnion = A | B; - -// type ThingType = T extends { thing: infer U } ? U : never; - -// type AThing = ThingType; -// type BThing = ThingType; diff --git a/packages/dictionary/src/utils/schemaUtils.ts b/packages/dictionary/src/utils/schemaUtils.ts index 7f6b520..69231cf 100644 --- a/packages/dictionary/src/utils/schemaUtils.ts +++ b/packages/dictionary/src/utils/schemaUtils.ts @@ -1,18 +1,33 @@ +import { TypeUtils } from '.'; import type { Schema, SchemaField } from '../metaSchema'; /** * Get an array of fields from this schema that have the required restriction set to true + * + * Note: this does not consider conditional restrictions that could make + * the field required or optional depending on the values of each record. * @param schema * @returns */ export const getRequiredFields = (schema: Schema): SchemaField[] => - schema.fields.filter((field) => field.restrictions?.required); + schema.fields.filter((field) => + TypeUtils.asArray(field.restrictions).some( + (restrictionObject) => restrictionObject && 'required' in restrictionObject && restrictionObject?.required, + ), + ); /** * Get an array of fields from this schema that are optional, - * meaning they do not have the required restriction set to true + * meaning they do not have the required restriction set to true. + * + * Note: this does not consider conditional restrictions that could make + * the field required or optional depending on the values of each record. * @param schema * @returns */ export const getOptionalFields = (schema: Schema): SchemaField[] => - schema.fields.filter((field) => !field.restrictions?.required); + schema.fields.filter((field) => + TypeUtils.asArray(field.restrictions).every( + (restrictionObject) => !(restrictionObject && 'required' in restrictionObject && restrictionObject?.required), + ), + ); diff --git a/packages/dictionary/src/utils/typeUtils.ts b/packages/dictionary/src/utils/typeUtils.ts index 7f404d0..e8e9c18 100644 --- a/packages/dictionary/src/utils/typeUtils.ts +++ b/packages/dictionary/src/utils/typeUtils.ts @@ -26,6 +26,44 @@ */ export const asArray = (val: T | T[]): T[] => (Array.isArray(val) ? val : [val]); +/** + * Given a predicate function that checks for type `T`, this will create a new predicate funcion that + * will check if a value is of type `T[]`. + * + * @example + * // Create type and predicate for `Person`: + * type Person = { name: string; age: number }; + * const isPerson = (value: unknown): value is Person => + * !!value && + * typeof value === 'object' && + * 'name' in value && + * typeof value.name === 'string' && + * 'age' in value && + * typeof value.age === 'number'; + * + * // Use `isArrayOf` and the new predicate to create `isPersonArray`: + * const isPersonArray = isArrayOf(isPerson); + * + * // Usage of `isPersonArray`: + * isPersonArray([{name:'Lisa', age: 8}, {name: 'Bart', age: 10}]); // true + * isPersonArray(['not a person']); // false + * isPersonArray('not an array'); // false + * isPersonArray([{name:'Lisa', age: 8}, {not: 'a person'}]); // false + * @param predicate + * @returns + */ +export const isArrayOf = + (predicate: (value: unknown) => value is T) => + (value: unknown) => + Array.isArray(value) && value.every(predicate); + +/** + * Determines if a variable is of type `boolean[]`. + * @param value + * @returns + */ +export const isBooleanArray = isArrayOf((value: unknown) => typeof value === 'boolean'); + /** * Checks that the input does not equal undefined (and lets the type checker know). * @@ -38,3 +76,42 @@ export const asArray = (val: T | T[]): T[] => (Array.isArray(val) ? val : [va * const stringArray = combinedArray.filter(isDefined); // type is: Array */ export const isDefined = (input: T | undefined) => input !== undefined; + +/** + * Determines if a variable is a number, with added restriction that it is Finite. + * This eliminates the values `NaN` and `Infinity`. + * + * Note: this is just a wrapper on `Number.isFinite` which is used by Lectern for identifying numbers in data value type checks. + * @param value + * @returns + */ +export const isNumber = (value: unknown): value is number => Number.isFinite(value); +/** + * Determines if variable is of type number[], with added restriction that every element is Finite. + * @param value + * @returns + */ +export const isNumberArray = isArrayOf(isNumber); + +/** + * Determines if a variable is of type string[] + * @param value + * @returns + */ +export const isStringArray = isArrayOf((value: unknown) => typeof value === 'string'); + +/** + * Determines if a variable is a number, with added restriction that it is an Integer. + * + * Note: This is a wrapper over `Number.isInteger` which is used by Lectern for identifying integers in data value type checks. + * @param value + * @returns + */ +export const isInteger = (value: unknown): value is number => Number.isInteger(value); + +/** + * Determines if a variables is of type number[], with add restriction that every element is an Integer. + * @param value + * @returns + */ +export const isIntegerArray = isArrayOf(isInteger); diff --git a/packages/dictionary/test/diff.spec.ts b/packages/dictionary/test/diff.spec.ts index 9a305dc..6ca1745 100644 --- a/packages/dictionary/test/diff.spec.ts +++ b/packages/dictionary/test/diff.spec.ts @@ -30,9 +30,9 @@ describe('Compute diff report between dictionary versions', () => { expect(diffReport.get('donor.donor_submitter_id')?.diff).to.deep.eq({ meta: { displayName: { type: 'deleted', data: 'Submitter Donor ID' } }, restrictions: { - script: { + regex: { type: 'updated', - data: { added: ['(field)=>field.length > 6'], deleted: ['(field)=>field.length > 5'] }, + data: '^[\\w]*$', }, }, }); diff --git a/packages/dictionary/test/fixtures/diff/initial.ts b/packages/dictionary/test/fixtures/diff/initial.ts index 54e0923..4233227 100644 --- a/packages/dictionary/test/fixtures/diff/initial.ts +++ b/packages/dictionary/test/fixtures/diff/initial.ts @@ -36,7 +36,7 @@ const DIFF_DICTIONARY_INITIAL: Dictionary = { key: true, }, restrictions: { - script: ['(field)=>field.length > 5'], + regex: '^[\\w]+$', }, }, { diff --git a/packages/dictionary/test/fixtures/diff/updated.ts b/packages/dictionary/test/fixtures/diff/updated.ts index 8186c54..664769b 100644 --- a/packages/dictionary/test/fixtures/diff/updated.ts +++ b/packages/dictionary/test/fixtures/diff/updated.ts @@ -35,7 +35,7 @@ const DIFF_DICTIONARY_UPDATED: Dictionary = { key: true, }, restrictions: { - script: ['(field)=>field.length > 6'], + regex: '^[\\w]*$', }, }, { diff --git a/packages/dictionary/test/fixtures/references/codeList_references/input.ts b/packages/dictionary/test/fixtures/references/codeList_references/input.ts index 3378a57..4b4d61d 100644 --- a/packages/dictionary/test/fixtures/references/codeList_references/input.ts +++ b/packages/dictionary/test/fixtures/references/codeList_references/input.ts @@ -35,7 +35,9 @@ const content: Dictionary = { meta: { default: 'Unknown', }, - restrictions: {}, + restrictions: { + codeList: '#/SINGLE_VALUE', + }, }, ], }, @@ -43,6 +45,7 @@ const content: Dictionary = { references: { ID_REG_EXP: '^[\\w\\s\\W]{5,}$', SEX: ['Male', 'Female'], + SINGLE_VALUE: 'this', }, }; export default content; diff --git a/packages/dictionary/test/fixtures/references/codeList_references/output.ts b/packages/dictionary/test/fixtures/references/codeList_references/output.ts index c18a710..0105cdf 100644 --- a/packages/dictionary/test/fixtures/references/codeList_references/output.ts +++ b/packages/dictionary/test/fixtures/references/codeList_references/output.ts @@ -35,7 +35,7 @@ const content: Dictionary = { meta: { default: 'Unknown', }, - restrictions: {}, + restrictions: { codeList: ['this'] }, }, ], }, diff --git a/packages/dictionary/test/fixtures/references/empty_references_section/input.ts b/packages/dictionary/test/fixtures/references/empty_references_section/input.ts index 865206a..0a7b348 100644 --- a/packages/dictionary/test/fixtures/references/empty_references_section/input.ts +++ b/packages/dictionary/test/fixtures/references/empty_references_section/input.ts @@ -17,7 +17,7 @@ const content: Dictionary = { key: true, }, restrictions: { - script: ['() => field.length > 5'], + regex: '^[\\w]*$', }, }, { diff --git a/packages/dictionary/test/fixtures/references/empty_references_section/output.ts b/packages/dictionary/test/fixtures/references/empty_references_section/output.ts index d0a85b2..f59a136 100644 --- a/packages/dictionary/test/fixtures/references/empty_references_section/output.ts +++ b/packages/dictionary/test/fixtures/references/empty_references_section/output.ts @@ -17,7 +17,7 @@ const content: Dictionary = { key: true, }, restrictions: { - script: ['() => field.length > 5'], + regex: '^[\\w]*$', }, }, { diff --git a/packages/dictionary/test/fixtures/references/nested_meta_references/input.ts b/packages/dictionary/test/fixtures/references/nested_meta_references/input.ts new file mode 100644 index 0000000..d7cadd9 --- /dev/null +++ b/packages/dictionary/test/fixtures/references/nested_meta_references/input.ts @@ -0,0 +1,29 @@ +import { Dictionary } from '../../../../src'; + +const content: Dictionary = { + name: 'Test Dictionary', + version: '1.2', + schemas: [ + { + name: 'donor', + description: 'Donor Entity', + fields: [ + { + name: 'donor_submitter_id', + valueType: 'string', + description: 'Unique identifier for donor; assigned by data provider', + meta: { + displayName: 'Submitter Donor ID', + exampleReference: '#/meta/example', + nested1: { nested2: { nested3: ['#/meta/nestedMeta/example', 'array'] } }, + }, + }, + ], + }, + ], + references: { + text: 'text', + meta: { example: 'This is an example', nestedMeta: { example: ['some', '#/text', 'in an'] } }, + }, +}; +export default content; diff --git a/packages/dictionary/test/fixtures/references/nested_meta_references/output.ts b/packages/dictionary/test/fixtures/references/nested_meta_references/output.ts new file mode 100644 index 0000000..0f3edc7 --- /dev/null +++ b/packages/dictionary/test/fixtures/references/nested_meta_references/output.ts @@ -0,0 +1,25 @@ +import { Dictionary } from '../../../../src'; + +const content: Dictionary = { + name: 'Test Dictionary', + version: '1.2', + schemas: [ + { + name: 'donor', + description: 'Donor Entity', + fields: [ + { + name: 'donor_submitter_id', + valueType: 'string', + description: 'Unique identifier for donor; assigned by data provider', + meta: { + displayName: 'Submitter Donor ID', + exampleReference: 'This is an example', + nested1: { nested2: { nested3: ['some', 'text', 'in an', 'array'] } }, + }, + }, + ], + }, + ], +}; +export default content; diff --git a/packages/dictionary/test/fixtures/references/script_references/input.ts b/packages/dictionary/test/fixtures/references/no_referece_tags/input.ts similarity index 53% rename from packages/dictionary/test/fixtures/references/script_references/input.ts rename to packages/dictionary/test/fixtures/references/no_referece_tags/input.ts index f386584..c0a3697 100644 --- a/packages/dictionary/test/fixtures/references/script_references/input.ts +++ b/packages/dictionary/test/fixtures/references/no_referece_tags/input.ts @@ -9,18 +9,23 @@ const content: Dictionary = { description: 'Donor Entity', fields: [ { - name: 'count', - valueType: 'number', + name: 'donor_submitter_id', + valueType: 'string', + description: 'Unique identifier for donor; assigned by data provider', + meta: { + displayName: 'Submitter Donor ID', + key: true, + }, restrictions: { - script: '#/IS_EVEN', + regex: '^[\\w]*$', }, }, { - name: 'score', + name: 'gender', valueType: 'string', description: 'Donor Biological Sex', restrictions: { - script: ['(value) => value/1000 > 9', '#/IS_EVEN'], + codeList: ['Male', 'Female', 'Other'], }, }, { @@ -36,7 +41,15 @@ const content: Dictionary = { }, ], references: { - IS_EVEN: '(value) => value % 2', + regex: { + REPEATED_TEXT: '(\\w+).*\\1', + ALPHA_ONLY: '^[A-Za-z]*$', + COMBINED: ['#/regex/ID_REG_EXP', '#/regex/ALPHA_ONLY'], + }, + enums: { + SEX: ['Male', 'Female', 'Other'], + }, }, }; + export default content; diff --git a/packages/dictionary/test/fixtures/references/script_references/output.ts b/packages/dictionary/test/fixtures/references/no_referece_tags/output.ts similarity index 64% rename from packages/dictionary/test/fixtures/references/script_references/output.ts rename to packages/dictionary/test/fixtures/references/no_referece_tags/output.ts index 169b191..f59a136 100644 --- a/packages/dictionary/test/fixtures/references/script_references/output.ts +++ b/packages/dictionary/test/fixtures/references/no_referece_tags/output.ts @@ -9,18 +9,23 @@ const content: Dictionary = { description: 'Donor Entity', fields: [ { - name: 'count', - valueType: 'number', + name: 'donor_submitter_id', + valueType: 'string', + description: 'Unique identifier for donor; assigned by data provider', + meta: { + displayName: 'Submitter Donor ID', + key: true, + }, restrictions: { - script: ['(value) => value % 2'], + regex: '^[\\w]*$', }, }, { - name: 'score', + name: 'gender', valueType: 'string', description: 'Donor Biological Sex', restrictions: { - script: ['(value) => value/1000 > 9', '(value) => value % 2'], + codeList: ['Male', 'Female', 'Other'], }, }, { diff --git a/packages/dictionary/test/fixtures/references/no_references_section/input.ts b/packages/dictionary/test/fixtures/references/no_references_section/input.ts index 174b9d3..873c207 100644 --- a/packages/dictionary/test/fixtures/references/no_references_section/input.ts +++ b/packages/dictionary/test/fixtures/references/no_references_section/input.ts @@ -17,7 +17,7 @@ const content: Dictionary = { key: true, }, restrictions: { - script: ['() => field.length > 5'], + regex: '^[\\w]*$', }, }, { diff --git a/packages/dictionary/test/fixtures/references/no_references_section/output.ts b/packages/dictionary/test/fixtures/references/no_references_section/output.ts index d0a85b2..f59a136 100644 --- a/packages/dictionary/test/fixtures/references/no_references_section/output.ts +++ b/packages/dictionary/test/fixtures/references/no_references_section/output.ts @@ -17,7 +17,7 @@ const content: Dictionary = { key: true, }, restrictions: { - script: ['() => field.length > 5'], + regex: '^[\\w]*$', }, }, { diff --git a/packages/dictionary/test/fixtures/references/regex_reference/input_with_array.ts b/packages/dictionary/test/fixtures/references/regex_reference_with_array/input.ts similarity index 90% rename from packages/dictionary/test/fixtures/references/regex_reference/input_with_array.ts rename to packages/dictionary/test/fixtures/references/regex_reference_with_array/input.ts index bb394be..8462445 100644 --- a/packages/dictionary/test/fixtures/references/regex_reference/input_with_array.ts +++ b/packages/dictionary/test/fixtures/references/regex_reference_with_array/input.ts @@ -19,7 +19,7 @@ const content: Dictionary = { ], references: { regex: { - ID_FORMAT: ['bad', 'reference'], + ID_FORMAT: ['good', 'reference'], }, }, }; diff --git a/packages/dictionary/test/fixtures/references/regex_reference_with_array/output.ts b/packages/dictionary/test/fixtures/references/regex_reference_with_array/output.ts new file mode 100644 index 0000000..03f856d --- /dev/null +++ b/packages/dictionary/test/fixtures/references/regex_reference_with_array/output.ts @@ -0,0 +1,21 @@ +import { Dictionary } from '../../../../src'; + +const content: Dictionary = { + name: 'Test Dictionary', + version: '1.0', + schemas: [ + { + name: 'participant', + fields: [ + { + name: 'some_id', + valueType: 'string', + restrictions: { + regex: ['good', 'reference'], + }, + }, + ], + }, + ], +}; +export default content; diff --git a/packages/dictionary/test/fixtures/references/restrictions_array_with_references/input.ts b/packages/dictionary/test/fixtures/references/restrictions_array_with_references/input.ts new file mode 100644 index 0000000..640b0f5 --- /dev/null +++ b/packages/dictionary/test/fixtures/references/restrictions_array_with_references/input.ts @@ -0,0 +1,53 @@ +import { Dictionary } from '../../../../src'; + +const content: Dictionary = { + name: 'Test Dictionary', + version: '1.2', + schemas: [ + { + name: 'donor', + description: 'Donor Entity', + fields: [ + { + name: 'imaginary_field', + valueType: 'string', + description: 'Nonsense example to test an array of restriction objects.', + meta: { + displayName: 'Submitter Donor ID', + key: true, + }, + restrictions: [ + { + regex: ['#/regex/REPEATED_TEXT', '#/regex/ALPHA_ONLY'], + }, + { + codeList: '#/enums/SEX', + required: true, + }, + ], + }, + { + name: 'nonsense_field', + valueType: 'string', + description: 'another meaningless example testing complex references within an array.', + restrictions: [ + { + regex: '#/regex/COMBINED', + }, + ], + }, + ], + }, + ], + references: { + regex: { + REPEATED_TEXT: '(\\w+).*\\1', + ALPHA_ONLY: '^[A-Za-z]*$', + COMBINED: ['#/regex/REPEATED_TEXT', '#/regex/ALPHA_ONLY'], + }, + enums: { + SEX: ['Male', 'Female', 'Other'], + }, + }, +}; +export default content; diff --git a/packages/dictionary/test/fixtures/references/restrictions_array_with_references/output.ts b/packages/dictionary/test/fixtures/references/restrictions_array_with_references/output.ts new file mode 100644 index 0000000..ee11260 --- /dev/null +++ b/packages/dictionary/test/fixtures/references/restrictions_array_with_references/output.ts @@ -0,0 +1,43 @@ +import { Dictionary } from '../../../../src'; + +const content: Dictionary = { + name: 'Test Dictionary', + version: '1.2', + schemas: [ + { + name: 'donor', + description: 'Donor Entity', + fields: [ + { + name: 'imaginary_field', + valueType: 'string', + description: 'Nonsense example to test an array of restriction objects.', + meta: { + displayName: 'Submitter Donor ID', + key: true, + }, + restrictions: [ + { + regex: ['(\\w+).*\\1', '^[A-Za-z]*$'], + }, + { + codeList: ['Male', 'Female', 'Other'], + required: true, + }, + ], + }, + { + name: 'nonsense_field', + valueType: 'string', + description: 'another meaningless example testing complex references within an array.', + restrictions: [ + { + regex: ['(\\w+).*\\1', '^[A-Za-z]*$'], + }, + ], + }, + ], + }, + ], +}; +export default content; diff --git a/packages/dictionary/test/dictionaryTypes.spec.ts b/packages/dictionary/test/metaSchema/dictionarySchemas.spec.ts similarity index 77% rename from packages/dictionary/test/dictionaryTypes.spec.ts rename to packages/dictionary/test/metaSchema/dictionarySchemas.spec.ts index 2637f8a..460365e 100644 --- a/packages/dictionary/test/dictionaryTypes.spec.ts +++ b/packages/dictionary/test/metaSchema/dictionarySchemas.spec.ts @@ -18,23 +18,9 @@ */ import { expect } from 'chai'; -import { - BooleanFieldRestrictions, - Dictionary, - DictionaryMeta, - Integer, - IntegerFieldRestrictions, - NameValue, - NumberFieldRestrictions, - RestrictionIntegerRange, - RestrictionNumberRange, - Schema, - SchemaField, - StringFieldRestrictions, - VersionString, -} from '../src'; +import { Dictionary, DictionaryMeta, NameValue, Schema, SchemaField, VersionString } from '../../src'; -describe('Dictionary Types', () => { +describe('Dictionary Schemas', () => { describe('NameValue', () => { it('Rejects empty string', () => { expect(NameValue.safeParse('').success).false; @@ -54,75 +40,116 @@ describe('Dictionary Types', () => { }); }); - describe('Integer', () => { - it("Can't be float", () => { - expect(Integer.safeParse(1.3).success).false; - expect(Integer.safeParse(2.0000001).success).false; - // Note: float precision issues, if the float resolves to a whole number the value will be accepted. - }); - it("Can't be string, boolean, object, array", () => { - expect(Integer.safeParse('1').success).false; - expect(Integer.safeParse(true).success).false; - expect(Integer.safeParse([1]).success).false; - expect(Integer.safeParse({}).success).false; - expect(Integer.safeParse({ thing: 1 }).success).false; - }); - it('Can be integer', () => { - expect(Integer.safeParse(1).success).true; - expect(Integer.safeParse(0).success).true; - expect(Integer.safeParse(-1).success).true; - expect(Integer.safeParse(1123).success).true; - }); - }); - describe('RangeRestriction', () => { - it("Integer Range Can't have exclusiveMin and Min", () => { - expect(RestrictionIntegerRange.safeParse({ exclusiveMin: 0, min: 0 }).success).false; - expect(RestrictionIntegerRange.safeParse({ min: 0 }).success).true; - expect(RestrictionIntegerRange.safeParse({ exclusiveMin: 0 }).success).true; - }); - it("Integer Range Can't have exclusiveMax and Max", () => { - expect(RestrictionIntegerRange.safeParse({ exclusiveMax: 0, max: 0 }).success).false; - expect(RestrictionIntegerRange.safeParse({ max: 0 }).success).true; - expect(RestrictionIntegerRange.safeParse({ exclusiveMax: 0 }).success).true; - }); - it("Number Range Can't have exclusiveMin and Min", () => { - expect(RestrictionNumberRange.safeParse({ exclusiveMin: 0, min: 0 }).success).false; - expect(RestrictionNumberRange.safeParse({ min: 0 }).success).true; - expect(RestrictionNumberRange.safeParse({ exclusiveMin: 0 }).success).true; - }); - it("Number Range Can't have exclusiveMax and Max", () => { - expect(RestrictionNumberRange.safeParse({ exclusiveMax: 0, max: 0 }).success).false; - expect(RestrictionNumberRange.safeParse({ max: 0 }).success).true; - expect(RestrictionNumberRange.safeParse({ exclusiveMax: 0 }).success).true; - }); - }); - describe('RegexRestriction', () => { - it('Accepts valid regex', () => { - expect(StringFieldRestrictions.safeParse({ regex: '[a-zA-Z]' }).success).true; - expect( - StringFieldRestrictions.safeParse({ - regex: '^({((([WUBRG]|([0-9]|[1-9][0-9]*))(/[WUBRG])?)|(X)|([WUBRG](/[WUBRG])?/[P]))})+$', - }).success, - ).true; - }); - it('Rejects invalid regex', () => { - expect(StringFieldRestrictions.safeParse({ regex: '[' }).success).false; + describe('Fields', () => { + it('Can have no restrictions', () => { + const fieldString: SchemaField = { + name: 'some-name', + valueType: 'string', + }; + expect(SchemaField.safeParse(fieldString).success, 'String field invalid.').true; + const fieldNumber: SchemaField = { + name: 'some-name', + valueType: 'number', + }; + expect(SchemaField.safeParse(fieldNumber).success, 'Number field invalid.').true; + const fieldInteger: SchemaField = { + name: 'some-name', + valueType: 'integer', + }; + expect(SchemaField.safeParse(fieldInteger).success, 'Integer field invalid.').true; + const fieldBoolean: SchemaField = { + name: 'some-name', + valueType: 'boolean', + }; + expect(SchemaField.safeParse(fieldBoolean).success, 'Boolean field invalid.').true; }); - }); - describe('UniqueRestriction', () => { - it('All fields accept unique restriction', () => { - expect(StringFieldRestrictions.safeParse({ unique: true }).success).true; - expect(NumberFieldRestrictions.safeParse({ unique: true }).success).true; - expect(IntegerFieldRestrictions.safeParse({ unique: true }).success).true; - expect(BooleanFieldRestrictions.safeParse({ unique: true }).success).true; + it('Can have a single object restriction', () => { + const fieldString: SchemaField = { + name: 'some-name', + valueType: 'string', + restrictions: { + codeList: ['a', 'b', 'c'], + }, + }; + expect(SchemaField.safeParse(fieldString).success, 'String field invalid.').true; + const fieldInteger: SchemaField = { + name: 'some-name', + valueType: 'integer', + restrictions: { + required: true, + }, + }; + expect(SchemaField.safeParse(fieldInteger).success, 'Integer field invalid.').true; + const fieldNumber: SchemaField = { + name: 'some-name', + valueType: 'number', + restrictions: { + required: true, + }, + }; + expect(SchemaField.safeParse(fieldNumber).success, 'Number field invalid.').true; + const fieldBoolean: SchemaField = { + name: 'some-name', + valueType: 'boolean', + restrictions: { + required: true, + }, + }; + expect(SchemaField.safeParse(fieldBoolean).success, 'Boolean field invalid.').true; }); - }); - describe('ScriptRestriction', () => { - it('All fields accept script restriction', () => { - expect(StringFieldRestrictions.safeParse({ script: ['()=>true'] }).success).true; - expect(NumberFieldRestrictions.safeParse({ script: ['()=>true'] }).success).true; - expect(IntegerFieldRestrictions.safeParse({ script: ['()=>true'] }).success).true; - expect(BooleanFieldRestrictions.safeParse({ script: ['()=>true'] }).success).true; + it('Can have an array of object restrictions', () => { + const fieldString: SchemaField = { + name: 'some-name', + valueType: 'string', + restrictions: [ + { + regex: '^[\\w]+$', + }, + { + regex: 'hello', + }, + ], + }; + expect(SchemaField.safeParse(fieldString).success, 'String field invalid.').true; + const fieldInteger: SchemaField = { + name: 'some-name', + valueType: 'integer', + restrictions: [ + { + required: true, + }, + { + codeList: [1, 2, 3], + }, + ], + }; + expect(SchemaField.safeParse(fieldInteger).success, 'Integer field invalid.').true; + const fieldNumber: SchemaField = { + name: 'some-name', + valueType: 'number', + restrictions: [ + { + required: true, + }, + { + codeList: [1, 2, 3], + }, + ], + }; + expect(SchemaField.safeParse(fieldNumber).success, 'Number field invalid.').true; + const fieldBoolean: SchemaField = { + name: 'some-name', + valueType: 'boolean', + restrictions: [ + { + required: true, + }, + { + required: false, + }, + ], + }; + expect(SchemaField.safeParse(fieldBoolean).success, 'Boolean field invalid.').true; }); }); describe('Schema', () => { diff --git a/packages/dictionary/test/metaSchema/restrictionsSchemas.spec.ts b/packages/dictionary/test/metaSchema/restrictionsSchemas.spec.ts new file mode 100644 index 0000000..42bdcc2 --- /dev/null +++ b/packages/dictionary/test/metaSchema/restrictionsSchemas.spec.ts @@ -0,0 +1,190 @@ +/* + * Copyright (c) 2023 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import assert from 'assert'; +import { expect } from 'chai'; +import { + Integer, + RestrictionIntegerRange, + RestrictionNumberRange, + SchemaBooleanField, + SchemaField, + SchemaIntegerField, + SchemaNumberField, + SchemaStringField, + StringFieldRestrictions, + type ConditionalRestriction, + type StringFieldRestrictionsObject, +} from '../../src'; + +describe('Restriction Schemas', () => { + describe('Integer', () => { + it("Can't be float", () => { + expect(Integer.safeParse(1.3).success).false; + expect(Integer.safeParse(2.0000001).success).false; + // Note: float precision issues, if the float resolves to a whole number the value will be accepted. + }); + it("Can't be string, boolean, object, array", () => { + expect(Integer.safeParse('1').success).false; + expect(Integer.safeParse(true).success).false; + expect(Integer.safeParse([1]).success).false; + expect(Integer.safeParse({}).success).false; + expect(Integer.safeParse({ thing: 1 }).success).false; + }); + it('Can be integer', () => { + expect(Integer.safeParse(1).success).true; + expect(Integer.safeParse(0).success).true; + expect(Integer.safeParse(-1).success).true; + expect(Integer.safeParse(1123).success).true; + }); + }); + describe('RangeRestriction', () => { + it("Integer Range Can't have exclusiveMin and Min", () => { + expect(RestrictionIntegerRange.safeParse({ exclusiveMin: 0, min: 0 }).success).false; + expect(RestrictionIntegerRange.safeParse({ min: 0 }).success).true; + expect(RestrictionIntegerRange.safeParse({ exclusiveMin: 0 }).success).true; + }); + it("Integer Range Can't have exclusiveMax and Max", () => { + expect(RestrictionIntegerRange.safeParse({ exclusiveMax: 0, max: 0 }).success).false; + expect(RestrictionIntegerRange.safeParse({ max: 0 }).success).true; + expect(RestrictionIntegerRange.safeParse({ exclusiveMax: 0 }).success).true; + }); + it("Number Range Can't have exclusiveMin and Min", () => { + expect(RestrictionNumberRange.safeParse({ exclusiveMin: 0, min: 0 }).success).false; + expect(RestrictionNumberRange.safeParse({ min: 0 }).success).true; + expect(RestrictionNumberRange.safeParse({ exclusiveMin: 0 }).success).true; + }); + it("Number Range Can't have exclusiveMax and Max", () => { + expect(RestrictionNumberRange.safeParse({ exclusiveMax: 0, max: 0 }).success).false; + expect(RestrictionNumberRange.safeParse({ max: 0 }).success).true; + expect(RestrictionNumberRange.safeParse({ exclusiveMax: 0 }).success).true; + }); + }); + describe('RegexRestriction', () => { + it('Accepts valid regex', () => { + expect(StringFieldRestrictions.safeParse({ regex: '[a-zA-Z]' }).success).true; + expect( + StringFieldRestrictions.safeParse({ + regex: '^({((([WUBRG]|([0-9]|[1-9][0-9]*))(/[WUBRG])?)|(X)|([WUBRG](/[WUBRG])?/[P]))})+$', + }).success, + ).true; + }); + it('Rejects invalid regex', () => { + expect(StringFieldRestrictions.safeParse({ regex: '[' }).success).false; + }); + }); + describe('UniqueRestriction', () => { + it('All fields accept unique property', () => { + expect(SchemaBooleanField.safeParse({ name: 'name', valueType: 'boolean', unique: true }).success).true; + expect(SchemaIntegerField.safeParse({ name: 'name', valueType: 'integer', unique: true }).success).true; + expect(SchemaNumberField.safeParse({ name: 'name', valueType: 'number', unique: true }).success).true; + expect(SchemaStringField.safeParse({ name: 'name', valueType: 'string', unique: true }).success).true; + }); + }); + + describe('ConditionalRestrictions', () => { + // These parsing functions seem unnecessary but they are checking for a failure case that was found: + // The restrictions property is a union between a restrictions object and conditional restriction schema, + // and the restriction object has all optional fields, so it will match with a conditional restriction + // object successfully and strip out the if/then/else properties. To avoid this scenario, the RestrictionObject + // schemas make the restriction validation `strict()`. These parsing tests are ensuring that this behaviour + // is not changed. + + it('Parses single conditional restriction', () => { + const restrictions: ConditionalRestriction = { + if: { + conditions: [{ fields: ['another-field'], match: { value: 'asdf' } }], + }, + then: { required: true }, + else: { empty: true }, + }; + const field: SchemaStringField = { + name: 'example-string', + valueType: 'string', + restrictions, + }; + const parseResult = SchemaField.safeParse(field); + expect(parseResult.success).true; + assert(parseResult.success === true); + + expect(parseResult.data).deep.equal(field); + }); + + it('Parses conditional restrictions in an array', () => { + const restrictions: Array = [ + { codeList: ['value1', 'value2'] }, + { + if: { + conditions: [{ fields: ['another-field'], match: { value: 'asdf' } }], + }, + then: { required: true }, + else: { empty: true }, + }, + ]; + const field: SchemaStringField = { + name: 'example-string', + valueType: 'string', + restrictions, + }; + const parseResult = SchemaField.safeParse(field); + expect(parseResult.success).true; + assert(parseResult.success === true); + + expect(parseResult.data).deep.equal(field); + }); + + it('Parses nested conditional restrictions', () => { + const restrictions: Array = [ + { codeList: ['value1', 'value2'] }, + { + if: { + conditions: [{ fields: ['first-dependent-field'], match: { value: 'asdf' } }], + }, + then: [ + { + if: { + conditions: [{ fields: ['second-dependent-field'], match: { range: { max: 10, min: 0 } } }], + }, + then: { required: true }, + else: { empty: true }, + }, + { + if: { + conditions: [{ fields: ['third-dependent-field'], match: { range: { max: 10, min: 0 } } }], + }, + then: { regex: 'asdf' }, + else: { codeList: ['a', 's', 'd', 'f'] }, + }, + ], + else: { empty: true }, + }, + ]; + const field: SchemaStringField = { + name: 'example-string', + valueType: 'string', + restrictions, + }; + const parseResult = SchemaField.safeParse(field); + expect(parseResult.success).true; + assert(parseResult.success === true); + + expect(parseResult.data).deep.equal(field); + }); + }); +}); diff --git a/packages/dictionary/test/references.spec.ts b/packages/dictionary/test/references.spec.ts index 823d8ba..3f20af4 100644 --- a/packages/dictionary/test/references.spec.ts +++ b/packages/dictionary/test/references.spec.ts @@ -20,69 +20,88 @@ import { expect } from 'chai'; import { replaceReferences } from '../src/references'; -import noReferencesSectionInput from './fixtures/references/no_references_section/input'; -import noReferencesSectionOutput from './fixtures/references/no_references_section/output'; -import emptyReferencesInput from './fixtures/references/empty_references_section/input'; -import emptyReferencesOutput from './fixtures/references/empty_references_section/output'; -import simpleReferencesInput from './fixtures/references/simple_references/input'; -import simpleReferencesOutput from './fixtures/references/simple_references/output'; +import assert from 'assert'; import codeListReferencesInput from './fixtures/references/codeList_references/input'; import codeListReferencesOutput from './fixtures/references/codeList_references/output'; +import cyclicReferencesInput from './fixtures/references/cyclic_references/input'; +import emptyReferencesInput from './fixtures/references/empty_references_section/input'; +import emptyReferencesOutput from './fixtures/references/empty_references_section/output'; +import nestedMetaReferencesInput from './fixtures/references/nested_meta_references/input'; +import nestedMetaReferencesOutput from './fixtures/references/nested_meta_references/output'; +import noReferencesSectionInput from './fixtures/references/no_references_section/input'; +import noReferencesSectionOutput from './fixtures/references/no_references_section/output'; +import noReferencesTagsInput from './fixtures/references/no_referece_tags/input'; +import noReferencesTagsOutput from './fixtures/references/no_referece_tags/output'; +import nonExistingReferencesInput from './fixtures/references/non_existing_references/input'; import referencesWithinReferencesInput from './fixtures/references/references_within_references/input'; import referencesWithinReferencesOutput from './fixtures/references/references_within_references/output'; -import scriptReferencesInput from './fixtures/references/script_references/input'; -import scriptReferencesOutput from './fixtures/references/script_references/output'; import regexReferencesInput from './fixtures/references/regex_reference/input'; import regexReferencesOutput from './fixtures/references/regex_reference/output'; -import regexArrayReferencesInput from './fixtures/references/regex_reference/input_with_array'; -import nonExistingReferencesInput from './fixtures/references/non_existing_references/input'; -import cyclicReferencesInput from './fixtures/references/cyclic_references/input'; +import regexArrayReferencesInput from './fixtures/references/regex_reference_with_array/input'; +import regexArrayReferencesOutput from './fixtures/references/regex_reference_with_array/output'; +import restrictionsArrayWithReferencesInput from './fixtures/references/restrictions_array_with_references/input'; +import restrictionsArrayWithReferencesOutput from './fixtures/references/restrictions_array_with_references/output'; import selfReferencesInput from './fixtures/references/self_references/input'; +import simpleReferencesInput from './fixtures/references/simple_references/input'; +import simpleReferencesOutput from './fixtures/references/simple_references/output'; describe('Replace References', () => { - it('Should return the same original schema if dictionary does not contain a references section', () => { + it('Returns unmodified schema when dictionary does not contain a references section', () => { const replacedDictionary = replaceReferences(noReferencesSectionInput); expect(replacedDictionary).to.deep.eq(noReferencesSectionOutput); }); - it('Should return the same original schema if dictionary contains an empty references section', () => { + it('Returns unmodified schema when dictionary contains an empty references section', () => { const replacedDictionary = replaceReferences(emptyReferencesInput); expect(replacedDictionary).to.deep.eq(emptyReferencesOutput); }); + it('Returns unmodified schema when no ReferenceTag values are used', () => { + const replacedDictionary = replaceReferences(noReferencesTagsInput); + expect(replacedDictionary).to.deep.eq(noReferencesTagsOutput); + }); + it('Throws an error when a ReferenceTag to an unknown path is provided', () => { + expect(() => replaceReferences(nonExistingReferencesInput)).to.throw( + "No reference found for provided tag '#/NON_EXISTING_REFERENCE'", + ); + }); + it('Throws an error if cyclic references are found', () => { + expect(() => replaceReferences(cyclicReferencesInput)).to.throw("Cyclical references found for '#/OTHER'"); + }); + it('Throws an error if self references are found', () => { + expect(() => replaceReferences(selfReferencesInput)).to.throw("Cyclical references found for '#/SELF_REFERENCE'"); + }); + it('Replaces references when restrictions are in arrays', () => { + restrictionsArrayWithReferencesInput; + const replacedDictionary = replaceReferences(restrictionsArrayWithReferencesInput); + expect(replacedDictionary).to.deep.eq(restrictionsArrayWithReferencesOutput); + }); // TODO: Check reference replacement in meta it('Should return the schema with simple references replaced', () => { const replacedDictionary = replaceReferences(simpleReferencesInput); expect(replacedDictionary).to.deep.eq(simpleReferencesOutput); }); - it('Should return the schema where references inside codeLists are replaced', () => { - const replacedDictionary = replaceReferences(codeListReferencesInput); - expect(replacedDictionary).to.deep.eq(codeListReferencesOutput); - }); it('Should return the schema where references inside references are replaced', () => { const output = replaceReferences(referencesWithinReferencesInput); expect(output).to.deep.eq(referencesWithinReferencesOutput); }); - it('Should return the schema where references inside scripts arrays are replaced', () => { - const output = replaceReferences(scriptReferencesInput); - expect(output).to.deep.eq(scriptReferencesOutput); - }); - it('Regex Reference replaced successfully', () => { - const output = replaceReferences(regexReferencesInput); - expect(output).to.deep.eq(regexReferencesOutput); + it('Replaces reference tag value in meta nested properties', () => { + const replacedDictionary = replaceReferences(nestedMetaReferencesInput); + expect(replacedDictionary).to.deep.eq(nestedMetaReferencesOutput); }); - it('Regex Reference cannot be an array', () => { - expect(() => replaceReferences(regexArrayReferencesInput)).to.throw( - `Field 'some_id' has restriction 'regex' with a reference '#/regex/ID_FORMAT' that resolves to an array. This restriction must be a string.`, - ); - }); - it('Should throw exception if reference does not exist', () => { - expect(() => replaceReferences(nonExistingReferencesInput)).to.throw( - "No reference found for provided tag '#/NON_EXISTING_REFERENCE'", - ); - }); - it('Should throw exception if cyclic references are found', () => { - expect(() => replaceReferences(cyclicReferencesInput)).to.throw("Cyclical references found for '#/OTHER'"); - }); - it('Should throw exception if self references are found', () => { - expect(() => replaceReferences(selfReferencesInput)).to.throw("Cyclical references found for '#/SELF_REFERENCE'"); + describe('String Restrictions', () => { + it('CodeList with references are replaced', () => { + // has a couple test cases in the test fixture dictionary: + // - array containing ReferenceTag is replaced by array with reference values added to array + // - CodeList with ReferenceTag to single value is replaced with an array with the single value + const replacedDictionary = replaceReferences(codeListReferencesInput); + expect(replacedDictionary).to.deep.eq(codeListReferencesOutput); + }); + it('Regex with ReferenceTag is replaced by single value', () => { + const output = replaceReferences(regexReferencesInput); + expect(output).to.deep.eq(regexReferencesOutput); + }); + it('Regex with ReferenceTag to array value throws an error', () => { + const output = replaceReferences(regexArrayReferencesInput); + expect(output).to.deep.eq(regexArrayReferencesOutput); + }); }); }); diff --git a/packages/dictionary/test/dataTypes.spec.ts b/packages/dictionary/test/types/booleanRegex.spec.ts similarity index 98% rename from packages/dictionary/test/dataTypes.spec.ts rename to packages/dictionary/test/types/booleanRegex.spec.ts index 9584c38..5fb2c68 100644 --- a/packages/dictionary/test/dataTypes.spec.ts +++ b/packages/dictionary/test/types/booleanRegex.spec.ts @@ -18,7 +18,7 @@ */ import { expect } from 'chai'; -import { REGEXP_BOOLEAN_VALUE } from '../src'; +import { REGEXP_BOOLEAN_VALUE } from '../../src'; describe('Data Types', () => { describe('Boolean RegExp', () => { diff --git a/packages/dictionary/test/versionUtils.spec.ts b/packages/dictionary/test/utils/versionUtils.spec.ts similarity index 98% rename from packages/dictionary/test/versionUtils.spec.ts rename to packages/dictionary/test/utils/versionUtils.spec.ts index d63a37d..65a7081 100644 --- a/packages/dictionary/test/versionUtils.spec.ts +++ b/packages/dictionary/test/utils/versionUtils.spec.ts @@ -18,7 +18,7 @@ */ import { expect } from 'chai'; -import { VersionUtils } from '../'; +import { VersionUtils } from '../../dist'; const { isValidVersion, incrementMinor, incrementMajor, isGreater } = VersionUtils; diff --git a/packages/validation/src/parseValues/matchCodeListFormatting.ts b/packages/validation/src/parseValues/matchCodeListFormatting.ts index 7276746..9553686 100644 --- a/packages/validation/src/parseValues/matchCodeListFormatting.ts +++ b/packages/validation/src/parseValues/matchCodeListFormatting.ts @@ -17,8 +17,25 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import type { SchemaField } from '@overture-stack/lectern-dictionary'; +import { TypeUtils, type SchemaField, type StringFieldRestrictionsObject } from '@overture-stack/lectern-dictionary'; +/** + * Loop through restrictins and nested conditional restrictions finding every codeList restriction and collect all their + * values into a single array. This is used to format data values during parsing. + */ +const collectAllNestedCodeLists = ( + restrictions: StringFieldRestrictionsObject | StringFieldRestrictionsObject[], +): string[] => { + return TypeUtils.asArray(restrictions).flatMap((restrictionsObject) => { + if ('if' in restrictionsObject) { + const thenCodeLists = restrictionsObject.then ? collectAllNestedCodeLists(restrictionsObject.then) : []; + const elseCodeLists = restrictionsObject.else ? collectAllNestedCodeLists(restrictionsObject) : []; + return [...thenCodeLists, ...elseCodeLists]; + } else { + return restrictionsObject.codeList ? restrictionsObject.codeList : []; + } + }); +}; /** * Given a string value, look for any matching values in code list restrictions and return that * value. This is used by the convertValue functions to ensure the value returned matches the letter @@ -36,10 +53,10 @@ import type { SchemaField } from '@overture-stack/lectern-dictionary'; * @returns */ export function matchCodeListFormatting(value: string, fieldDefinition: SchemaField): string { - const { valueType, restrictions } = fieldDefinition; - + const { valueType } = fieldDefinition; if (valueType === 'string') { - const codeList = restrictions?.codeList; + const codeList = fieldDefinition.restrictions && collectAllNestedCodeLists(fieldDefinition.restrictions); + if (Array.isArray(codeList)) { // We have found a code list to compare to! diff --git a/packages/validation/src/parseValues/parseValues.ts b/packages/validation/src/parseValues/parseValues.ts index 0a3d241..66af9a4 100644 --- a/packages/validation/src/parseValues/parseValues.ts +++ b/packages/validation/src/parseValues/parseValues.ts @@ -22,6 +22,7 @@ import { failure, failWith, success, + TypeUtils, type ArrayDataValue, type DataRecord, type DataRecordValue, @@ -32,7 +33,6 @@ import { type SchemaFieldValueType, type UnprocessedDataRecord, } from '@overture-stack/lectern-dictionary'; -import { isInteger, isNumber } from '../utils/typeUtils'; import type { ParseDictionaryData, ParseDictionaryFailure, @@ -44,6 +44,8 @@ import type { } from './ParseValuesResult'; import { matchCodeListFormatting } from './matchCodeListFormatting'; +const { isInteger, isNumber } = TypeUtils; + /* === Type Specific conversion functions === */ // Note: These are intended to be passed only normalized values that have already passed through the diff --git a/packages/validation/src/utils/isValidValueType.ts b/packages/validation/src/utils/isValidValueType.ts index f7c8bc6..47ad49c 100644 --- a/packages/validation/src/utils/isValidValueType.ts +++ b/packages/validation/src/utils/isValidValueType.ts @@ -17,8 +17,8 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import type { DataRecordValue, SchemaField } from '@overture-stack/lectern-dictionary'; -import { isBooleanArray, isInteger, isIntegerArray, isNumber, isNumberArray, isStringArray } from './typeUtils'; +import { type DataRecordValue, type SchemaField, TypeUtils } from '@overture-stack/lectern-dictionary'; +const { isBooleanArray, isInteger, isIntegerArray, isNumber, isNumberArray, isStringArray } = TypeUtils; /** * Checks that a value matches the expected type for a given field, based on the value type specified in its field diff --git a/packages/validation/src/utils/resultForArrayTestCase.ts b/packages/validation/src/utils/resultForArrayTestCase.ts new file mode 100644 index 0000000..fd0d15e --- /dev/null +++ b/packages/validation/src/utils/resultForArrayTestCase.ts @@ -0,0 +1,29 @@ +import type { ArrayTestCase } from '@overture-stack/lectern-dictionary'; + +/** + * `ArrayTestCase` values dictate how many results in an array need to be successful in order for + * an entire test to be considered a success. This function takes an array of boolean values and + * an `ArrayTestCase` value and determine if the entire list is successful. + * + * The possible test case values and their behaviours are: + * - all: every boolean in the results must be true + * - any: at least one result must be true + * - none: no result can be true (all must be false) + * + * @param results Array of booleans representing a list of test results where true is a success + * @param testCase + * @returns + */ +export const resultForArrayTestCase = (results: boolean[], testCase: ArrayTestCase): boolean => { + switch (testCase) { + case 'all': { + return results.every((result) => result); + } + case 'any': { + return results.some((result) => result); + } + case 'none': { + return results.every((result) => !result); + } + } +}; diff --git a/packages/validation/src/utils/typeUtils.ts b/packages/validation/src/utils/typeUtils.ts deleted file mode 100644 index 0720e8d..0000000 --- a/packages/validation/src/utils/typeUtils.ts +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved - * - * This program and the accompanying materials are made available under the terms of - * the GNU Affero General Public License v3.0. You should have received a copy of the - * GNU Affero General Public License along with this program. - * If not, see . - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT - * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER - * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/** - * Given a predicate function that checks for type `T`, this will create a new predicate funcion that - * will check if a value is of type `T[]`. - * - * @example - * // Create type and predicate for `Person`: - * type Person = { name: string; age: number }; - * const isPerson = (value: unknown): value is Person => - * !!value && - * typeof value === 'object' && - * 'name' in value && - * typeof value.name === 'string' && - * 'age' in value && - * typeof value.age === 'number'; - * - * // Use `isArrayOf` and the new predicate to create `isPersonArray`: - * const isPersonArray = isArrayOf(isPerson); - * - * // Usage of `isPersonArray`: - * isPersonArray([{name:'Lisa', age: 8}, {name: 'Bart', age: 10}]); // true - * isPersonArray(['not a person']); // false - * isPersonArray('not an array'); // false - * isPersonArray([{name:'Lisa', age: 8}, {not: 'a person'}]); // false - * @param predicate - * @returns - */ -export const isArrayOf = - (predicate: (value: unknown) => value is T) => - (value: unknown) => - Array.isArray(value) && value.every(predicate); - -/** - * Determines if a variable is of type `boolean[]`. - * @param value - * @returns - */ -export const isBooleanArray = isArrayOf((value: unknown) => typeof value === 'boolean'); - -/** - * Determines if a variable is a number, with added restriction that it is Finite. - * This eliminates the values `NaN` and `Infinity`. - * - * Note: this is just a wrapper on `Number.isFinite` which is used by Lectern for identifying numbers in data value type checks. - * @param value - * @returns - */ -export const isNumber = (value: unknown): value is number => Number.isFinite(value); -/** - * Determines if variable is of type number[], with added restriction that every element is Finite. - * @param value - * @returns - */ -export const isNumberArray = isArrayOf(isNumber); - -/** - * Determines if a variable is of type string[] - * @param value - * @returns - */ -export const isStringArray = isArrayOf((value: unknown) => typeof value === 'string'); - -/** - * Determines if a variable is a number, with added restriction that it is an Integer. - * - * Note: This is a wrapper over `Number.isInteger` which is used by Lectern for identifying integers in data value type checks. - * @param value - * @returns - */ -export const isInteger = (value: unknown): value is number => Number.isInteger(value); - -/** - * Determines if a variables is of type number[], with add restriction that every element is an Integer. - * @param value - * @returns - */ -export const isIntegerArray = isArrayOf(isInteger); diff --git a/packages/validation/src/validateField/FieldRestrictionRule.ts b/packages/validation/src/validateField/FieldRestrictionRule.ts index 41117cb..8446325 100644 --- a/packages/validation/src/validateField/FieldRestrictionRule.ts +++ b/packages/validation/src/validateField/FieldRestrictionRule.ts @@ -22,13 +22,16 @@ import type { RestrictionCodeList, RestrictionRange, RestrictionRegex, - RestrictionScript, } from '@overture-stack/lectern-dictionary'; export type FieldRestrictionRuleCodeList = { type: typeof FieldRestrictionTypes.codeList; rule: RestrictionCodeList; }; +export type FieldRestrictionRuleEmpty = { + type: typeof FieldRestrictionTypes.empty; + rule: boolean; +}; export type FieldRestrictionRuleRange = { type: typeof FieldRestrictionTypes.range; @@ -45,18 +48,9 @@ export type FieldRestrictionRuleRegex = { rule: RestrictionRegex; }; -// export type FieldRestrictionRuleScript = { -// type: typeof FieldRestrictionTypes.script; -// rule: RestrictionScript; -// }; - -// export type FieldRestrictionRuleUnique = { -// type: typeof FieldRestrictionTypes.unique; -// rule: boolean; -// }; - export type FieldRestrictionRule = | FieldRestrictionRuleCodeList + | FieldRestrictionRuleEmpty | FieldRestrictionRuleRange | FieldRestrictionRuleRequired | FieldRestrictionRuleRegex; diff --git a/packages/validation/src/validateField/conditions/index.ts b/packages/validation/src/validateField/conditions/index.ts new file mode 100644 index 0000000..c49a91d --- /dev/null +++ b/packages/validation/src/validateField/conditions/index.ts @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +export * from './testMatchCodeList'; +export * from './testMatchCount'; +export * from './testMatchExists'; +export * from './testMatchRange'; +export * from './testMatchRegex'; +export * from './testMatchValue'; diff --git a/packages/validation/src/validateField/conditions/testConditionalRestriction.ts b/packages/validation/src/validateField/conditions/testConditionalRestriction.ts new file mode 100644 index 0000000..53c1565 --- /dev/null +++ b/packages/validation/src/validateField/conditions/testConditionalRestriction.ts @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { + ARRAY_TEST_CASE_DEFAULT, + ArrayTestCase, + type ArrayDataValue, + type ConditionalRestrictionTest, + type DataRecord, + type DataRecordValue, + type RestrictionCondition, + type SingleDataValue, +} from '@overture-stack/lectern-dictionary'; +import { resultForArrayTestCase } from '../../utils/resultForArrayTestCase'; +import { testMatchCount } from './testMatchCount'; +import { testMatchCodeList } from './testMatchCodeList'; +import { testMatchExists } from './testMatchExists'; +import { testMatchRange } from './testMatchRange'; +import { testMatchRegex } from './testMatchRegex'; +import { testMatchValue } from './testMatchValue'; + +/** + * Test values extracted from other fields vs a match test. This function should be passed the + * + * @param values + * @param rule + * @param matchTest + * @param arrayCase Intentionally using `| undefined` vs `/?: arrayCase` to ensure that this argument is provided when this function is called + * @returns + */ +const fieldsPassMatchTest = ( + values: DataRecordValue[], + matchTest: (value: DataRecordValue) => boolean, + arrayCase: ArrayTestCase | undefined, +): boolean => { + const fieldTestResults = values.map((value) => matchTest(value)); + return resultForArrayTestCase(fieldTestResults, arrayCase || ARRAY_TEST_CASE_DEFAULT); +}; + +const testCondition = (condition: RestrictionCondition, _value: DataRecordValue, record: DataRecord): boolean => { + const fieldValues = condition.fields.map((fieldName) => record[fieldName]); + const matchCodeList = condition.match.codeList; + if (matchCodeList !== undefined) { + if (!fieldsPassMatchTest(fieldValues, (value) => testMatchCodeList(matchCodeList, value), condition.case)) { + return false; + } + } + const matchCount = condition.match.count; + if (matchCount !== undefined) { + if (!fieldsPassMatchTest(fieldValues, (value) => testMatchCount(matchCount, value), condition.case)) { + return false; + } + } + + const matchExists = condition.match.exists; + if (matchExists !== undefined) { + if (!fieldsPassMatchTest(fieldValues, (value) => testMatchExists(matchExists, value), condition.case)) { + return false; + } + } + + const matchRange = condition.match.range; + if (matchRange !== undefined) { + if (!fieldsPassMatchTest(fieldValues, (value) => testMatchRange(matchRange, value), condition.case)) { + return false; + } + } + + const matchRegex = condition.match.regex; + if (matchRegex !== undefined) { + if (!fieldsPassMatchTest(fieldValues, (value) => testMatchRegex(matchRegex, value), condition.case)) { + return false; + } + } + + const matchValue = condition.match.value; + if (matchValue !== undefined) { + if (!fieldsPassMatchTest(fieldValues, (value) => testMatchValue(matchValue, value), condition.case)) { + return false; + } + } + return true; +}; + +/** + * Check all conditions inside the `if` object of a Conditiohnal Restriction to determine if the condition is met for + * a given field and its data record. This will apply all match rules inside each condition versus a field value and data record. + */ +export const testConditionalRestriction = ( + conditionalTest: ConditionalRestrictionTest, + value: DataRecordValue, + record: DataRecord, +): boolean => { + const results = conditionalTest.conditions.map((condition) => testCondition(condition, value, record)); + return resultForArrayTestCase(results, conditionalTest.case || ARRAY_TEST_CASE_DEFAULT); +}; diff --git a/packages/validation/src/validateField/conditions/testMatchCodeList.ts b/packages/validation/src/validateField/conditions/testMatchCodeList.ts new file mode 100644 index 0000000..b75e3fa --- /dev/null +++ b/packages/validation/src/validateField/conditions/testMatchCodeList.ts @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import type { DataRecordValue, MatchRuleCodeList } from '@overture-stack/lectern-dictionary'; +import { isNumberArray, isStringArray } from '@overture-stack/lectern-dictionary/dist/utils/typeUtils'; + +/** + * Check if the value (or at least one value from an array) is found in the code list. + */ +export const testMatchCodeList = (codeList: MatchRuleCodeList, value: DataRecordValue): boolean => { + if (isStringArray(codeList)) { + if (isStringArray(value)) { + // If we can find at least one match from our codeList inside the array of values then we return true + return value.some((singleValue) => codeList.includes(singleValue)); + } + if (typeof value === 'string') { + return codeList.includes(value); + } + } + if (isNumberArray(codeList)) { + if (isNumberArray(value)) { + return value.some((singleValue) => codeList.includes(singleValue)); + } + if (typeof value === 'number') { + return codeList.includes(value); + } + } + + // If the code reaches here, we have a mismatch between the type of the codeList and the value, for example + // the code list is an array of strings but the value is a number. Since these mismatched types will never match + // we return false. + return false; +}; diff --git a/packages/validation/src/validateField/conditions/testMatchCount.ts b/packages/validation/src/validateField/conditions/testMatchCount.ts new file mode 100644 index 0000000..e6fe8e9 --- /dev/null +++ b/packages/validation/src/validateField/conditions/testMatchCount.ts @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import type { DataRecordValue, MatchRuleCount } from '@overture-stack/lectern-dictionary'; +import { testRange } from '../restrictions'; + +/** + * Test if the number of elements in an array value is an exact number, or within a range. + * + * Note: This test is only meant to match on array fields. It is counting the number of elements in an array. + * This will always return false for non-array values, it does not count the character length of strings + * or numbers. + * @param count + * @param value + * @returns + */ +export const testMatchCount = (count: MatchRuleCount, value: DataRecordValue): boolean => { + if (!Array.isArray(value)) { + // can only match with arrays + return false; + } + + // count match rule is either a range object or a number + if (typeof count === 'object') { + // here it is the range object so we can use the testRange fuctionality to determine if we have + // the correct number of elements + return testRange(count, value.length).valid; + } + + // whats left + return value.length === count; +}; diff --git a/packages/validation/src/validateField/conditions/testMatchExists.ts b/packages/validation/src/validateField/conditions/testMatchExists.ts new file mode 100644 index 0000000..8b55d27 --- /dev/null +++ b/packages/validation/src/validateField/conditions/testMatchExists.ts @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { + TypeUtils, + type DataRecordValue, + type MatchRuleExists, + type SingleDataValue, +} from '@overture-stack/lectern-dictionary'; + +const valueExists = (value: SingleDataValue) => { + if (value === undefined) { + return false; + } + switch (typeof value) { + case 'string': { + // empty string, and all whitespace, are treated as empty values + return value.trim() !== ''; + } + case 'number': { + // Treate NaN and Infinity values as missing values + return Number.isFinite(value); + } + case 'boolean': { + return true; + } + } +}; + +/** + * Test if the value exists, ie. that it is not undefined or an empty array. When the rule is true, this will + * return true when the value exists, and when the rule is false this will return true only when the value does + * not exist. + * + * Notes: + * - Boolean value `false` is an existing value + * - Empty strings represent a missing value, so empty string value is not teated as an existing value + */ +export const testMatchExists = (exists: MatchRuleExists, value: DataRecordValue): boolean => { + const isEmptyArray = Array.isArray(value) && value.length === 0; + const isValueExists = !isEmptyArray && TypeUtils.asArray(value).every(valueExists); + + return exists === isValueExists; +}; diff --git a/apps/server/test/functional/normalize.spec.ts b/packages/validation/src/validateField/conditions/testMatchRange.ts similarity index 53% rename from apps/server/test/functional/normalize.spec.ts rename to packages/validation/src/validateField/conditions/testMatchRange.ts index b435833..ec5e86e 100644 --- a/apps/server/test/functional/normalize.spec.ts +++ b/packages/validation/src/validateField/conditions/testMatchRange.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 The Ontario Institute for Cancer Research. All rights reserved + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved * * This program and the accompanying materials are made available under the terms of * the GNU Affero General Public License v3.0. You should have received a copy of the @@ -17,19 +17,18 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { expect } from 'chai'; -import { normalizeSchema } from '../../src/services/schemaService'; -import DICTIONARY_RN_LINEBREAKS from '../fixtures/dictionaries/linebreak/rnLinebreaks'; -import DICTIONARY_N_LINEBREAKS from '../fixtures/dictionaries/linebreak/nLinebreaks'; -import DICTIONARY_NORMALIZED_LINEBREAKS from '../fixtures/dictionaries/linebreak/normalizedLinebreaks'; +import { type DataRecordValue, type MatchRuleRange } from '@overture-stack/lectern-dictionary'; +import { isNumberArray } from '@overture-stack/lectern-dictionary/dist/utils/typeUtils'; +import { testRange } from '../restrictions'; -describe('New Line symbol normalization', () => { - it('Should convert \\r\\n to \\n in scripts', () => { - const normalizedSchema = normalizeSchema(DICTIONARY_N_LINEBREAKS.schemas[0]); - expect(normalizedSchema).to.deep.eq(DICTIONARY_NORMALIZED_LINEBREAKS.schemas[0]); - }); - it('Should not alter already formatted files', () => { - const normalizedSchema = normalizeSchema(DICTIONARY_RN_LINEBREAKS.schemas[0]); - expect(normalizedSchema).to.deep.eq(DICTIONARY_NORMALIZED_LINEBREAKS.schemas[0]); - }); -}); +export const testMatchRange = (range: MatchRuleRange, value: DataRecordValue): boolean => { + if (typeof value === 'number') { + return testRange(range, value).valid; + } + if (isNumberArray(value)) { + return value.some((item) => testRange(range, item).valid); + } + + // value is not a type that can match the range rule, we return false; + return false; +}; diff --git a/packages/validation/src/validateField/conditions/testMatchRegex.ts b/packages/validation/src/validateField/conditions/testMatchRegex.ts new file mode 100644 index 0000000..2ed76fe --- /dev/null +++ b/packages/validation/src/validateField/conditions/testMatchRegex.ts @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { type DataRecordValue, type MatchRuleRegex } from '@overture-stack/lectern-dictionary'; +import { isStringArray } from '@overture-stack/lectern-dictionary/dist/utils/typeUtils'; +import { testRegex } from '../restrictions'; + +export const testMatchRegex = (regex: MatchRuleRegex, value: DataRecordValue): boolean => { + if (typeof value === 'string') { + return testRegex(regex, value).valid; + } + + if (isStringArray(value)) { + return value.some((item) => testRegex(regex, item).valid); + } + + // value is not a type that can match the regex rule, we return false; + return false; +}; diff --git a/packages/validation/src/validateField/conditions/testMatchValue.ts b/packages/validation/src/validateField/conditions/testMatchValue.ts new file mode 100644 index 0000000..5b9b4fc --- /dev/null +++ b/packages/validation/src/validateField/conditions/testMatchValue.ts @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { type DataRecordValue, type MatchRuleValue, type SingleDataValue } from '@overture-stack/lectern-dictionary'; +import { isStringArray } from '@overture-stack/lectern-dictionary/dist/utils/typeUtils'; +import { testRegex } from '../restrictions'; + +const normalizeValue = (value: SingleDataValue): SingleDataValue => { + if (typeof value === 'string') { + return value.trim().toLowerCase(); + } + return value; +}; + +/** + * Tests if the value has the same value as the value match rule. No type coercion is performed, strings + * values only match strings, and arrays only match arrays. + * + * Strings are matched case insensitive and after trimming and forcing each string to lowercase. + * @param valueRule + * @param value + * @returns + */ +export const testMatchValue = (valueRule: MatchRuleValue, value: DataRecordValue): boolean => { + if (Array.isArray(valueRule)) { + if (!Array.isArray(value)) { + return false; + } + + if (value.length !== valueRule.length) { + return false; + } + + const sortedValue = [...value].map(normalizeValue).sort(); + const sortedRule = [...valueRule].map(normalizeValue).sort(); + + const allMatch = sortedRule.every((item, index) => item === sortedValue[index]); + + return allMatch; + } + + if (Array.isArray(value)) { + return false; + } + + return normalizeValue(value) === normalizeValue(valueRule); +}; diff --git a/packages/validation/src/validateField/resolveFieldRestrictions.ts b/packages/validation/src/validateField/resolveFieldRestrictions.ts deleted file mode 100644 index 1f6952e..0000000 --- a/packages/validation/src/validateField/resolveFieldRestrictions.ts +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved - * - * This program and the accompanying materials are made available under the terms of - * the GNU Affero General Public License v3.0. You should have received a copy of the - * GNU Affero General Public License along with this program. - * If not, see . - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT - * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER - * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -import type { DataRecord, DataRecordValue, SchemaField } from '@overture-stack/lectern-dictionary'; -import type { FieldRestrictionRule } from './FieldRestrictionRule'; - -/** - * Convert the restrictions found in a SchemaField definition into a list of rules that apply for this specific value - * and DataRecord. - */ -export const resolveFieldRestrictions = ( - _value: DataRecordValue, - _record: DataRecord, - field: SchemaField, -): FieldRestrictionRule[] => { - // TODO: This function requires value and record parameters so that conditional restrictions can be resolved. - // The original implementation with a static set of available restrictions does not need these parameters. - if (!field.restrictions) { - return []; - } - - switch (field.valueType) { - case 'boolean': { - const output: FieldRestrictionRule[] = []; - if (field.restrictions.required) { - output.push({ type: 'required', rule: field.restrictions.required }); - } - // if (field.restrictions.script) { - // output.push({ type: 'script', rule: asArray(field.restrictions.script) }); - // } - return output; - } - case 'integer': { - const output: FieldRestrictionRule[] = []; - if (Array.isArray(field.restrictions.codeList)) { - output.push({ type: 'codeList', rule: field.restrictions.codeList }); - } - if (field.restrictions.range) { - output.push({ type: 'range', rule: field.restrictions.range }); - } - if (field.restrictions.required) { - output.push({ type: 'required', rule: field.restrictions.required }); - } - // if (field.restrictions.script) { - // output.push({ type: 'script', rule: asArray(field.restrictions.script) }); - // } - return output; - } - case 'number': { - const output: FieldRestrictionRule[] = []; - if (Array.isArray(field.restrictions.codeList)) { - output.push({ type: 'codeList', rule: field.restrictions.codeList }); - } - if (field.restrictions.range) { - output.push({ type: 'range', rule: field.restrictions.range }); - } - if (field.restrictions.required) { - output.push({ type: 'required', rule: field.restrictions.required }); - } - // if (field.restrictions.script) { - // output.push({ type: 'script', rule: asArray(field.restrictions.script) }); - // } - return output; - } - case 'string': { - const output: FieldRestrictionRule[] = []; - if (Array.isArray(field.restrictions.codeList)) { - output.push({ type: 'codeList', rule: field.restrictions.codeList }); - } - if (field.restrictions.regex) { - output.push({ type: 'regex', rule: field.restrictions.regex }); - } - if (field.restrictions.required) { - output.push({ type: 'required', rule: field.restrictions.required }); - } - // if (field.restrictions.script) { - // output.push({ type: 'script', rule: asArray(field.restrictions.script) }); - // } - return output; - } - } -}; diff --git a/packages/validation/src/validateField/restrictions/index.ts b/packages/validation/src/validateField/restrictions/index.ts index 682deae..2f7b542 100644 --- a/packages/validation/src/validateField/restrictions/index.ts +++ b/packages/validation/src/validateField/restrictions/index.ts @@ -17,7 +17,9 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +export * from './resolveFieldRestrictions'; export * from './testCodeList'; +export * from './testEmpty'; export * from './testRange'; export * from './testRegex'; export * from './testRequired'; diff --git a/packages/validation/src/validateField/restrictions/resolveFieldRestrictions.ts b/packages/validation/src/validateField/restrictions/resolveFieldRestrictions.ts new file mode 100644 index 0000000..b655b77 --- /dev/null +++ b/packages/validation/src/validateField/restrictions/resolveFieldRestrictions.ts @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { + TypeUtils, + type AnyFieldRestrictions, + type DataRecord, + type DataRecordValue, + type SchemaField, + type SchemaRestrictions, +} from '@overture-stack/lectern-dictionary'; +import type { FieldRestrictionRule } from '../FieldRestrictionRule'; +import { testConditionalRestriction } from '../conditions/testConditionalRestriction'; + +const extractRulesFromRestriction = (restrictions: AnyFieldRestrictions): FieldRestrictionRule[] => { + const rules: FieldRestrictionRule[] = []; + + if ('codeList' in restrictions) { + if (Array.isArray(restrictions.codeList)) { + rules.push({ type: 'codeList', rule: restrictions.codeList }); + } + } + if ('empty' in restrictions) { + if (restrictions.empty) { + rules.push({ type: 'empty', rule: restrictions.empty }); + } + } + if ('range' in restrictions) { + if (restrictions.range) { + rules.push({ type: 'range', rule: restrictions.range }); + } + } + if ('regex' in restrictions) { + if (restrictions.regex) { + rules.push({ type: 'regex', rule: restrictions.regex }); + } + } + if ('required' in restrictions) { + if (restrictions.required) { + rules.push({ type: 'required', rule: restrictions.required }); + } + } + + return rules; +}; + +const recursiveResolveRestrictions = ( + restrictions: SchemaRestrictions, + value: DataRecordValue, + record: DataRecord, +): FieldRestrictionRule[] => { + if (!restrictions) { + return []; + } + + const output = TypeUtils.asArray(restrictions).flatMap((restrictionObject) => { + if ('if' in restrictionObject) { + // This object is a conditional restriction, we will test the record vs the conditions then extract rules + // from either the `then` or `else` block. + const result = testConditionalRestriction(restrictionObject.if, value, record); + if (result) { + return recursiveResolveRestrictions(restrictionObject.then, value, record); + } else { + return recursiveResolveRestrictions(restrictionObject.else, value, record); + } + } else { + // The restriction object here is not conditional, so we can grab the rules and add them to the output + return extractRulesFromRestriction(restrictionObject); + } + }); + + return output; +}; + +/** + * Convert the restrictions found in a SchemaField definition into a list of rules that apply for this specific value + * and DataRecord. This will check all conditional restrictions versus the field value and its data record, exracting + * the restriction rules that apply based on the conditional logic. + */ +export const resolveFieldRestrictions = ( + value: DataRecordValue, + record: DataRecord, + field: SchemaField, +): FieldRestrictionRule[] => recursiveResolveRestrictions(field.restrictions, value, record); diff --git a/packages/validation/src/validateField/restrictions/testEmpty.ts b/packages/validation/src/validateField/restrictions/testEmpty.ts new file mode 100644 index 0000000..758d782 --- /dev/null +++ b/packages/validation/src/validateField/restrictions/testEmpty.ts @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { type ArrayDataValue } from '@overture-stack/lectern-dictionary'; +import { invalid, valid, type TestResult } from '../../types/testResult'; +import type { + FieldRestrictionSingleValueTestFunction, + FieldRestrictionTestFunction, + RestrictionTestInvalidInfo, +} from '../FieldRestrictionTest'; +import { createFieldRestrictionTestForArrays } from './createFieldRestrictionTestForArrays'; + +const testEmptySingleValue: FieldRestrictionSingleValueTestFunction = (rule, value) => { + if (rule === false) { + return valid(); + } + if (value === undefined || value === '') { + return valid(); + } + + return invalid({ message: `This field must be empty but was provided a value.` }); +}; + +/** + * This function is the common pattern for applying a fieldRestriction value test to an array. + * For the empty restriction, we wanted to perform a couple additional checks to modify how + * this works, so this is used inside testEmptyArray after those additional checks are complete. + */ +const internalTestEmptyArray = createFieldRestrictionTestForArrays( + testEmptySingleValue, + `This field must be empty but was provided a value.`, +); + +/** + * Test for required value on an array field. Before using the common pattern of applying the value test to + * each item in the array, we first check: + * - if the rule is `false` then the value is always valid + * - if the length of the array is 0 then the value is invalid + * @param rule + * @param values + * @returns + */ +const testEmptyArray = (rule: boolean, values: ArrayDataValue): TestResult => { + if (rule === false) { + return valid(); + } + if (values.length === 0) { + return valid(); + } + return internalTestEmptyArray(rule, values); +}; + +/** + * Validate if a value is valid based on the required restriction value. + * + * When a field has the restriction `required: true`, it cannot be `undefined` or be an empty string. If + * the field is an array it cannot be an empty array (length 0). + * + * When a field has the restriction `required: false` then this test will always return as `valid: true` + * @param rule + * @param value + * @returns + */ +export const testEmpty: FieldRestrictionTestFunction = (rule, value) => + Array.isArray(value) ? testEmptyArray(rule, value) : testEmptySingleValue(rule, value); diff --git a/packages/validation/src/validateField/restrictions/testRegex.ts b/packages/validation/src/validateField/restrictions/testRegex.ts index 4681b22..ed59d4a 100644 --- a/packages/validation/src/validateField/restrictions/testRegex.ts +++ b/packages/validation/src/validateField/restrictions/testRegex.ts @@ -17,7 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { type RestrictionRegex } from '@overture-stack/lectern-dictionary'; +import { TypeUtils, type RestrictionRegex } from '@overture-stack/lectern-dictionary'; import { invalid, valid } from '../../types/testResult'; import type { FieldRestrictionSingleValueTestFunction, FieldRestrictionTestFunction } from '../FieldRestrictionTest'; import { createFieldRestrictionTestForArrays } from './createFieldRestrictionTestForArrays'; @@ -34,12 +34,16 @@ const testRegexSingleValue: FieldRestrictionSingleValueTestFunction { + const regexPattern = new RegExp(regexRule); - if (regexPattern.test(value)) { - return valid(); - } - return invalid({ message: `The value must match the regular expression.` }); + if (!regexPattern.test(value)) { + return false; + } + return true; + }); + return regexResult ? valid() : invalid({ message: `The value must match the regular expression.` }); + // TODO: update message to communicate which regex failed (if array) }; const testRegexArray = createFieldRestrictionTestForArrays( @@ -47,5 +51,8 @@ const testRegexArray = createFieldRestrictionTestForArrays( (_rule) => `All values in the array must match the regular expression.`, ); +// TODO: The error messages returned here don't inform which regular expressions failed, if there is a list. +// ...The message doesnt even acknowledge that there could be a list + export const testRegex: FieldRestrictionTestFunction = (rule, value) => Array.isArray(value) ? testRegexArray(rule, value) : testRegexSingleValue(rule, value); diff --git a/packages/validation/src/validateField/validateField.ts b/packages/validation/src/validateField/validateField.ts index 11823f4..a2b318a 100644 --- a/packages/validation/src/validateField/validateField.ts +++ b/packages/validation/src/validateField/validateField.ts @@ -21,18 +21,26 @@ import type { DataRecord, DataRecordValue, SchemaField } from '@overture-stack/l import { invalid, valid, type TestResult } from '../types'; import { isValidValueType } from '../utils/isValidValueType'; import type { FieldRestrictionRule } from './FieldRestrictionRule'; -import { resolveFieldRestrictions } from './resolveFieldRestrictions'; +import { resolveFieldRestrictions } from './restrictions/resolveFieldRestrictions'; import { testCodeList } from './restrictions/testCodeList'; import { testRange } from './restrictions/testRange'; import { testRegex } from './restrictions/testRegex'; import { testRequired } from './restrictions/testRequired'; import type { FieldValidationError, FieldValidationErrorRestrictionInfo } from './FieldValidationError'; +import type { RestrictionTestInvalidInfo } from './FieldRestrictionTest'; +import { testEmpty } from './restrictions/testEmpty'; -const testRestriction = (value: DataRecordValue, restriction: FieldRestrictionRule) => { +const testRestriction = ( + value: DataRecordValue, + restriction: FieldRestrictionRule, +): TestResult => { switch (restriction.type) { case 'codeList': { return testCodeList(restriction.rule, value); } + case 'empty': { + return testEmpty(restriction.rule, value); + } case 'range': { return testRange(restriction.rule, value); } @@ -42,12 +50,6 @@ const testRestriction = (value: DataRecordValue, restriction: FieldRestrictionRu case 'required': { return testRequired(restriction.rule, value); } - // case 'unique': { - // return testRequired(restriction.rule, value); - // } - // case 'script': { - // return valid(); - // } } }; diff --git a/packages/validation/src/validateSchema/validateSchema.ts b/packages/validation/src/validateSchema/validateSchema.ts index 96a0831..f42ee48 100644 --- a/packages/validation/src/validateSchema/validateSchema.ts +++ b/packages/validation/src/validateSchema/validateSchema.ts @@ -49,7 +49,7 @@ export const validateSchema = (records: Array, schema: Schema): Test const uniqueFieldMaps = new Map>(); schema.fields.forEach((field) => { - if (field.restrictions?.unique) { + if (field.unique) { uniqueFieldMaps.set(field.name, generateDataSetHashMap(records, [field.name])); } }); diff --git a/packages/validation/test/fixtures/fields/conditionalRestrictions/fieldStringConditionalExists.ts b/packages/validation/test/fixtures/fields/conditionalRestrictions/fieldStringConditionalExists.ts new file mode 100644 index 0000000..42cc5e9 --- /dev/null +++ b/packages/validation/test/fixtures/fields/conditionalRestrictions/fieldStringConditionalExists.ts @@ -0,0 +1,29 @@ +import { SchemaField, type SchemaStringField } from '@overture-stack/lectern-dictionary'; +import { fieldStringNoRestriction } from '../noRestrictions/fieldStringNoRestriction'; +import { validateFixture } from '../../../testUtils/validateFixture'; + +export const fieldStringConditionalExists = { + name: 'conditional-field', + valueType: 'string', + description: 'Required if `fieldStringNoRestriction` field exists, otherwise must be empty', + restrictions: { + if: { + conditions: [ + { + fields: [fieldStringNoRestriction.name], + match: { + exists: true, + }, + }, + ], + }, + then: { + required: true, + }, + else: { + empty: true, + }, + }, +} as const satisfies SchemaStringField; + +validateFixture(fieldStringConditionalExists, SchemaField, 'fieldStringConditionalExists is not a valid SchemaField'); diff --git a/packages/validation/test/fixtures/fields/conditionalRestrictions/fieldStringConditionalMultipleConditions.ts b/packages/validation/test/fixtures/fields/conditionalRestrictions/fieldStringConditionalMultipleConditions.ts new file mode 100644 index 0000000..0bc2d17 --- /dev/null +++ b/packages/validation/test/fixtures/fields/conditionalRestrictions/fieldStringConditionalMultipleConditions.ts @@ -0,0 +1,48 @@ +import { SchemaField, type SchemaStringField } from '@overture-stack/lectern-dictionary'; +import { fieldStringNoRestriction } from '../noRestrictions/fieldStringNoRestriction'; +import { validateFixture } from '../../../testUtils/validateFixture'; +import { fieldNumberNoRestriction } from '../noRestrictions/fieldNumberNoRestriction'; +import { fieldBooleanNoRestriction } from '../noRestrictions/fieldBooleanNoRestriction'; + +export const fieldStringConditionalMultipleConditions: SchemaField = { + name: 'conditional-field', + valueType: 'string', + description: + 'Conditionally required or empty, based on the existence of `any-string`, `any-number`, and `any-boolean` fields.', + restrictions: { + if: { + conditions: [ + { + fields: [fieldStringNoRestriction.name], + match: { + exists: true, + }, + }, + { + fields: [fieldNumberNoRestriction.name], + match: { + exists: true, + }, + }, + { + fields: [fieldBooleanNoRestriction.name], + match: { + exists: true, + }, + }, + ], + }, + then: { + required: true, + }, + else: { + empty: true, + }, + }, +}; + +validateFixture( + fieldStringConditionalMultipleConditions, + SchemaField, + 'fieldStringConditionalMultipleConditions is not a valid SchemaField', +); diff --git a/packages/validation/test/fixtures/fields/conditionalRestrictions/fieldStringConditionalMultipleFieldsRegex.ts b/packages/validation/test/fixtures/fields/conditionalRestrictions/fieldStringConditionalMultipleFieldsRegex.ts new file mode 100644 index 0000000..11376be --- /dev/null +++ b/packages/validation/test/fixtures/fields/conditionalRestrictions/fieldStringConditionalMultipleFieldsRegex.ts @@ -0,0 +1,34 @@ +import { SchemaField, type SchemaStringField } from '@overture-stack/lectern-dictionary'; +import { validateFixture } from '../../../testUtils/validateFixture'; +import { regexAlphaOnly } from '../../restrictions/regexFixtures'; + +export const fieldStringConditionalMultipleFieldsRegex: SchemaField = { + name: 'conditional-field', + valueType: 'string', + description: + 'Checks fields named `first`, `second`, and `third` are all alpha only, required if so, empty otherwise.', + restrictions: { + if: { + conditions: [ + { + fields: ['first', 'second', 'third'], + match: { + regex: regexAlphaOnly, + }, + }, + ], + }, + then: { + required: true, + }, + else: { + empty: true, + }, + }, +} satisfies SchemaStringField; + +validateFixture( + fieldStringConditionalMultipleFieldsRegex, + SchemaField, + 'fieldStringConditionalMultipleFieldsRegex is not a valid SchemaField', +); diff --git a/packages/validation/test/fixtures/fields/conditionalRestrictions/fieldStringNestedConditional.ts b/packages/validation/test/fixtures/fields/conditionalRestrictions/fieldStringNestedConditional.ts new file mode 100644 index 0000000..fb00473 --- /dev/null +++ b/packages/validation/test/fixtures/fields/conditionalRestrictions/fieldStringNestedConditional.ts @@ -0,0 +1,64 @@ +import { SchemaField, type SchemaStringField } from '@overture-stack/lectern-dictionary'; +import { fieldStringNoRestriction } from '../noRestrictions/fieldStringNoRestriction'; +import { validateFixture } from '../../../testUtils/validateFixture'; +import { regexAlphaOnly, regexRepeatedText } from '../../restrictions/regexFixtures'; +import { fieldNumberNoRestriction } from '../noRestrictions/fieldNumberNoRestriction'; +import { fieldBooleanNoRestriction } from '../noRestrictions/fieldBooleanNoRestriction'; + +export const fieldStringNestedConditional = { + name: 'conditional-field', + valueType: 'string', + description: + 'Nested conditional restrictions. If `any-string` has repeated text, then two different conditions are tested, otherwise this should be empty. Nested condition 1: `any-number` has a value of 0 or greater, then we have the regex restriction alpha-only. Nested condition 2: `any-boolean` has the value `true`, then we have the `required` restriction.', + restrictions: [ + { + if: { + conditions: [ + { + fields: [fieldStringNoRestriction.name], + match: { + regex: regexRepeatedText, + }, + }, + ], + }, + then: [ + { + if: { + conditions: [ + { + fields: [fieldNumberNoRestriction.name], + match: { + range: { min: 0 }, + }, + }, + ], + }, + then: { + regex: regexAlphaOnly, + }, + }, + { + if: { + conditions: [ + { + fields: [fieldBooleanNoRestriction.name], + match: { + value: true, + }, + }, + ], + }, + then: { + required: true, + }, + }, + ], + else: { + empty: true, + }, + }, + ], +} as const satisfies SchemaStringField; + +validateFixture(fieldStringNestedConditional, SchemaField, 'fieldStringNestedConditional is not a valid SchemaField'); diff --git a/packages/validation/test/fixtures/fields/conditionalRestrictions/fieldStringRequiredConditionalRange.ts b/packages/validation/test/fixtures/fields/conditionalRestrictions/fieldStringRequiredConditionalRange.ts new file mode 100644 index 0000000..78fda03 --- /dev/null +++ b/packages/validation/test/fixtures/fields/conditionalRestrictions/fieldStringRequiredConditionalRange.ts @@ -0,0 +1,35 @@ +import { SchemaField, type SchemaStringField } from '@overture-stack/lectern-dictionary'; +import { validateFixture } from '../../../testUtils/validateFixture'; +import { fieldNumberNoRestriction } from '../noRestrictions/fieldNumberNoRestriction'; + +export const fieldStringRequiredConditionalRange = { + name: 'required-and-conditional-field', + valueType: 'string', + description: 'Required string, value must match a code list if `any-number` field has value 10 or greater.', + restrictions: [ + { + required: true, + }, + { + if: { + conditions: [ + { + fields: [fieldNumberNoRestriction.name], + match: { + range: { min: 10 }, + }, + }, + ], + }, + then: { + codeList: ['big', 'large', 'huge'], + }, + }, + ], +} as const satisfies SchemaStringField; + +validateFixture( + fieldStringRequiredConditionalRange, + SchemaField, + 'fieldStringRequiredConditionalRange is not a valid SchemaField', +); diff --git a/packages/validation/test/fixtures/fields/multipleRestrictions/fieldStringArrayMultipleRegex.ts b/packages/validation/test/fixtures/fields/multipleRestrictions/fieldStringArrayMultipleRegex.ts new file mode 100644 index 0000000..998ddc8 --- /dev/null +++ b/packages/validation/test/fixtures/fields/multipleRestrictions/fieldStringArrayMultipleRegex.ts @@ -0,0 +1,22 @@ +import type { SchemaStringField } from '@overture-stack/lectern-dictionary'; +import { regexAlphaOnly, regexRepeatedText } from '../../restrictions/regexFixtures'; + +/** + * Example field using an array of multiple restriction objects. + */ +export const fieldStringArrayMultipleRegex = { + name: 'array-multiple-regex-rules', + valueType: 'string', + description: 'String field that must pass multiple regex tests', + meta: { + examples: ['hello', 'byebye', 'thisandthat'], + }, + restrictions: [ + { + regex: regexRepeatedText, + }, + { + regex: regexAlphaOnly, + }, + ], +} as const satisfies SchemaStringField; diff --git a/packages/validation/test/fixtures/fields/schemaRestrictions/fieldStringUnique.ts b/packages/validation/test/fixtures/fields/schemaRestrictions/fieldStringUnique.ts index 364b358..9f44cf5 100644 --- a/packages/validation/test/fixtures/fields/schemaRestrictions/fieldStringUnique.ts +++ b/packages/validation/test/fixtures/fields/schemaRestrictions/fieldStringUnique.ts @@ -3,5 +3,5 @@ import type { SchemaStringField } from '@overture-stack/lectern-dictionary'; export const fieldStringUnique = { name: 'unique-string', valueType: 'string', - restrictions: { unique: true }, + unique: true, } as const satisfies SchemaStringField; diff --git a/packages/validation/test/fixtures/fields/schemaRestrictions/fieldStringUniqueArray.ts b/packages/validation/test/fixtures/fields/schemaRestrictions/fieldStringUniqueArray.ts index 3b486aa..a0500b4 100644 --- a/packages/validation/test/fixtures/fields/schemaRestrictions/fieldStringUniqueArray.ts +++ b/packages/validation/test/fixtures/fields/schemaRestrictions/fieldStringUniqueArray.ts @@ -4,5 +4,5 @@ export const fieldStringUniqueArray = { name: 'unique-string-array', valueType: 'string', isArray: true, - restrictions: { unique: true }, + unique: true, } as const satisfies SchemaStringField; diff --git a/packages/validation/test/fixtures/restrictions/regexFixtures.ts b/packages/validation/test/fixtures/restrictions/regexFixtures.ts index 61b9178..b674232 100644 --- a/packages/validation/test/fixtures/restrictions/regexFixtures.ts +++ b/packages/validation/test/fixtures/restrictions/regexFixtures.ts @@ -1,5 +1,6 @@ import type { RestrictionRegex } from '@overture-stack/lectern-dictionary'; - -export const regexYearMonthDay: RestrictionRegex = '^([12]\\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12]\\d|3[01]))$'; // Example: "1999-01-31" +export const regexAlphaOnly: RestrictionRegex = '^[A-Za-z]*$'; +export const regexRepeatedText: RestrictionRegex = '(\\w+).*\\1'; export const regexMTGMana: RestrictionRegex = '^({((([WUBRG]|([0-9]|[1-9][0-9]*))(/[WUBRG])?)|(X)|([WUBRG](/[WUBRG])?/[P]))})+$'; +export const regexYearMonthDay: RestrictionRegex = '^([12]\\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12]\\d|3[01]))$'; // Example: "1999-01-31" diff --git a/packages/validation/test/validateField/conditions/testMatchCodeList.spec.ts b/packages/validation/test/validateField/conditions/testMatchCodeList.spec.ts new file mode 100644 index 0000000..07adf5e --- /dev/null +++ b/packages/validation/test/validateField/conditions/testMatchCodeList.spec.ts @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { expect } from 'chai'; +import { testMatchCodeList } from '../../../src/validateField/conditions'; + +describe('ConditionalRestriction - testMatchCodeList', () => { + it('Tests true when the primitive value matches an item in the list', () => { + expect(testMatchCodeList(['hello', 'world'], 'hello')).true; + expect(testMatchCodeList(['hello', 'world'], 'world')).true; + expect(testMatchCodeList([123, 456, 789, 1011], 123)).true; + expect(testMatchCodeList([123, 456, 789, 1011], 456)).true; + expect(testMatchCodeList([123, 456, 789, 1011], 789)).true; + expect(testMatchCodeList([123, 456, 789, 1011], 1011)).true; + }); + it('Tests false when the primitive value is not in the list', () => { + expect(testMatchCodeList(['hello', 'world'], 'goodbye')).false; + expect(testMatchCodeList([123, 456, 789, 1011], -123)).false; + }); + it('Tests true when an array has at least one item form the code list', () => { + expect(testMatchCodeList(['hello', 'world'], ['hello', 'everyone'])).true; + expect(testMatchCodeList(['hello', 'world'], ['hello', 'world'])).true; + expect(testMatchCodeList([123, 456, 789, 1011], [1, 12, 123])).true; + }); + it('Tests false when an array has no items form the code list', () => { + expect(testMatchCodeList(['hello', 'world'], ['good', 'bye'])).false; + expect(testMatchCodeList([123, 456, 789, 1011], [-1, -12, -123])).false; + }); +}); diff --git a/packages/validation/test/validateField/conditions/testMatchCount.spec.ts b/packages/validation/test/validateField/conditions/testMatchCount.spec.ts new file mode 100644 index 0000000..2d39f38 --- /dev/null +++ b/packages/validation/test/validateField/conditions/testMatchCount.spec.ts @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { expect } from 'chai'; +import { testMatchCount } from '../../../src/validateField/conditions'; + +describe('ConditionalRestriction - testMatchCount', () => { + describe('Exact value rule', () => { + it('Tests true when the value is the specified length', () => { + expect(testMatchCount(3, [1, 2, 3])).true; + expect(testMatchCount(1, ['test'])).true; + expect(testMatchCount(0, [])).true; + }); + it('Tests false when the value is not the specified length', () => { + expect(testMatchCount(4, [1, 2, 3])).false; + expect(testMatchCount(2, [1, 2, 3])).false; + expect(testMatchCount(2, ['test'])).false; + expect(testMatchCount(0, ['test'])).false; + }); + it('Tests false for non array values', () => { + expect(testMatchCount(4, 'test')).false; + expect(testMatchCount(0, 'test')).false; + expect(testMatchCount(0, '')).false; + expect(testMatchCount(0, undefined)).false; + expect(testMatchCount(0, 0)).false; + expect(testMatchCount(1, 1)).false; + }); + }); + describe('Range rule', () => { + it('Tests true when the value has a length within the range', () => { + expect(testMatchCount({ min: 2, max: 5 }, [1, 2, 3])).true; + expect(testMatchCount({ min: 1 }, [1])).true; + expect(testMatchCount({ exclusiveMax: 10 }, [1, 2, 3, 4, 5, 6, 7, 8, 9])).true; + }); + it('Tests false when the value has a length outside the range', () => { + expect(testMatchCount({ min: 4, max: 5 }, [1, 2, 3])).false; + expect(testMatchCount({ exclusiveMax: 9 }, [1, 2, 3, 4, 5, 6, 7, 8, 9])).false; + expect(testMatchCount({ exclusiveMin: 1 }, [1])).false; + }); + it('Tests false for non array values', () => { + expect(testMatchCount({ min: 1 }, 'test')).false; + expect(testMatchCount({ min: 1 }, 'test')).false; + expect(testMatchCount({ min: 1 }, '')).false; + expect(testMatchCount({ min: 1 }, undefined)).false; + expect(testMatchCount({ min: 1 }, 0)).false; + expect(testMatchCount({ min: 1 }, 1)).false; + }); + }); +}); diff --git a/packages/validation/test/validateField/conditions/testMatchExists.spec.ts b/packages/validation/test/validateField/conditions/testMatchExists.spec.ts new file mode 100644 index 0000000..94d3d9b --- /dev/null +++ b/packages/validation/test/validateField/conditions/testMatchExists.spec.ts @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { expect } from 'chai'; +import { testMatchExists } from '../../../src/validateField/conditions'; + +describe('ConditionalRestriction - testMatchExists', () => { + it('Tests true with all primitive values', () => { + expect(testMatchExists(true, 'hello')).true; + expect(testMatchExists(true, 123)).true; + expect(testMatchExists(true, 0)).true; + expect(testMatchExists(true, true)).true; + }); + it('Tests true with array values with some elements', () => { + expect(testMatchExists(true, ['hello'])).true; + expect(testMatchExists(true, ['hello', 'world', 'how are you?'])).true; + expect(testMatchExists(true, [123, 456, 789])).true; + expect(testMatchExists(true, [true, false, true, false])).true; + }); + it('Tests true for value `false`', () => { + expect(testMatchExists(true, false)).true; + }); + it('Tests false for `undefined`', () => { + expect(testMatchExists(true, undefined)).false; + }); + it('Tests false for empty string values', () => { + expect(testMatchExists(true, '')).false; + }); + it('Tests false for string values with only whitespace', () => { + expect(testMatchExists(true, ' ')).false; + }); + it('Tests false for non-finite numbers (NaN, Infinity)', () => { + expect(testMatchExists(true, NaN)).false; + expect(testMatchExists(true, Infinity)).false; + expect(testMatchExists(true, -Infinity)).false; + }); + it('Tests false for empty array value', () => { + expect(testMatchExists(true, [])).false; + }); + it('Tests false for arrays with only non existing elements', () => { + expect(testMatchExists(true, [''])).false; + expect(testMatchExists(true, ['', ' '])).false; + expect(testMatchExists(true, [NaN, Infinity])).false; + }); + describe('Inverse rule - exists = false', () => { + it('Tests true when value is missing and exists=false', () => { + expect(testMatchExists(false, undefined)).true; + expect(testMatchExists(false, '')).true; + expect(testMatchExists(false, [])).true; + }); + it('Exist rule `false` resolves `false` when value is provided and exists=false', () => { + expect(testMatchExists(false, 'hello')).false; + expect(testMatchExists(false, true)).false; + expect(testMatchExists(false, 123)).false; + }); + }); +}); diff --git a/packages/validation/test/validateField/conditions/testMatchRange.spec.ts b/packages/validation/test/validateField/conditions/testMatchRange.spec.ts new file mode 100644 index 0000000..6e299a7 --- /dev/null +++ b/packages/validation/test/validateField/conditions/testMatchRange.spec.ts @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { expect } from 'chai'; +import { testMatchRange } from '../../../src/validateField/conditions'; + +describe('ConditionalRestriction - testMatchRange', () => { + it('Tests true for number values within range', () => { + expect(testMatchRange({ min: 0, max: 10 }, 5)).true; + expect(testMatchRange({ min: 1000, max: 2000 }, 1005)).true; + expect(testMatchRange({ exclusiveMin: 0 }, 1)).true; + expect(testMatchRange({ exclusiveMax: 0 }, -1)).true; + }); + it('Tests false for number values outside of range', () => { + expect(testMatchRange({ min: 0, max: 10 }, 15)).false; + expect(testMatchRange({ min: 1000, max: 2000 }, 2005)).false; + expect(testMatchRange({ exclusiveMin: 0 }, 0)).false; + expect(testMatchRange({ exclusiveMax: 0 }, 0)).false; + }); + it('Tests false for non-number primitive values', () => { + expect(testMatchRange({ min: 0, max: 10 }, 'hello')).false; + expect(testMatchRange({ min: 0, max: 10 }, '5')).false; + expect(testMatchRange({ min: 0, max: 10 }, true)).false; + expect(testMatchRange({ min: 0, max: 10 }, false)).false; + expect(testMatchRange({ min: 0, max: 10 }, undefined)).false; + }); + it('Tests true for array with at least one number value within range', () => { + expect(testMatchRange({ min: 0, max: 10 }, [5])).true; + expect(testMatchRange({ min: 0, max: 10 }, [5, 6, 7, 8, 9])).true; + expect(testMatchRange({ min: 0, max: 10 }, [5, 15, 25, 35])).true; + }); + it('Tests false for array with no value within range', () => { + expect(testMatchRange({ min: 0, max: 10 }, [])).false; + expect(testMatchRange({ min: 0, max: 10 }, [15, 25, 35])).false; + expect(testMatchRange({ min: 0, max: 10 }, ['5', 'hello'])).false; + }); +}); diff --git a/packages/validation/test/validateField/conditions/testMatchRegex.spec.ts b/packages/validation/test/validateField/conditions/testMatchRegex.spec.ts new file mode 100644 index 0000000..a476871 --- /dev/null +++ b/packages/validation/test/validateField/conditions/testMatchRegex.spec.ts @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { expect } from 'chai'; +import { testMatchRegex } from '../../../src/validateField/conditions'; +import { regexAlphaOnly, regexRepeatedText } from '../../fixtures/restrictions/regexFixtures'; + +describe('ConditionalRestriction - testMatchRegex', () => { + it('Tests true with string value that matches regex', () => { + expect(testMatchRegex(regexAlphaOnly, 'qwerty')).true; + expect(testMatchRegex(regexRepeatedText, '123 asdf 123')).true; + }); + it('Tests false with string value that does not matche regex', () => { + expect(testMatchRegex(regexAlphaOnly, 'letters and spaces and numbers 123')).false; + expect(testMatchRegex(regexRepeatedText, '123 asdf 456')).false; + }); + it('Tests true with empty string if that passes the regex rule', () => { + expect(testMatchRegex(regexAlphaOnly, '')).true; + }); + it('Tests false with non-string primitive values', () => { + expect(testMatchRegex(regexRepeatedText, 123123)).false; + expect(testMatchRegex(regexRepeatedText, true)).false; + expect(testMatchRegex(regexRepeatedText, undefined)).false; + }); + + it('Tests true when value is an array with at least one matching element', () => { + expect(testMatchRegex(regexAlphaOnly, ['asdf'])).true; + expect(testMatchRegex(regexAlphaOnly, ['asdf', '123', 'qwerty12345'])).true; + expect(testMatchRegex(regexAlphaOnly, ['qwerty12345', '1234', 'ok', 'not ok!'])).true; + }); + + it('Tests false when value is an array with no matching elements', () => { + expect(testMatchRegex(regexAlphaOnly, [])).false; + expect(testMatchRegex(regexAlphaOnly, ['123', 'qwerty12345'])).false; + expect(testMatchRegex(regexAlphaOnly, ['qwerty12345', '1234', 'not ok!'])).false; + expect(testMatchRegex(regexAlphaOnly, [123, 456, 789])).false; + }); +}); diff --git a/packages/validation/test/validateField/conditions/testMatchValue.spec.ts b/packages/validation/test/validateField/conditions/testMatchValue.spec.ts new file mode 100644 index 0000000..b8471dd --- /dev/null +++ b/packages/validation/test/validateField/conditions/testMatchValue.spec.ts @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { expect } from 'chai'; +import { testMatchValue } from '../../../src/validateField/conditions'; + +describe('ConditionalRestriction - testMatchValue', () => { + it('Primitive values that match test true', () => { + expect(testMatchValue('hello', 'hello')).true; + expect(testMatchValue('hello world', 'hello world')).true; + expect(testMatchValue(123, 123)).true; + expect(testMatchValue(0, 0)).true; + expect(testMatchValue(true, true)).true; + expect(testMatchValue(false, false)).true; + }); + it('Primitive values that do not match test false', () => { + expect(testMatchValue('hello', 'goodbye')).false; + expect(testMatchValue('hello world', 'nevermore')).false; + expect(testMatchValue(123, 1234)).false; + expect(testMatchValue(0, 10)).false; + expect(testMatchValue(true, false)).false; + expect(testMatchValue(false, true)).false; + }); + describe('Array values', () => { + it('Array values never match a primitive match rule', () => { + expect(testMatchValue('hello', ['hello'])).false; + expect(testMatchValue('hello world', ['hello world'])).false; + expect(testMatchValue(123, [123])).false; + expect(testMatchValue(0, [0])).false; + expect(testMatchValue(true, [true])).false; + expect(testMatchValue(false, [false])).false; + }); + it('Primitive values never match an array match rule', () => { + expect(testMatchValue(['hello'], 'hello')).false; + expect(testMatchValue(['hello world'], 'hello world')).false; + expect(testMatchValue([123], 123)).false; + expect(testMatchValue([0], 0)).false; + expect(testMatchValue([true], true)).false; + expect(testMatchValue([false], false)).false; + }); + it('Exact arrays teset true', () => { + expect(testMatchValue(['hello', 'world'], ['hello', 'world'])).true; + expect(testMatchValue(['hello world'], ['hello world'])).true; + expect(testMatchValue([123, 456, 789], [123, 456, 789])).true; + expect(testMatchValue([0], [0])).true; + expect(testMatchValue([true, true, true, false], [true, true, true, false])).true; + expect(testMatchValue([false], [false])).true; + }); + it('Arrays with same values in different order test true', () => { + expect(testMatchValue(['hello', 'world'], ['world', 'hello'])).true; + expect(testMatchValue([123, 123, -456, 789], [-456, 123, 789, 123])).true; + expect(testMatchValue([true, true, true, false, false], [false, true, false, true, true])).true; + }); + it('Arrays with extra values test false', () => { + expect(testMatchValue(['hello', 'world'], ['world', 'hello', 'extra'])).false; + expect(testMatchValue([123, 123, -456, 789], [-456, 123, 789, 123, 0])).false; + expect(testMatchValue([true, true, true, false, false], [false, true, false, true, true, true])).false; + }); + it('Arrays with missing values test false', () => { + expect(testMatchValue(['hello', 'world'], ['world'])).false; + expect(testMatchValue([123, 123, -456, 789], [-456, 123, 789])).false; + expect(testMatchValue([true, true, true, false, false], [false, true, false, true])).false; + }); + it('Arrays different missing values test false', () => { + expect(testMatchValue(['hello', 'world'], ['world', 'helo'])).false; + expect(testMatchValue([123, 123, -456, 789], [-456, 124, 123, 789])).false; + expect(testMatchValue([true, true, true, false, false], [false, true, false, true, false])).false; + }); + }); +}); diff --git a/packages/validation/test/validateField/resolveFieldRestrictions.spec.ts b/packages/validation/test/validateField/resolveFieldRestrictions.spec.ts new file mode 100644 index 0000000..1f460dc --- /dev/null +++ b/packages/validation/test/validateField/resolveFieldRestrictions.spec.ts @@ -0,0 +1,419 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { ARRAY_TEST_CASE_DEFAULT, type DataRecord } from '@overture-stack/lectern-dictionary'; +import assert from 'assert'; +import { expect } from 'chai'; +import { cloneDeep } from 'lodash'; +import { resolveFieldRestrictions } from '../../src/validateField/restrictions/resolveFieldRestrictions'; +import { fieldStringConditionalExists } from '../fixtures/fields/conditionalRestrictions/fieldStringConditionalExists'; +import { fieldStringConditionalMultipleConditions } from '../fixtures/fields/conditionalRestrictions/fieldStringConditionalMultipleConditions'; +import { fieldStringConditionalMultipleFieldsRegex } from '../fixtures/fields/conditionalRestrictions/fieldStringConditionalMultipleFieldsRegex'; +import { fieldStringNestedConditional } from '../fixtures/fields/conditionalRestrictions/fieldStringNestedConditional'; +import { fieldStringRequiredConditionalRange } from '../fixtures/fields/conditionalRestrictions/fieldStringRequiredConditionalRange'; +import { fieldStringArrayMultipleRegex } from '../fixtures/fields/multipleRestrictions/fieldStringArrayMultipleRegex'; +import { fieldStringManyRestrictions } from '../fixtures/fields/multipleRestrictions/fieldStringManyRestrictions'; +import { fieldBooleanNoRestriction } from '../fixtures/fields/noRestrictions/fieldBooleanNoRestriction'; +import { fieldNumberNoRestriction } from '../fixtures/fields/noRestrictions/fieldNumberNoRestriction'; +import { fieldStringNoRestriction } from '../fixtures/fields/noRestrictions/fieldStringNoRestriction'; +import { regexAlphaOnly } from '../fixtures/restrictions/regexFixtures'; + +describe('Field - resolveFieldRestrictions', () => { + it('Returns empty array when there are no restrictions', () => { + const restrictions = resolveFieldRestrictions(undefined, {}, fieldStringNoRestriction); + expect(restrictions.length).equal(0); + }); + it('Returns array with rules matching restrictions in a single restrictions object', () => { + const restrictions = resolveFieldRestrictions(undefined, {}, fieldStringManyRestrictions); + expect(restrictions.length).equal(3); + const regexRestriction = restrictions.find((restriction) => restriction.type === 'regex'); + expect(regexRestriction).not.undefined; + const requiredRestriction = restrictions.find((restriction) => restriction.type === 'required'); + expect(requiredRestriction).not.undefined; + const codeListRestriction = restrictions.find((restriction) => restriction.type === 'codeList'); + expect(codeListRestriction).not.undefined; + }); + it('Returns array with rules from all objects in restrictions array', () => { + const restrictions = resolveFieldRestrictions(undefined, {}, fieldStringArrayMultipleRegex); + expect(restrictions.length).equal(2); + expect(restrictions.every((restriction) => restriction.type === 'regex')).true; + }); + describe('Conditional Restrictions', () => { + it('Returns `then` restrictions when condition is true', () => { + const record: DataRecord = { + [fieldStringNoRestriction.name]: 'has value', + [fieldStringConditionalExists.name]: 'anything', + }; + const restrictions = resolveFieldRestrictions( + record[fieldStringConditionalExists.name], + record, + fieldStringConditionalExists, + ); + expect(restrictions.length).equal(1); + expect(restrictions[0]?.type).equal('required'); + expect(restrictions[0]?.rule).equal(true); + }); + it('Returns `else` restrictions when condition is false', () => { + const record: DataRecord = { + [fieldStringNoRestriction.name]: undefined, + [fieldStringConditionalExists.name]: 'anything', + }; + const restrictions = resolveFieldRestrictions( + record[fieldStringConditionalExists.name], + record, + fieldStringConditionalExists, + ); + expect(restrictions.length).equal(1); + expect(restrictions[0]?.type).equal('empty'); + expect(restrictions[0]?.rule).equal(true); + }); + it('Combines conditional restrictions with other restrictions', () => { + const record: DataRecord = { + [fieldNumberNoRestriction.name]: 15, + [fieldStringRequiredConditionalRange.name]: 'big', + }; + const restrictions = resolveFieldRestrictions( + record[fieldStringRequiredConditionalRange.name], + record, + fieldStringRequiredConditionalRange, + ); + expect(restrictions.length).equal(2); + + // one of the restrictions is 'required' + const requiredRestriction = restrictions.find((restriction) => restriction.type === 'required'); + expect(requiredRestriction).not.undefined; + + // one of the restrictions is 'codeList' + const codeListRestriction = restrictions.find((restriction) => restriction.type === 'codeList'); + expect(codeListRestriction).not.undefined; + }); + it('Does not add any restriction when condition fails and there is no `else`', () => { + const record: DataRecord = { + [fieldNumberNoRestriction.name]: 0, + [fieldStringRequiredConditionalRange.name]: 'anything goes', + }; + const restrictions = resolveFieldRestrictions( + record[fieldStringRequiredConditionalRange.name], + record, + fieldStringRequiredConditionalRange, + ); + expect(restrictions.length).equal(1); + + // one of the restrictions is 'required' + const requiredRestriction = restrictions.find((restriction) => restriction.type === 'required'); + expect(requiredRestriction).not.undefined; + + // no 'codeList' restriction + const codeListRestriction = restrictions.find((restriction) => restriction.type === 'codeList'); + expect(codeListRestriction).undefined; + }); + describe('ConditionalRestrictionTest Case', () => { + // This is the `case` property at the level of the `conditions` array. It specifies how many conditions must match + // for this test to be true and apply the `then` restrictions. If the restriction resolves to `required` this indicates + // the `then` object is returned and therefore the condition tested true. The condition fail case returns the `else` + // with an `empty` restriction. Therefore, `required` is test passes, `empty` is test failed. + + const recordAllFields: DataRecord = { + [fieldStringNoRestriction.name]: 'asdf', + [fieldNumberNoRestriction.name]: 1, + [fieldBooleanNoRestriction.name]: true, + }; + const recordOneField: DataRecord = { + [fieldStringNoRestriction.name]: 'asdf', + }; + const recordNoFields: DataRecord = {}; + + it('Default case is `all`', () => { + expect(ARRAY_TEST_CASE_DEFAULT).equal('all'); + + // also repeating the case=`all` test with no case value specified, makes sure this default is being applied + const restrictionsAllFields = resolveFieldRestrictions( + 'irrelevant', + recordAllFields, + fieldStringConditionalMultipleConditions, + ); + const restrictionsOneField = resolveFieldRestrictions( + 'irrelevant', + recordOneField, + fieldStringConditionalMultipleConditions, + ); + const restrictionsNoFields = resolveFieldRestrictions( + 'irrelevant', + recordNoFields, + fieldStringConditionalMultipleConditions, + ); + + expect(restrictionsAllFields[0]?.type).equal('required'); // test passed + expect(restrictionsOneField[0]?.type).equal('empty'); // test failed + expect(restrictionsNoFields[0]?.type).equal('empty'); // test failed + }); + it('Case `all` requires every condition to be true', () => { + const fieldCaseAll = cloneDeep(fieldStringConditionalMultipleConditions); + assert( + typeof fieldCaseAll.restrictions === 'object' && + 'if' in fieldCaseAll.restrictions && + fieldCaseAll.restrictions?.if, + ); + fieldCaseAll.restrictions.if.case = 'all'; + + const restrictionsAllFields = resolveFieldRestrictions('irrelevant', recordAllFields, fieldCaseAll); + const restrictionsOneField = resolveFieldRestrictions('irrelevant', recordOneField, fieldCaseAll); + const restrictionsNoFields = resolveFieldRestrictions('irrelevant', recordNoFields, fieldCaseAll); + + expect(restrictionsAllFields[0]?.type).equal('required'); + expect(restrictionsOneField[0]?.type).equal('empty'); + expect(restrictionsNoFields[0]?.type).equal('empty'); + }); + it('Case `any` requires at least one condition to be true', () => { + const fieldCaseAny = cloneDeep(fieldStringConditionalMultipleConditions); + assert( + typeof fieldCaseAny.restrictions === 'object' && + 'if' in fieldCaseAny.restrictions && + fieldCaseAny.restrictions?.if, + ); + fieldCaseAny.restrictions.if.case = 'any'; + + const restrictionsAllFields = resolveFieldRestrictions('irrelevant', recordAllFields, fieldCaseAny); + const restrictionsOneField = resolveFieldRestrictions('irrelevant', recordOneField, fieldCaseAny); + const restrictionsNoFields = resolveFieldRestrictions('irrelevant', recordNoFields, fieldCaseAny); + + expect(restrictionsAllFields[0]?.type).equal('required'); // test resolves true + expect(restrictionsOneField[0]?.type).equal('required'); // test resolves true due to at least one field present + expect(restrictionsNoFields[0]?.type).equal('empty'); + }); + it('Case `none` requires all conditions to be false', () => { + const fieldCaseNone = cloneDeep(fieldStringConditionalMultipleConditions); + assert( + typeof fieldCaseNone.restrictions === 'object' && + 'if' in fieldCaseNone.restrictions && + fieldCaseNone.restrictions?.if, + ); + fieldCaseNone.restrictions.if.case = 'none'; + + const restrictionsAllFields = resolveFieldRestrictions('irrelevant', recordAllFields, fieldCaseNone); + const restrictionsOneField = resolveFieldRestrictions('irrelevant', recordOneField, fieldCaseNone); + const restrictionsNoFields = resolveFieldRestrictions('irrelevant', recordNoFields, fieldCaseNone); + + expect(restrictionsAllFields[0]?.type).equal('empty'); // test resolves false + expect(restrictionsOneField[0]?.type).equal('empty'); // test resolves false + expect(restrictionsNoFields[0]?.type).equal('required'); // test resolves true + }); + }); + + describe('RestrictionCondition Case', () => { + /* + The case property inside a condition indicates how many of the fields must pass the match test in order + for the condition to be true. + + The field to test this lists three string fields in the `fields` property and will match each of them with + the alpha-only regex. + + When the condition passes, the restrictions will resolve with `required`, and when it fails it will resolve + to `empty`. + */ + const recordAllMatch: DataRecord = { + first: 'asdf', + second: 'qwerty', + third: 'hello', + }; + const recordOneMatch: DataRecord = { + first: undefined, + second: 'qwerty', // matches alpha-only req + third: '1234', + }; + const recordNoMatches: DataRecord = { + first: undefined, + second: 'hello! world!', //symbols and whitespace don't match + third: '1234', + }; + + it('Default case is `all`', () => { + const restrictionsAllMatch = resolveFieldRestrictions( + 'irrelevant', + recordAllMatch, + fieldStringConditionalMultipleFieldsRegex, + ); + const restrictionsOneMatch = resolveFieldRestrictions( + 'irrelevant', + recordOneMatch, + fieldStringConditionalMultipleFieldsRegex, + ); + const restrictionsNoMatches = resolveFieldRestrictions( + 'irrelevant', + recordNoMatches, + fieldStringConditionalMultipleFieldsRegex, + ); + + expect(restrictionsAllMatch[0]?.type).equal('required'); // test passed + expect(restrictionsOneMatch[0]?.type).equal('empty'); // test failed + expect(restrictionsNoMatches[0]?.type).equal('empty'); // test failed + }); + it('Case `all` requires every field to match', () => { + const fieldCaseAll = cloneDeep(fieldStringConditionalMultipleFieldsRegex); + assert( + typeof fieldCaseAll.restrictions === 'object' && + 'if' in fieldCaseAll.restrictions && + fieldCaseAll.restrictions?.if && + fieldCaseAll.restrictions.if.conditions[0], + ); + fieldCaseAll.restrictions.if.conditions[0].case = 'all'; + + const restrictionsAllMatch = resolveFieldRestrictions('irrelevant', recordAllMatch, fieldCaseAll); + const restrictionsOneMatch = resolveFieldRestrictions('irrelevant', recordOneMatch, fieldCaseAll); + const restrictionsNoMatches = resolveFieldRestrictions('irrelevant', recordNoMatches, fieldCaseAll); + + expect(restrictionsAllMatch[0]?.type).equal('required'); // test passed + expect(restrictionsOneMatch[0]?.type).equal('empty'); // test failed + expect(restrictionsNoMatches[0]?.type).equal('empty'); // test failed + }); + + it('Case `any` requires at lest one field to match', () => { + const fieldCaseAny = cloneDeep(fieldStringConditionalMultipleFieldsRegex); + assert( + typeof fieldCaseAny.restrictions === 'object' && + 'if' in fieldCaseAny.restrictions && + fieldCaseAny.restrictions?.if && + fieldCaseAny.restrictions.if.conditions[0], + ); + fieldCaseAny.restrictions.if.conditions[0].case = 'any'; + + const restrictionsAllMatch = resolveFieldRestrictions('irrelevant', recordAllMatch, fieldCaseAny); + const restrictionsOneMatch = resolveFieldRestrictions('irrelevant', recordOneMatch, fieldCaseAny); + const restrictionsNoMatches = resolveFieldRestrictions('irrelevant', recordNoMatches, fieldCaseAny); + + expect(restrictionsAllMatch[0]?.type).equal('required'); // test passed + expect(restrictionsOneMatch[0]?.type).equal('required'); // test passed + expect(restrictionsNoMatches[0]?.type).equal('empty'); // test failed + }); + + it('Case `none` requires at lest one field to match', () => { + const fieldCaseNone = cloneDeep(fieldStringConditionalMultipleFieldsRegex); + assert( + typeof fieldCaseNone.restrictions === 'object' && + 'if' in fieldCaseNone.restrictions && + fieldCaseNone.restrictions?.if && + fieldCaseNone.restrictions.if.conditions[0], + ); + fieldCaseNone.restrictions.if.conditions[0].case = 'none'; + + const restrictionsAllMatch = resolveFieldRestrictions('irrelevant', recordAllMatch, fieldCaseNone); + const restrictionsOneMatch = resolveFieldRestrictions('irrelevant', recordOneMatch, fieldCaseNone); + const restrictionsNoMatches = resolveFieldRestrictions('irrelevant', recordNoMatches, fieldCaseNone); + + expect(restrictionsAllMatch[0]?.type).equal('empty'); // test passed + expect(restrictionsOneMatch[0]?.type).equal('empty'); // test passed + expect(restrictionsNoMatches[0]?.type).equal('required'); // test failed + }); + }); + describe('Nested Conditions', () => { + /* + The field we are testing with has 3 conditions we can test, structured like: + if(A - `any-string` has repeated text): + then: + if(B - `any-number` is 0 or greater): + then: regex + if(C - `any-boolean` === `true`): + then: required + else: empty + + to test this, we want to try the following cases: + case 1 - A false. only restriction is empty + case 2 - A true, B and C false. No restrictions. + case 3 - A true, B and C true. regex and required restrictions. + case 4 - A true, B true, and C false. only regex required. + case 5 - A true, B false, and C true. only required restriction. + */ + it('Case 1 - Root condition false, only root else restrictions returned', () => { + const record: DataRecord = { + [fieldStringNoRestriction.name]: 'asdf', + [fieldNumberNoRestriction.name]: 1, + [fieldBooleanNoRestriction.name]: true, + }; + const restrictions = resolveFieldRestrictions('irrelevant', record, fieldStringNestedConditional); + expect(restrictions.length).equal(1); + expect(restrictions[0]?.type).equal('empty'); + }); + it('Case 2 - Root condition true, other conditions false, resolves with no restrictions', () => { + const record: DataRecord = { + [fieldStringNoRestriction.name]: 'repeated repeated', + [fieldNumberNoRestriction.name]: -1, + [fieldBooleanNoRestriction.name]: false, + }; + const restrictions = resolveFieldRestrictions('irrelevant', record, fieldStringNestedConditional); + expect(restrictions.length).equal(0); + }); + + it('Case 3 - All conditions true, all then restrictions resolved', () => { + const record: DataRecord = { + [fieldStringNoRestriction.name]: 'repeated repeated', + [fieldNumberNoRestriction.name]: 1, + [fieldBooleanNoRestriction.name]: true, + }; + const restrictions = resolveFieldRestrictions('irrelevant', record, fieldStringNestedConditional); + expect(restrictions.length).equal(2); + + // one of the restrictions is 'required' + const requiredRestriction = restrictions.find((restriction) => restriction.type === 'required'); + expect(requiredRestriction).not.undefined; + expect(requiredRestriction?.rule).equal(true); + + // one of the restrictions is 'regex' + const regexRestriction = restrictions.find((restriction) => restriction.type === 'regex'); + expect(regexRestriction).not.undefined; + expect(regexRestriction?.rule).equal(regexAlphaOnly); + }); + it('Case 4 - Root condition true, number condition true and boolean condition false, returns regex restriction', () => { + const record: DataRecord = { + [fieldStringNoRestriction.name]: 'repeated repeated', + [fieldNumberNoRestriction.name]: 1, + [fieldBooleanNoRestriction.name]: false, + }; + const restrictions = resolveFieldRestrictions('irrelevant', record, fieldStringNestedConditional); + expect(restrictions.length).equal(1); + + // one of the restrictions is 'required' + const requiredRestriction = restrictions.find((restriction) => restriction.type === 'required'); + expect(requiredRestriction).undefined; + + // one of the restrictions is 'regex' + const regexRestriction = restrictions.find((restriction) => restriction.type === 'regex'); + expect(regexRestriction).not.undefined; + expect(regexRestriction?.rule).equal(regexAlphaOnly); + }); + it('Case 5 - Root condition true, number condition true and boolean condition false, returns regex restriction', () => { + const record: DataRecord = { + [fieldStringNoRestriction.name]: 'repeated repeated', + [fieldNumberNoRestriction.name]: -1, + [fieldBooleanNoRestriction.name]: true, + }; + const restrictions = resolveFieldRestrictions('irrelevant', record, fieldStringNestedConditional); + expect(restrictions.length).equal(1); + + // one of the restrictions is 'required' + const requiredRestriction = restrictions.find((restriction) => restriction.type === 'required'); + expect(requiredRestriction).not.undefined; + expect(requiredRestriction?.rule).equal(true); + + // one of the restrictions is 'regex' + const regexRestriction = restrictions.find((restriction) => restriction.type === 'regex'); + expect(regexRestriction).undefined; + }); + }); + }); +}); diff --git a/packages/validation/test/validateField/validateField.spec.ts b/packages/validation/test/validateField/validateField.spec.ts index ecaafa9..2078967 100644 --- a/packages/validation/test/validateField/validateField.spec.ts +++ b/packages/validation/test/validateField/validateField.spec.ts @@ -32,6 +32,7 @@ import { fieldStringArrayRequired } from '../fixtures/fields/simpleRestrictions/ import { fieldStringCodeList } from '../fixtures/fields/simpleRestrictions/string/fieldStringCodeList'; import { fieldStringRegex } from '../fixtures/fields/simpleRestrictions/string/fieldStringRegex'; import { fieldStringRequired } from '../fixtures/fields/simpleRestrictions/string/fieldStringRequired'; +import { fieldStringArrayMultipleRegex } from '../fixtures/fields/multipleRestrictions/fieldStringArrayMultipleRegex'; const emptyDataRecord = {}; @@ -395,5 +396,20 @@ describe('Field - validateField', () => { expect(codeListError).exist; }); }); + + describe('String with array of restrictions, multiple regex', () => { + it('Valid with value that matches both regexs', () => { + // from examples in field: ['hello', 'byebye', 'thisandthat'] + expect(validateField('hello', emptyDataRecord, fieldStringArrayMultipleRegex).valid).true; + expect(validateField('byebye', emptyDataRecord, fieldStringArrayMultipleRegex).valid).true; + expect(validateField('thisandthat', emptyDataRecord, fieldStringArrayMultipleRegex).valid).true; + }); + it('Invalid with value that one or both regexes', () => { + // has non alpha characters + expect(validateField('hello123', emptyDataRecord, fieldStringArrayMultipleRegex).valid).false; + // no repeated characters + expect(validateField('asdf', emptyDataRecord, fieldStringArrayMultipleRegex).valid).false; + }); + }); }); }); diff --git a/packages/validation/test/validateRecord/validateRecord.spec.ts b/packages/validation/test/validateRecord/validateRecord.spec.ts index d338f5d..1f8870c 100644 --- a/packages/validation/test/validateRecord/validateRecord.spec.ts +++ b/packages/validation/test/validateRecord/validateRecord.spec.ts @@ -143,7 +143,6 @@ describe('Record - validateRecord', () => { }, schemaAllDataTypesMixedRestrictions, ); - console.log(JSON.stringify(result, null, 2)); expect(result.valid).false; assert(result.valid === false); diff --git a/scripts/src/generateMetaSchema.ts b/scripts/src/generateMetaSchema.ts index c419a3e..5b71a0c 100644 --- a/scripts/src/generateMetaSchema.ts +++ b/scripts/src/generateMetaSchema.ts @@ -3,15 +3,23 @@ * It will be output into the file ./generated/DictionaryMetaSchema.json */ import { + BooleanFieldRestrictions, + ConditionalRestrictionTest, Dictionary, - DictionaryBase, DictionaryMeta, + IntegerFieldRestrictions, NameValue, + NumberFieldRestrictions, ReferenceArray, ReferenceTag, References, Schema, + SchemaBooleanField, SchemaField, + SchemaIntegerField, + SchemaNumberField, + SchemaStringField, + StringFieldRestrictions, } from '@overture-stack/lectern-dictionary'; import fs from 'fs'; import { zodToJsonSchema } from 'zod-to-json-schema'; @@ -22,11 +30,20 @@ console.log('Generating JSON Schema Meta-Schema...'); const jsonSchema = zodToJsonSchema(Dictionary, { name: 'Dictionary', definitions: { - ReferenceTag, - ReferenceArray, - References, + SchemaBooleanField, + SchemaIntegerField, + SchemaNumberField, + SchemaStringField, + BooleanFieldRestrictions, + IntegerFieldRestrictions, + NumberFieldRestrictions, + StringFieldRestrictions, + ConditionalRestrictionTest, Meta: DictionaryMeta, Name: NameValue, + ReferenceArray, + ReferenceTag, + References, Schema, SchemaField, }, From 7169764ad1fd9de39f7279101431809914a39e4c Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Thu, 22 Aug 2024 12:00:03 -0400 Subject: [PATCH 26/27] Update dictionary-reference to show unique as field property --- docs/dictionary-reference.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/dictionary-reference.md b/docs/dictionary-reference.md index e06bb0a..fe1f14c 100644 --- a/docs/dictionary-reference.md +++ b/docs/dictionary-reference.md @@ -76,6 +76,7 @@ In addition to schemas, a Lectern Dictionary can contain reference values that c | `meta` | Optional | Empty object, no value | [`MetaData`](#meta-data-structure) object | Schema implementor defined fields to capture any additional properties not defined in standard Lectern fields. | `{ "displayName": "Example Field" }` | | `isArray` | Optional | `false` | `boolean` | Type of value stored in this field | | | `restrictions` | Optional | No Restrictions | `RestrictionsObject` or `Array` | An object containing all validation rules for this field. This can be a single object containing all [restrictions](#field-restrictions) applied to this field or a list of objects whose restrictions will be combined. [Conditional restrictions](#conditional-restrictions) can also be used to apply validation rules based on values of other fields in the record. | `{ "required": true }` | +| `unique` | Optional | `false` | `boolean` | Indicates that every record in this schema should have a unique value for this field. This rule is applied when a collection of records are validated together, ensuring that no two records in that collection repeat a value. | `true` | @@ -105,7 +106,6 @@ The full list of available restrictions are: | `range` | `integer`, `number` | | Uses a [RangeRule](#rangerule-data-structure) object to define minimum and/or maximum values for this field | `{"min": 5}`, `{"exclusiveMax": 50}`, `{"exclusiveMin": 5, "max": 50}` | | `regex` | `string` | | A regular expression that all values must match. | `^[a-z0-9]+$` | | `required` | all | | A value must be provided, missing/undefined values will fail validation. Empty strings will not be accepted, though `0` (for `number` and `int` fields) and `false` (for `boolean` fields) are accepted. An array field with this restriction must have at least one entry. | `true`, `false` | -| `unique` | all | | When a field has the `unique` restriction, each record must have a distinct value for this field. Uniqueness tests are case insensitive, so `Abc` and `abc` are treated as the same value. This restriction is only applied when a collection of records are tested together, ensuring that no two records in that collection share a value. | `true`, `false` | #### Conditional Restrictions @@ -170,14 +170,14 @@ A requirement condition is defined by providing a field name or list of field na > ``` ##### Conditional Match Rules -| Property | Used with Field Types | Type | Description | Example | -| ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------- | -------------------------------------------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | -| `codeList` | all | Array of type of specified fields | A list of values that the field could match. This rule passes when the specified field's value can be found in this list. | `["value_one", "value_two"]` | -| `count` | Array type fields | Integer, or [RangeRule](#rangerule-data-structure) | Matches the number of values in an array field. This condition can be provided as a number, in which case this condition matches if the array is that exact length. This condition can be provided as a Range object as well, in which case it will match if the number of elements in the array pass the minimum and maximum conditions provided in the condition. | `2` - Field must have exactly 2 elements.
`{ max: 10 }` - Field must have no more than 10 items. | -| `exists` | all | Boolean | This condition requires a field to either have a value or have no value. When the `exists` condition is set to `true`, the field must have a value. When `exists` is sdet to `false`, the field must have no value. For array fields, `exists=false` only matches when the array is completely empty, and `exists=true` passes if the array has 1 or more values - `arrayCase` has no interaction with the `exists` condition. | `true` | -| `range` | `number`, `integer` | [RangeRule](#rangerule-data-structure) | Maximum and minimum value conditions that a numeric field must pass. | `{ min: 5, exclusiveMax: 10 }` Represents an integer from 5-9. | -| `regex` | `string` | String (Regular Expression) | A regular expression pattern that the value must match. | `^NCIT:C\d+$` Value must match an NCI Thesaurus ID. | -| `value` | all | Type of specified fields | Field value matches the value of the specified field. Strings are matched case insensitive. When arrays are matched, the order of their elements is ignored - a field matches this condition if the elements in field are the same elements as in the value match rule. For example, the rule `['abc', 'def']` matches the value `['def', 'abc']` but does not match `['abc', 'def', 'ghi']`. | `some_value`, `[1, 2, 3]` | +| Property | Used with Field Types | Type | Description | Example | +| ---------- | --------------------- | -------------------------------------------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | +| `codeList` | all | Array of type of specified fields | A list of values that the field could match. This rule passes when the specified field's value can be found in this list. | `["value_one", "value_two"]` | +| `count` | Array type fields | Integer, or [RangeRule](#rangerule-data-structure) | Matches the number of values in an array field. This condition can be provided as a number, in which case this condition matches if the array is that exact length. This condition can be provided as a Range object as well, in which case it will match if the number of elements in the array pass the minimum and maximum conditions provided in the condition. | `2` - Field must have exactly 2 elements.
`{ max: 10 }` - Field must have no more than 10 items. | +| `exists` | all | Boolean | This condition requires a field to either have a value or have no value. When the `exists` condition is set to `true`, the field must have a value. When `exists` is sdet to `false`, the field must have no value. For array fields, `exists=false` only matches when the array is completely empty, and `exists=true` passes if the array has 1 or more values - `arrayCase` has no interaction with the `exists` condition. | `true` | +| `range` | `number`, `integer` | [RangeRule](#rangerule-data-structure) | Maximum and minimum value conditions that a numeric field must pass. | `{ min: 5, exclusiveMax: 10 }` Represents an integer from 5-9. | +| `regex` | `string` | String (Regular Expression) | A regular expression pattern that the value must match. | `^NCIT:C\d+$` Value must match an NCI Thesaurus ID. | +| `value` | all | Type of specified fields | Field value matches the value of the specified field. Strings are matched case insensitive. When arrays are matched, the order of their elements is ignored - a field matches this condition if the elements in field are the same elements as in the value match rule. For example, the rule `['abc', 'def']` matches the value `['def', 'abc']` but does not match `['abc', 'def', 'ghi']`. | `some_value`, `[1, 2, 3]` | ### Meta Data Structure From 97f1e9b625148115401c85245948c2c67ce9cf0e Mon Sep 17 00:00:00 2001 From: Jon Eubank Date: Fri, 30 Aug 2024 15:38:48 -0400 Subject: [PATCH 27/27] Cleanup for Beta-version Client Release (#224) * Strictly define package exported functions, types, and files * Standardize package versions to `2.0.0-beta.1` * READMEs cleaned up for NPM publishing * Rename exported function variables to have same grammatic case * Remove redundancy in lectern-server description * README typos and clarifications * correct TypeScript casing typos * Remove unused imports, clean TSDoc comment blocks * Correcting typos and grammatical errors in documentation --------- Co-authored-by: Anders Richardsson <2107110+justincorrigible@users.noreply.github.com> --- README.md | 20 +-- apps/server/README.md | 11 +- apps/server/package.json | 2 +- docs/important-concepts.md | 21 ++- packages/client/README.md | 36 +++-- packages/client/package.json | 4 +- .../src/changeAnalysis/changeAnalyzer.ts | 10 +- packages/client/src/index.ts | 75 ++++++++++- packages/client/src/processing/index.ts | 26 ++-- packages/client/src/rest/index.ts | 127 +++++------------- packages/client/test/changeAnalyzer.spec.ts | 7 +- packages/dictionary/README.md | 14 +- packages/dictionary/package.json | 5 +- packages/validation/README.md | 43 +++++- .../validation/docs/validation-reports.md | 107 +++++++++++++++ packages/validation/package.json | 5 +- 16 files changed, 352 insertions(+), 161 deletions(-) create mode 100644 packages/validation/docs/validation-reports.md diff --git a/README.md b/README.md index eb4d01c..a091ee3 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # Lectern - Data Dictionary Management and Validation -[](http://slack.overture.bio) +[](http://slack.overture.bio) [](https://github.com/overture-stack/lectern/blob/develop/LICENSE) -[](CODE_OF_CONDUCT.md) +[](CODE_OF_CONDUCT.md) Lectern is Overture's Data Dictionary Schema Manager, providing a system for defining Schemas that will validate the structured data collected by an application. The core of Lectern is a web-server application that handles storage and version management of data dictionaries. Lectern data dictionaries are collections of schemas that define the structure of tabular data files (like TSV). This application provides functionality to validate the structure of data dictionaries, maintain a list of dictionary versions, and to compute the difference between dictionary versions. @@ -36,12 +36,12 @@ The modules in the monorepo are organized into two categories: * __packages/__ - Reusable packages shared between applications and other packages. Packages are published to [NPM](https://npmjs.com). * __scripts__ - Utility scripts for use within this repo. -| Component | Package Name | Path | Published Location | Description | -| --------------------------------------------------- | ---------------------------------- | ---------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [Lectern Server](apps/server/README.md) | @overture-stack/lectern-server | apps/server/ | [![Lectern GHCR Packages](https://img.shields.io/badge/GHCR-lectern-brightgreen?style=for-the-badge&logo=github)](https://github.com/overture-stack/lectern/pkgs/container/lectern) | Lectern Server web application. | -| [Lectern Client](packages/client/README.md) | @overture-stack/lectern-client | packages/client | [![Lectern Client NPM Package](https://img.shields.io/npm/v/@overture-stack/lectern-client?color=%23cb3837&style=for-the-badge&logo=npm)](https://www.npmjs.com/package/@overture-stack/lectern-client) | TypeScript Client to interact with Lectern Server and Lectern data dictionaries. This library provides a REST client to assist in fetching data from the Lectern server. It also exposes the functionality from the Lectern Validation library to use a Lectern data dictionary to validate data. | -| [Lectern Dictionary](packages/dictionary/README.md) | | @overture-stack/lectern-dictionary | packages/dictionary/ | [![Lectern Client NPM Package](https://img.shields.io/npm/v/@overture-stack/lectern-dictionary?color=%23cb3837&style=for-the-badge&logo=npm)](https://www.npmjs.com/package/@overture-stack/lectern-dictionary) | Dictionary meta-schema definition, includes TS types, and Zod schemas. This also exports all utilities for getting the diff of two dictionaries. | -| [Lectern Validation](packages/validation/README.md) | @overture-stack/lectern-validation | packages/validation/ | [![Lectern Validation NPM Package](https://img.shields.io/npm/v/@overture-stack/lectern-client?color=%23cb3837&style=for-the-badge&logo=npm)](https://www.npmjs.com/package/@overture-stack/lectern-client) | Validate data using Lectern Dictionaries. | +| Component | Package Name | Path | Published Location | Description | +| --------------------------------------------------- | ---------------------------------- | ---------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [Lectern Server](apps/server/README.md) | @overture-stack/lectern-server | apps/server/ | [![Lectern GHCR Packages](https://img.shields.io/badge/GHCR-lectern-brightgreen?style=for-the-badge&logo=github)](https://github.com/overture-stack/lectern/pkgs/container/lectern) | Lectern Server web application. | +| [Lectern Client](packages/client/README.md) | @overture-stack/lectern-client | packages/client | [![Lectern Client NPM Package](https://img.shields.io/npm/v/@overture-stack/lectern-client?color=%23cb3837&style=for-the-badge&logo=npm)](https://www.npmjs.com/package/@overture-stack/lectern-client) | TypeScript Client to interact with Lectern Server and Lectern data dictionaries. This library provides a REST client to assist in fetching data from the Lectern server. It also exposes the functionality from the Lectern Validation library to use a Lectern data dictionary to validate data. | +| [Lectern Dictionary](packages/dictionary/README.md) | | @overture-stack/lectern-dictionary | [![Lectern Client NPM Package](https://img.shields.io/npm/v/@overture-stack/lectern-dictionary?color=%23cb3837&style=for-the-badge&logo=npm)](https://www.npmjs.com/package/@overture-stack/lectern-dictionary) | Dictionary meta-schema definition, includes TS types, and Zod schemas. This also exports all utilities for getting the diff of two dictionaries. | +| [Lectern Validation](packages/validation/README.md) | @overture-stack/lectern-validation | packages/validation/ | [![Lectern Validation NPM Package](https://img.shields.io/npm/v/@overture-stack/lectern-validation?color=%23cb3837&style=for-the-badge&logo=npm)](https://www.npmjs.com/package/@overture-stack/lectern-client) | Validate data using Lectern Dictionaries. | ## Developer Instructions @@ -49,9 +49,9 @@ You can install all dependencies for the entire repo from the root (as defined `pnpm install` -Using `nx` will ensure all local dependencies are built, in the correct sequence, when building, running, or testing any of the applications and packages in the repo. To run a package.json script from any module - after installing dependencies - use a command of the form `pnpm nx