diff --git a/README.md b/README.md index c1cc124..f03434b 100644 --- a/README.md +++ b/README.md @@ -36,11 +36,11 @@ The modules in the monorepo are organized into three categories: * __libraries/__ - Interal modules shared between other apps, libraries, and packages. * __packages/__ - Packages published to [NPM](https://npmjs.com) meant to be imported into other TypeScript applications. -| Component | Type | Package Name | Path | Published Location | Description | -| ----------------------------------- | ----------- | ------------------------------ | --------------------- | ------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------- | -| [Lectern Server](apps/server/README.md) | Application | server | apps/server/ | [GHCR](https://github.com/overture-stack/lectern/pkgs/container/lectern) | Lectern Server web application. | -| [Lectern Client](https://github.com/overture-stack/js-lectern-client) | Package | @overture-stack/js-lectern-client | | [NPM](https://www.npmjs.com/package/@overturebio-stack/lectern-client) | TypeScript Client to interact with Lectern Server and perform data validation. | -| [common](libraries/common/README.md) | Library | common | libraries/common/ | N/A | Non-specific but commonly reusable utilities. Includes shared Error classes. | +| Component | Type | Package Name | Path | Published Location | Description | +| -------------------------------------------- | ----------- | ------------------------------ | --------------------- | ------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [Lectern Server](apps/server/README.md) | Application | server | apps/server/ | [GHCR](https://github.com/overture-stack/lectern/pkgs/container/lectern) | Lectern Server web application. | +| [Lectern Client](packages/client/README.md) | Package | @overture-stack/lectern-client | packages/client | [NPM](https://www.npmjs.com/package/@overturebio-stack/lectern-client) | TypeScript Client to interact with Lectern Server and perform data validation. | +| [common](libraries/common/README.md) | Library | common | libraries/common/ | N/A | Non-specific but commonly reusable utilities. Includes shared Error classes. | | [dictionary](libraries/dictionary/README.md) | Library | dictionary | libraries/dictionary/ | N/A | Dictionary meta-schema definition, includes TS types, and Zod schemas. This also exports all utilities for getting the diff of two dictionaries, and for validating data records with a Dictionary. | ## Developer Instructions diff --git a/apps/server/test/integration/dictionaryRoutes.spec.ts b/apps/server/test/integration/dictionaryRoutes.spec.ts index 93563b0..2e9c89e 100644 --- a/apps/server/test/integration/dictionaryRoutes.spec.ts +++ b/apps/server/test/integration/dictionaryRoutes.spec.ts @@ -262,7 +262,7 @@ describe('Dictionary Routes', () => { }); it('Should get a single dictionary by name and version', (done: Mocha.Done) => { - const name = 'Test Dictionary'; + const name = 'test_dictionary'; chai .request(app) .get(`/dictionaries/?name=${name}&version=${testVersion}`) diff --git a/apps/server/test/integration/fixtures/createDictionary.json b/apps/server/test/integration/fixtures/createDictionary.json index 858a96d..529cd67 100644 --- a/apps/server/test/integration/fixtures/createDictionary.json +++ b/apps/server/test/integration/fixtures/createDictionary.json @@ -1,5 +1,5 @@ { - "name": "Test Dictionary", + "name": "test_dictionary", "version": "1.2", "schemas": [ { diff --git a/apps/server/test/integration/fixtures/createKeyValue.json b/apps/server/test/integration/fixtures/createKeyValue.json index 89b666c..4832ad6 100644 --- a/apps/server/test/integration/fixtures/createKeyValue.json +++ b/apps/server/test/integration/fixtures/createKeyValue.json @@ -1,5 +1,5 @@ { - "name": "Key Value Meta", + "name": "key_value_meta", "version": "1.0", "references": { "enums": { diff --git a/apps/server/test/integration/fixtures/createKeyValueBad.json b/apps/server/test/integration/fixtures/createKeyValueBad.json index 6b7cb4b..692d7b0 100644 --- a/apps/server/test/integration/fixtures/createKeyValueBad.json +++ b/apps/server/test/integration/fixtures/createKeyValueBad.json @@ -1,5 +1,5 @@ { - "name": "Key Value Meta", + "name": "key_value_meta", "version": "1.1", "schemas": [ { diff --git a/apps/server/test/integration/fixtures/createKeyValueBadReferenceFormat.json b/apps/server/test/integration/fixtures/createKeyValueBadReferenceFormat.json index 1d28c67..7d78562 100644 --- a/apps/server/test/integration/fixtures/createKeyValueBadReferenceFormat.json +++ b/apps/server/test/integration/fixtures/createKeyValueBadReferenceFormat.json @@ -1,5 +1,5 @@ { - "name": "Key Value Meta", + "name": "key_value_meta", "version": "1.0", "references": { "enums": { diff --git a/apps/server/test/integration/fixtures/createKeyValueBadReferenceValueType.json b/apps/server/test/integration/fixtures/createKeyValueBadReferenceValueType.json index e8ac312..20571c6 100644 --- a/apps/server/test/integration/fixtures/createKeyValueBadReferenceValueType.json +++ b/apps/server/test/integration/fixtures/createKeyValueBadReferenceValueType.json @@ -1,5 +1,5 @@ { - "name": "Key Value Meta", + "name": "key_value_meta", "version": "1.0", "references": { "scripts": { diff --git a/apps/server/test/integration/fixtures/createKeyValueBadUnknownReference.json b/apps/server/test/integration/fixtures/createKeyValueBadUnknownReference.json index 10a06cc..022232e 100644 --- a/apps/server/test/integration/fixtures/createKeyValueBadUnknownReference.json +++ b/apps/server/test/integration/fixtures/createKeyValueBadUnknownReference.json @@ -1,5 +1,5 @@ { - "name": "Key Value Meta", + "name": "key_value_meta", "version": "1.0", "references": { "enums": { diff --git a/libraries/common/package.json b/libraries/common/package.json index dca1016..866fcd2 100644 --- a/libraries/common/package.json +++ b/libraries/common/package.json @@ -5,9 +5,12 @@ "main": "dist/index.js", "scripts": { "build": "pnpm build:clean && tsc", - "build:clean": "rm -rf dist/ && mkdir dist" + "build:clean": "rimraf -rf dist/ && mkdir dist" }, "keywords": [], "author": "Ontario Institute for Cancer Research", - "license": "AGPL-3.0" + "license": "AGPL-3.0", + "devDependencies": { + "rimraf": "^3.0.2" + } } diff --git a/libraries/common/tsconfig.json b/libraries/common/tsconfig.json index d14a93f..90dcbbd 100644 --- a/libraries/common/tsconfig.json +++ b/libraries/common/tsconfig.json @@ -1,6 +1,6 @@ { "compilerOptions": { - "target": "ES5", + "target": "ESNext", "lib": ["ESNext"], "module": "CommonJS", "moduleResolution": "node", diff --git a/libraries/dictionary/README.md b/libraries/dictionary/README.md index a466b00..0756db8 100644 --- a/libraries/dictionary/README.md +++ b/libraries/dictionary/README.md @@ -1,9 +1,275 @@ # Lectern Dictionary Schema and Utilities +This module defines the structure of Lectern Dictionaries, including providing the TypeScript type definitions to use the dictionary in code and the schemas to validate that a given JSON object is a valid Lectern Dictionary. + +The Lectern Dictionary meta-schema is formally defined in TypeScript and exported as the type `Dictionary` from the file [`./src/types/dictionaryTypes.ts`](./src/types/dictionaryTypes.ts). This definition is created using `Zod` schemas, which are also exported from this package and available for use to confirm a given object is a valid Lectern Dictionary. + +A JSON Schema description of the Lectern Dictionary structure is provided at [`generated/DictionaryMetaSchema.json`](../../generated/DictionaryMetaSchema.json) . + +## Dictionary Structure + +A Lectern Dictionary is a collection of Lectern Schemas. Each schema describes the structure of a TSV file, providing a list of the columns for that file and the data types and restrictions on the content of those columns. + +In addition to schemas, a Lectern Dictionary can contain reference values that can be reused throughout the schema definitions to define property restrictions with shared rules. + +> **Dictionary Structure Example** +> ```json +> { +> "name": "example_dictionary", +> "description": "Collection of schemas to demonstrate Lectern functionality", +> "meta": { /* Custom meta data about the dictionary here */ }, +> +> "version": "1.0", +> +> "schemas": [ /* Schemas Here */ ], +> "references": { /* Reference Variables Here */ } +> } +> ``` + +| Property | Type | Required | Description | Example | +| ------------- | -------------------------------------------------------------- | -------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------- | +| `name` | `string` | Required | Display name of the dictionary | `"Example Lectern Dictionary"` | +| `version` | `string`, as a semantic version number `major`.`minor`.`patch` | Required | Version of the dictionary. | `"1.23.4"` | +| `schemas` | `Array<`[`LecternSchema`](#dictionary-schema-structure)`>` | Required | An array containing Lectern Schemas. Minimum of 1 Schema is required. | [Dictionary Schema Structure](#dictionary-schema-structure) | +| `description` | `string` | Optional | Free text description of the schema, for use as a reference for users of the dictionary. This description is not used by Lcetern for dictionary validation. | `"Collection of schemas to demonstrate Lectern functionality"` | +| `meta` | [MetaData](#meta-data-structure) object | Optional | Schema implementor defined fields to capture any additional properties not defined in standard Lectern dictionaries. These properties are not used by Lctern for dictionary validation | `{ "author": "Guy Incognito" }` | +| `references` | [References](#references-structure) object | Optional | Reference values that can be referenced throughout the dictionary. | `{ "customRegex": { "ncitIds": "^NCIT:C\d+$" } }` | + +### Dictionary Schema Structure +> **Dictionary Schema Example** +> ```json +> { +> "name": "example-schema", +> "description": "Demonstrating structure of Lectern Schema", +> "meta": { /* Custom meta data about the schema here */ }, +> +> "fields": [ /* Fields Here */ ] +> } +> ``` + + +| Property | Type | Required | Default | Description | Example | +| ------------- | ------------------------------------------------------- | -------- | ------- | :------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------- | +| `name` | NameString (no whitespace or `.`) | Required | | Name of the schema. This will be used in paths that reference this schema, and for identifying files containing records for this schema. | `"example-schema"` | +| `fields` | `Array<`[`LecterField`](#dictionary-field-structure)`>` | Required | | List of fields contained in this Schema. | [Dictionary Field Structure](#dictionary-field-structure) | +| `description` | `string` | Optional | None | Free text description of the schema, for use as a reference for users of the schema. This description is not used in dictionary validation. | `"Demonstrating structure of Lectern Schema"` | +| `meta` | [`MetaData`](#meta-data-structure) | Optional | None | Schema implementor defined fields to capture any additional properties not defined in standard Lectern schemas. | [Meta Data Structure](#meta-data-structure) | + +### Dictionary Field Structure +> **Example Dictionary Field Definition** +> ```json +> { +> "name": "example_field", +> "description": "Shows a string field with a required restriction", +> "meta": { /* Custom meta data abou the field here */ }, +> "isArray": false, +> +> "valueType": "string", +> "restrictions": { +> "required": true +> } +> } +> ``` + +| Property | Required | Default | Type | Description | Example | +| ---------------- | -------- | ---------------------- | ----------------------------------------- | :----------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------- | +| `name` | Required | | NameString (no whitespace or `.`) | Name of the field. This will be used as the header in TSV files in this field's schema, and in any paths referencing this field. | `"example_field` | +| `valueType` | Required | | [Field Data Type](#field-data-types) | Type of value stored in this field | `"string"` | +| `arrayDelimiter` | Optional | `\|` | `string` | Character or string that will be used to split multiple values into an array. The default delimiter is the `\|` character. | | +| `description` | Optional | `""` No value | `string` | Free text description of the field, for use as a reference for users of the schema. This description is not used in dictionary validation. | `"Shows a string field with a required restriction"` | +| `meta` | Optional | Empty object, no value | [`MetaData`](#meta-data-structure) object | Schema implementor defined fields to capture any additional properties not defined in standard Lectern fields. | `{ "displayName": "Example Field" }` | +| `isArray` | Optional | `false` | `boolean` | Type of value stored in this field | | +| `restrictions` | Optional | No Restrictions | `RestrictionsObject` or `Array` | An object containing all validation rules for this field. This can be a single object containing all [restrictions](#field-restrictions) applied to this field or a list of objects whose restrictions will be combined. [Conditional restrictions](#conditional-restrictions) can also be used to apply validation rules based on values of other fields in the record. | `{ "required": true }` | + + + +#### Field Data Types + +| valueType | Description | Examples | +| --------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------- | +| `boolean` | Boolean value, either `true` or `false`. Accepts values with any letter casing, for example `true`, `True`, and `TRUE` will all be interpretted as `true` | `true`, `false` | +| `integer` | Numeric integer value. Will accept positive and negative values (ex. `21` or `-8`) but will reject any decimals (ex. `1.23`) | `21`, `-8` | +| `number` | Numeric value. Will accept any numeric value, including those with decimals. | `1.23`, `-4.567` | +| `string` | String fields. Value can have any length and use any character other than the file delimiter (by default `tab`) or the array delimiter for an array field (by default ` \| `) | `asdf`, `Hello World`, `Another longer example of a string` | + +#### Field Restrictions + +Restrictions on a field are a list of rules that all values for this field must adhere to, these are the list of validations on the contents of each field. Two examples of restrictions are that a value is `required`, and that a value must take a value from a list of available options (`codeList`). The full list of available restrictions are described in the table below. + +The restrictions property of a field can have a value that is either a single restrictions object, or an array with any number of restrictions objects. If an array of restriction objects is provided, each set of restrictions will be applied in turn - for data to be valid, all restrictions in the array must pass. A restrictions object can either contain a set of restrictions from the table below, or be a [conditional restriction](#conditional-restrictions). + +The full list of available restrictions are: + +| Restriction | Used with Field Types | Type | Description | Examples | +| ----------- | ----------------------------- | --------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `codeList` | `integer`, `number`, `string` | Array of type of the field | An array of values of the type matching this field. Data provided for this field must have one of the values in this list. | `["Weak", "Average", "Strong"]` | +| `compare` | all | [ComparedFieldsRule](#comparedfieldsrule-data-structure) object | Enforces that this field has a value based on the provided value in another field. Examples would be to ensure that the two values are not equal, or for numeric values ensure one is greater than the other. | `{ "fields": ["age_at_diagnosis"], "relation": "greaterThanOrEqual" }` Ensure that a field such as `age_at_death` is greater than the provided `age_at_diagnosis` | +| `count` | Array fields of all types | `integer` or [`RangeRule`](#rangerule-data-structure) object | Enfroces the number of entries in an array. Can specify an exact array size, or provide range rules that set maximum and minimum counts. | `7` or `{"min": 5, "max": 10}` | +| `empty` | all | | Requires that no value is provided. This is useful when used on a [conditional restriction](#conditional-restrictions) in order to prevent a value from being given when the condition is true. | n/a | +| `range` | `integer`, `number` | | Uses a [RangeRule](#rangerule-data-structure) object to define minimum and/or maximum values for this field | `{"min": 5}`, `{"exclusiveMax": 50}`, `{"exclusiveMin": 5, "max": 50}` | +| `regex` | `string` | | A regular expression that all values must match. | `^[a-z0-9]+$` | +| `required` | all | | A value must be provided, missing/undefined values will fail validation. Empty strings will not be accepted, though `0` (for `number` and `int` fields) and `false` (for `boolean` fields) are accepted. | `true`, `false` | +| `unique` | all | | When a field has the `unique` restriction, each record must have a distinct value for this field. Uniqueness tests are case sensitive, so `Abc` and `abc` are both distinct values. This restriction is only applied when a collection of records are tested together, ensuring that no two records in that collection share a value. | `true`, `false` | + +#### Conditional Restrictions + +Restrictions can be added with conditions so that the validations are only applied based on the values provided to other fields within a record. + +A conditional restriction uses an if/then/else style syntax: + +The `if` property will be an object containing an array of `conditions` that look at other fields on the same record and apply matching rules to their values. When those field values match the rules in the condition than the condition passes. An optional `case` property can be added to the `if` object that defines how many of the `conditions` have to pass in order for the whole condition block to resolve as `true` - default is `all`, requiring all conditions to be met. + +The `then` object contains the restrictions that will be applied when the `if` condition is `true`, and the `else` condition contains restrictions to apply when the `if` condition is `false`. The `then` property is required but using an `else` property is optional. + +| Property | Required | Default | Type | Description | Example | +| -------- | -------- | ----------------------------- | --------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | +| `if` | Required | | `RequirementsConditions` | Contains the conditional cases that will be checked before applying this object's restrictions. This object contains a list of `conditions` and a `case` that indicates how many of the conditions need to be found `true` for the entire conditions block to be considered `true`. The case options are `any`, `all`, and `none`, with `all` being default (if case is not provided). | `{ "conditions": [ { "field": "another_field", "match": { "value": "Some Value" }} ], "case": "all" }` | +| `then` | Required | | `RestrictionsObject` or `Array` | The restriction rules to apply when the `if` condition is found to be `true`. | `{ "required": true}` | +| `else` | Optional | Empty object, no restrictions | `RestrictionsObject` or `Array` | The restriction rules to apply when the `if` condition is found to be `false`. | `{ "empty": true}` | + +```json +{ + "if": { + "conditions": [ /* Restriction conditions */ ], + "case": "all" + }, + "then": {/* Restrictons */} OR [ /* Restrictions objects (restriction values or nested conditional restrictions */ ], + "else": {/* Restrictons */} OR [ /* Restrictions objects (restriction values or nested conditional restrictions */ ] +} +``` + +##### Conditions Structure + +A requirement condition is defined by providing a field name or list of field names from this schema, and the matching rules that satisfy this condition. If multiple field names are provided, a `case` property can be added to specify how many of their values must pass the matching rules (`all`, `any`, or `none` of them). + +| Property | Required | Default | Type | Description | Example | +| ---------------- | -------- | ------- | -------------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------- | +| `fields` | Required | | `Array` | Names of fields from the same schema. This match rule will be applied to all fields listed - see `case` to determine the rules for how many of these fields must match. All specified fields must store values of the same type. | `["some_field"]` | +| `match` | Required | | `MatchRules` object | Matching rules for the values of the `fields`. All rules included in this object will be tested and all must be pass - this is not affected by the `case` property. [Conditional Match Rules](#conditional-match-rules) | `{ "value": "Hello World" }` | +| `arrayFieldCase` | Optional | `all` | `all`, `any`, `none` | When a specified field is an array type, the `arrayFieldCase` dictates how many of the values in the array must pass the matching rules. `all` requires all values in the array to pass the matching rule. `any` requires at least one value in the array to match. `non` requires that none of the values in the array match. | `any` | +| `case` | Optional | `all` | `all`, `any`, `none` | Defines how many of the listed `fields` must have a value that matches the `match` rules. `all` requires all fields values to have matching values. `any` requires at least one field to have a matching value. `none` requires that there none of the specified fields have values that match. | `any` | + +> **Example Conditional Restriction**: match single value +> +> Condition where `shirt_size` is `Small` +> ```json +> { +> "fields": ["shirt_size"], +> "match": { +> "value": "Small" +> } +> } +> ``` + +> **Example Conditional Restriction**: match value from list +> +> Condition where `shirt_size` is any value in a list (`Medium` or `Large`) +> ```json +> { +> "fields": ["shirt_size"], +> "match": { +> "codeList": ["Medium", "Large`"] +> } +> } +> ``` + +##### Conditional Match Rules +| Property | Used with Field Types | Type | Description | Example | +| ---------- | --------------------- | -------------------------------------------------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | +| `codeList` | all | Array of type of specified fields | A list of values that the field could match. This rule passes when the specified field's value can be found in this list. | `["value_one", "value_two"]` | +| `compare` | all | [ComparedFieldsRule](#comparedfieldsrule-data-structure) | Compare the value of the specified fields to values of another field (or set of fields). This can be configured to check if the fields match, dont match, and for numeric fields can check if the field is Greater or Lesser than. | `{ "fields": ["compared_to_field"], "relation": "equal" }` | +| `count` | Array type fields | Integer, or [RangeRule](#rangerule-data-structure) | Matches the number of values in an array field. This condition can be provided as a number, in which case this condition matches if the array is that exact length. This condition can be provided as a Range object as well, in which case it will match if the number of elements in the array pass the minimum and maximum conditions provided in the condition. | `2` - Field must have exactly 2 elements.
`{ max: 10 }` - Field must have no more than 10 items. | +| `exists` | all | Boolean | This condition requires a field to either have a value or have no value. When the `exists` condition is set to `true`, the field must have a value. When `exists` is sdet to `false`, the field must have no value. For array fields, `exists=false` only matches when the array is completely empty, and `exists=true` passes if the array has 1 or more values - `arrayCase` has no interaction with the `exists` condition. | `true` | +| `range` | `number`, `integer` | [RangeRule](#rangerule-data-structure) | Maximum and minimum value conditions that a numeric field must pass. | `{ min: 5, exclusiveMax: 10 }` Represents an integer from 5-9. | +| `regex` | `string` | String (Regular Expression) | A regular expression pattern that the value must match. | `^NCIT:C\d+$` Value must match an NCI Thesaurus ID. | +| `value` | all | Type of specified fields | Field value exactly matches the value of the specified field. Strings are matched case sensitive. | `some_value` | + +### Meta Data Structure + +> **Meta Example** +> ```json +> { +> "displayName": "Nicely Formatted Name", +> "externalReferenceId": "ABCD:1234", +> "exampleBooleanPropery": true, +> "exampleNumericProperty": 123 +> } +> ``` + +A `meta` object is available to allow the dictionary creator to add custom properties to the Lectern Dictionary. The `meta` property is available to all Dictionary, Schema, and Field objects. Providing a `meta` value is optional. If provided the `meta` value is a JSON object. There are no restrictions on the field names that can be added to the `meta` object other than they must be valid JSON. The values for properties of the `meta` can either be another nested meta object, or are one of the allowed value types: + - `string` + - `number` + - `boolean` + - `Array` + - `Array` + +### References Structure + +References are defined at the dictionary level so they can be reused across schemas. References can be used to store values that can be used in `meta` or `restrictions` + +#### Using References +Reference variables can be used in a `meta` object or a `restrictions` object as either a restriction value or a conditional match value. + +To use a reference, replace the value in the value of the meta or restriction property with a string containing a `ReferenceTag`. A `ReferenceTags` + +### RangeRule Data Structure + +> **RangeRule Example** +> ```json +> { +> "min": 5, +> "exclusiveMax": 10 +> } +> ``` + +`RangeRule` objects are used to define restrictions and conditions where a numeric minimum or maximum needs to be defined. This object must define at least 1 property (ie. could define a minimum but not maximum, or vice-versa). + +There is an inclusive and an exclusive version of the minimum and maximum properties. `min` and `max` are _inclusive_, and the alternate form `exclusiveMin` and `exclusiveMax` are _exclusive_. By example, `{ "min":5 }` allows the value `5` and greater, while `{ "exclusiveMin": 5 }` allows only values greater than `5` but not `5` itself. + +A `RangeRule` cannot include but an inclusive and exclusive version of min, or of max (ie. it cannot have `min` and `exclusiveMin`.) + +| Property | Description | +| -------------- | :---------------------------------------------------------------- | +| `exclusiveMax` | Allows values less than this value, but not this value itself. | +| `exclusiveMin` | Allows values greater than this value, but not this value itself. | +| `max` | Allows this value and values lesser than this value. | +| `min` | Allows this value and values greater than this value. | + +### ComparedFieldsRule Data Structure + +> **ComparedFieldsRule** Example +> +> ```json +> { +> "fields": "some_field", +> "relation": "equal", +> } +> ``` + +| Property | Required | Default | Type | Description | +| ---------- | -------- | -------------------- | ---------------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `fields` | Required | | `string` or `Array` | The field(s) that the values of will be compared to. These fields will be refered to throughout this section as _compared to_ fields. All these fields need to be the same type as the field(s) they will be compared to. | +| `relation` | Required | | `equal`, `notEqual`, `contains`, `containedIn`, `greaterThan`, `greaterThanOrEqual`, `lesserThan`, `lesserThanOrEqual` | The relation between the values of the test field and the compared to fields. See [ComparedFieldsRule Relations](#comparedfieldsrule-relations). | +| `case` | Optional | `all`, `any`, `none` | MatchCase (RangeRule or one of: `all`, `any`, `none`) | How many of the _compared to_ fields must pass the comparison for this rule to pass. | + +#### ComparedFieldsRule Relations + +| Relation Value | Allowable Field Types | Description | +| ------------------------ | --------------------- | :--------------------------------------------------------------------------------------------------------- | +| **`equal`**: | all | Checks that the current field and the comapred field(s) have the same value | +| **`notEqual`**: | all | Checks that the current field and the comapred field(s) do not have the same value | +| **`contains`** | `string` | Checks that the value of the current field completely contains the value of the compared field(s) | +| **`containedIn`** | `string` | Checks that the value of the current field is completely contained in the value of the compared field(s) | +| **`greaterThan`** | `number`, `integer` | Checks that the value of the current field is greater than (exclusive) the value of the compared field(s). | +| **`greaterThanOrEqual`** | `number`, `integer` | Checks that the value of the current field is greater than or equal to the value of the compared field(s). | +| **`lesserThan`** | `number`, `integer` | Checks that the value of the current field is lesser than (exclusive) the value of the compared field(s). | +| **`lesserThanOrEqual`** | `number`, `integer` | Checks that the value of the current field is lesser than or equal to the value of the compared field(s). | + + ## Schema Definition for Lectern Developers The Lectern server uses a TypeScript native schema generated with [Zod](https://zod.dev/) that can be found in the [`./src/types/dictionaryTypes.ts`](./src/types/dictionaryTypes.ts) file. There is a custom script `npm run generate` that will regenerate the content in the `DictionaryMetaSchema.json`. The JSON Schema file is generated thanks to the libary [zod-to-JSON Schema](https://www.npmjs.com/package/zod-to-JSON Schema). Whenever changes are made to the Zod Schemas in `./src/types` the generation script needs to be re-run and the updated Dictionary Meta Schema committed to the repository. -If the generation script needs updating, it can be found at [`./scripts/buildMetaSchema.ts`](./scripts/buildMetaSchema.ts). +If the generation script needs updating, it can be found at [`./scripts/buildMetaSchema.ts`](./scripts/buildMetaSchema.ts). \ No newline at end of file diff --git a/libraries/dictionary/package.json b/libraries/dictionary/package.json index 6223256..6b6c9cf 100644 --- a/libraries/dictionary/package.json +++ b/libraries/dictionary/package.json @@ -5,7 +5,7 @@ "main": "dist/index.js", "scripts": { "build": "pnpm build:clean && tsc", - "build:clean": "rm -rf dist/ && mkdir dist", + "build:clean": "rimraf -rf dist/ && mkdir dist", "test": "nyc mocha" }, "keywords": [], @@ -18,6 +18,7 @@ "zod": "^3.21.4" }, "devDependencies": { - "@types/lodash": "^4.14.195" + "@types/lodash": "^4.14.195", + "rimraf": "^3.0.2" } } diff --git a/libraries/dictionary/src/types/dbTypes.ts b/libraries/dictionary/src/types/dbTypes.ts index f198aa2..6e76be6 100644 --- a/libraries/dictionary/src/types/dbTypes.ts +++ b/libraries/dictionary/src/types/dbTypes.ts @@ -1,5 +1,5 @@ import { z as zod } from 'zod'; -import { DictionaryBase } from './dictionaryTypes'; +import { DictionaryBase } from './dictionary'; /** * A Dictionary stored in the DB is represented as a document and gets an `_id` property diff --git a/libraries/dictionary/src/types/dictionary/commonDictionaryTypes.ts b/libraries/dictionary/src/types/dictionary/commonDictionaryTypes.ts new file mode 100644 index 0000000..2eb8518 --- /dev/null +++ b/libraries/dictionary/src/types/dictionary/commonDictionaryTypes.ts @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { z as zod } from 'zod'; + +export const NameString = zod + .string() + .trim() + .min(1, 'Name fields cannot be empty.') + .regex(RegExp('^[^\\s.]+$'), 'Name fields cannot contain `.` or whitespace characters.'); +//TODO: names should probably only allow a subset of characters, and have a standard and separate field defined for name display value +export type NameString = zod.infer; + +export const Integer = zod.number().int(); diff --git a/libraries/dictionary/src/types/dictionary/dictionaryTypes.ts b/libraries/dictionary/src/types/dictionary/dictionaryTypes.ts new file mode 100644 index 0000000..ba68c2b --- /dev/null +++ b/libraries/dictionary/src/types/dictionary/dictionaryTypes.ts @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { z as zod } from 'zod'; +import { NameString } from './commonDictionaryTypes'; +import { DictionaryMeta } from './metaTypes'; +import { References } from './referenceTypes'; +import { Schema } from './schemaTypes'; +import allUnique from '../../utils/allUnique'; + +export const VersionString = zod.string().regex(RegExp('^[0-9]+.[0-9]+$')); +// TODO: Semantic Versioning version string, reference: https://gist.github.com/jhorsman/62eeea161a13b80e39f5249281e17c39 +// update requires dictionary service changes, leaving like this for now +// .regex(RegExp('^([0-9]+).([0-9]+).([0-9]+)(?:-([0-9A-Za-z-]+(?:.[0-9A-Za-z-]+)*))?(?:+[0-9A-Za-z-]+)?$')); + +// Dictionary Base is the dictionary object types only, no refinements enforcing the restriction rules. +// This is needed to use for the base of the dbType DictionaryDocument +export const DictionaryBase = zod + .object({ + name: NameString, + description: zod.string().optional(), + meta: DictionaryMeta.optional(), + references: References.optional(), + schemas: zod.array(Schema).min(1), + version: VersionString, + }) + .strict(); +export const Dictionary = DictionaryBase.refine( + (dictionary) => allUnique(dictionary.schemas.map((schema) => schema.name)), + // TODO: Can improve uniqueness check error by providing list of duplicate field names. + 'All schemas in the dictionary must have a unique name.', +).superRefine((dictionary, ctx) => { + /** + * Enforce schema foreignKey restrictions: + * 1. make sure that the foreign schema referenced in all provided foreignKey restrictions have matching schemas in the dictionary + * 2. make sure all foreign fields listed in foreignKey restrictions have matching fields in the specified foreign schemas + */ + + // List of all schemas and all their fields, for cross reference from the foreignKey checks + const schemaNames = dictionary.schemas.map((schema) => schema.name); + const schemaFieldMap: Record = dictionary.schemas.reduce>( + (acc, schema) => { + const fieldNames = schema.fields.map((field) => field.name); + return { ...acc, [schema.name]: fieldNames }; + }, + {}, + ); + + // Loop through each schema and apply checks if they have a foreignKey restriction + dictionary.schemas.forEach((schema) => { + if (schema.restrictions?.foreignKey) { + // We have a foreign key restriction, now we will now check its properties all reference existing schemas and fields + + schema.restrictions.foreignKey.forEach((fkRestriction) => { + if (schemaNames.includes(fkRestriction.schema)) { + fkRestriction.mappings.forEach((fkMapping) => { + if (!schemaFieldMap[fkRestriction.schema]?.includes(fkMapping.foreign)) { + // foreign field does not exist on foreign schema + ctx.addIssue({ + code: zod.ZodIssueCode.custom, + message: `Schema ForeignKey restriction references a foreign field that does not exist on the foreign schema. The schema '${schema.name}' references the foreign schema named '${fkRestriction.schema}' with foreign field '${fkMapping.foreign}'.`, + }); + } + + // ensure mapped fields are the same type + const localFieldType = schema.fields.find((field) => field.name === fkMapping.local)?.valueType; + const foreignFieldType = dictionary.schemas + .find((s) => s.name === fkRestriction.schema) + ?.fields.find((field) => field.name === fkMapping.foreign)?.valueType; + if (localFieldType && foreignFieldType && localFieldType !== foreignFieldType) { + //dont match undefined === undefined. Missing field validations are output elsewhere so don't check that here. + ctx.addIssue({ + code: zod.ZodIssueCode.custom, + message: `Schema ForeignKey restriction maps two fields of different types: the restriction in schema '${schema}' maps local field '${fkMapping.local}' of type '${localFieldType}' to foreign schema '${fkRestriction.schema}' and field '${fkMapping.foreign}' of type '${foreignFieldType}'.`, + }); + } + }); + } else { + // foreign schema name is not included in this dictionary, add error to context + ctx.addIssue({ + code: zod.ZodIssueCode.custom, + message: `Schema ForeignKey restriction references a schema name that is not included in the dictionary. The schema '${schema.name}' references foreign schema name '${fkRestriction.schema}'.`, + }); + } + }); + } + }); +}); +export type Dictionary = zod.infer; diff --git a/libraries/dictionary/src/types/dictionary/index.ts b/libraries/dictionary/src/types/dictionary/index.ts new file mode 100644 index 0000000..78b3177 --- /dev/null +++ b/libraries/dictionary/src/types/dictionary/index.ts @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +export * from './commonDictionaryTypes'; +export * from './dictionaryTypes'; +export * from './metaTypes'; +export * from './referenceTypes'; +export * from './schemaFieldTypes'; +export * from './schemaTypes'; diff --git a/libraries/dictionary/src/types/dictionary/metaTypes.ts b/libraries/dictionary/src/types/dictionary/metaTypes.ts new file mode 100644 index 0000000..58845a6 --- /dev/null +++ b/libraries/dictionary/src/types/dictionary/metaTypes.ts @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { z as zod } from 'zod'; + +// Meta accepts as values only strings, numbers, booleans, arrays of numbers or arrays of strings +// Another Meta object can be nested inside a Meta property +export const DictionaryMetaValue = zod.union([ + zod.string(), + zod.number(), + zod.boolean(), + zod.array(zod.string()), + zod.array(zod.number()), +]); +export type DictionaryMetaValue = zod.infer; +export type DictionaryMeta = { [key: string]: DictionaryMetaValue | DictionaryMeta }; +export const DictionaryMeta: zod.ZodType = zod.record( + zod.union([DictionaryMetaValue, zod.lazy(() => DictionaryMeta)]), +); diff --git a/libraries/dictionary/src/types/referenceTypes.ts b/libraries/dictionary/src/types/dictionary/referenceTypes.ts similarity index 100% rename from libraries/dictionary/src/types/referenceTypes.ts rename to libraries/dictionary/src/types/dictionary/referenceTypes.ts diff --git a/libraries/dictionary/src/types/dictionary/schemaFieldTypes.ts b/libraries/dictionary/src/types/dictionary/schemaFieldTypes.ts new file mode 100644 index 0000000..904e44d --- /dev/null +++ b/libraries/dictionary/src/types/dictionary/schemaFieldTypes.ts @@ -0,0 +1,191 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { z as zod } from 'zod'; +import { Integer, NameString } from './commonDictionaryTypes'; +import { ReferenceTag } from './referenceTypes'; +import { DictionaryMeta } from './metaTypes'; + +export const SchemaFieldValueType = zod.enum(['string', 'integer', 'number', 'boolean']); +export type SchemaFieldValueType = zod.infer; + +/* ************ * + * Restrictions * + * ************ */ +export const RestrictionScript = zod.array(zod.string().or(ReferenceTag)).min(1); //TODO: script formatting validation +export type RestrictionScript = zod.infer; +export const RestrictionNumberRange = zod + .object({ + exclusiveMax: zod.number().optional(), + exclusiveMin: zod.number().optional(), + max: zod.number().optional(), + min: zod.number().optional(), + }) + .refine( + (data) => + data.exclusiveMax !== undefined || + data.max !== undefined || + data.exclusiveMin !== undefined || + data.min !== undefined, + 'Range restriction requires one of `exclusiveMax`, `exclusiveMin`, `max` or `min`.', + ) + .refine( + (data) => !(data.exclusiveMin !== undefined && data.min !== undefined), + 'Range restriction cannot have both `exclusiveMin` and `min`.', + ) + .refine( + (data) => !(data.exclusiveMax !== undefined && data.max !== undefined), + 'Range restriction cannot have both `exclusiveMax` and `max`.', + ); + +export const RestrictionIntegerRange = zod + .object({ + exclusiveMax: Integer.optional(), + exclusiveMin: Integer.optional(), + max: Integer.optional(), + min: Integer.optional(), + }) + .refine( + (data) => + data.exclusiveMax !== undefined || + data.max !== undefined || + data.exclusiveMin !== undefined || + data.min !== undefined, + 'Range restriction requires one of `exclusiveMax`, `exclusiveMin`, `max` or `min`.', + ) + .refine( + (data) => !(data.exclusiveMin !== undefined && data.min !== undefined), + 'Range restriction cannot have both `exclusiveMin` and `min`.', + ) + .refine( + (data) => !(data.exclusiveMax !== undefined && data.max !== undefined), + 'Range restriction cannot have both `exclusiveMax` and `max`.', + ); +export type RestrictionRange = zod.infer; + +export const RestrictionRegex = zod.string().superRefine((data, context) => { + try { + // Attempt to build regexp from the value + RegExp(data); + } catch (e) { + // Thrown error creating regex, so we add validation issue. + const errorMessage = e instanceof Error ? e.message : `${e}`; + context.addIssue({ + code: zod.ZodIssueCode.custom, + message: `Error converting expression to Regex: ${errorMessage}`, + }); + } +}); +export type RestrictionRegex = zod.infer; + +/* ****************************** * + * Field Type Restriction Objects * + * ****************************** */ +export const StringFieldRestrictions = zod + .object({ + codeList: zod.union([zod.string(), ReferenceTag]).array().min(1).or(ReferenceTag), + required: zod.boolean(), + script: RestrictionScript.or(ReferenceTag), + regex: RestrictionRegex.or(ReferenceTag), + unique: zod.boolean(), + }) + .partial(); +export type StringFieldRestrictions = zod.infer; + +export const NumberFieldRestrictions = zod + .object({ + codeList: zod.array(zod.number()).min(1).or(ReferenceTag), + required: zod.boolean(), + script: RestrictionScript.or(ReferenceTag), + range: RestrictionNumberRange, + unique: zod.boolean(), + }) + .partial(); +export type NumberFieldRestrictions = zod.infer; + +export const IntegerFieldRestrictions = zod + .object({ + codeList: zod.array(Integer).min(1).or(ReferenceTag), + required: zod.boolean(), + script: RestrictionScript.or(ReferenceTag), + range: RestrictionIntegerRange, + unique: zod.boolean(), + }) + .partial(); +export type IntegerFieldRestrictions = zod.infer; + +export const BooleanFieldRestrictions = zod + .object({ required: zod.boolean(), script: RestrictionScript.or(ReferenceTag), unique: zod.boolean() }) + .partial(); +export type BooleanFieldRestrictions = zod.infer; + +/* ***************** * + * Field Definitions * + * ***************** */ +export const SchemaFieldBase = zod + .object({ + name: NameString, + description: zod.string().optional(), + isArray: zod.boolean().optional(), + meta: DictionaryMeta.optional(), + }) + .strict(); +export type SchemaFieldBase = zod.infer; + +export const SchemaStringField = SchemaFieldBase.merge( + zod.object({ + valueType: zod.literal(SchemaFieldValueType.Values.string), + restrictions: StringFieldRestrictions.optional(), + }), +).strict(); +export type SchemaStringField = zod.infer; + +export const SchemaNumberField = SchemaFieldBase.merge( + zod.object({ + valueType: zod.literal(SchemaFieldValueType.Values.number), + restrictions: NumberFieldRestrictions.optional(), + }), +).strict(); +export type SchemaNumberField = zod.infer; + +export const SchemaIntegerField = SchemaFieldBase.merge( + zod.object({ + valueType: zod.literal(SchemaFieldValueType.Values.integer), + restrictions: IntegerFieldRestrictions.optional(), + }), +).strict(); +export type SchemaIntegerField = zod.infer; + +export const SchemaBooleanField = SchemaFieldBase.merge( + zod.object({ + valueType: zod.literal(SchemaFieldValueType.Values.boolean), + restrictions: BooleanFieldRestrictions.optional(), + }), +).strict(); +export type SchemaBooleanField = zod.infer; + +export const SchemaField = zod.discriminatedUnion('valueType', [ + SchemaStringField, + SchemaNumberField, + SchemaIntegerField, + SchemaBooleanField, +]); +export type SchemaField = zod.infer; + +export type SchemaRestrictions = SchemaField['restrictions']; diff --git a/libraries/dictionary/src/types/dictionary/schemaTypes.ts b/libraries/dictionary/src/types/dictionary/schemaTypes.ts new file mode 100644 index 0000000..b9713bb --- /dev/null +++ b/libraries/dictionary/src/types/dictionary/schemaTypes.ts @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { z as zod } from 'zod'; +import { NameString } from './commonDictionaryTypes'; +import { DictionaryMeta } from './metaTypes'; +import { SchemaField } from './schemaFieldTypes'; +import allUnique from '../../utils/allUnique'; + +export const ForeignKeyRestrictionMapping = zod.object({ + schema: NameString, + mappings: zod.array( + zod.object({ + local: NameString, + foreign: NameString, + }), + ), +}); + +export const Schema = zod + .object({ + name: NameString, + description: zod.string().optional(), + fields: zod.array(SchemaField).min(1), + meta: DictionaryMeta.optional(), + restrictions: zod + .object({ + foreignKey: zod.array(ForeignKeyRestrictionMapping).min(1), + uniqueKey: zod.array(NameString).min(1), + }) + .partial() + .optional(), + }) + .strict() + .refine( + (schema) => allUnique(schema.fields.map((field) => field.name)), + 'All fields in the schema must have a unique name.', + ) + .refine((schema) => { + if (schema.restrictions && schema.restrictions.uniqueKey) { + return schema.restrictions.uniqueKey + .map((requiredField) => schema.fields.some((field) => field.name === requiredField)) + .every((requiredField) => requiredField); + } else { + return true; + } + }, "A field listed in schema restrictions.uniqueKey is not included the schema's fields") + .superRefine((schema, ctx) => { + const fieldNames = schema.fields.map((field) => field.name); + if (schema.restrictions?.foreignKey) { + schema.restrictions.foreignKey.forEach((fkRestriction) => { + if (fkRestriction.schema === schema.name) { + ctx.addIssue({ + code: zod.ZodIssueCode.custom, + message: `Schema '${schema.name}' has a foreignKey restriction that references itself. ForeignKey restrictions must reference another schema.`, + }); + } else { + fkRestriction.mappings.forEach((fkMapping) => { + if (!fieldNames.includes(fkMapping.local)) { + ctx.addIssue({ + code: zod.ZodIssueCode.custom, + message: `Schema '${schema.name}' has a foreign key restriction with local field '${fkMapping.local}' that does not exist in the schema's fields.`, + }); + } + }); + } + }); + } + }); +export type Schema = zod.infer; diff --git a/libraries/dictionary/src/types/dictionaryTypes.ts b/libraries/dictionary/src/types/dictionaryTypes.ts deleted file mode 100644 index 5d81c00..0000000 --- a/libraries/dictionary/src/types/dictionaryTypes.ts +++ /dev/null @@ -1,362 +0,0 @@ -/* - * Copyright (c) 2023 The Ontario Institute for Cancer Research. All rights reserved - * - * This program and the accompanying materials are made available under the terms of - * the GNU Affero General Public License v3.0. You should have received a copy of the - * GNU Affero General Public License along with this program. - * If not, see . - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT - * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER - * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -import { z as zod } from 'zod'; -import allUnique from '../utils/allUnique'; -import { ReferenceTag, References } from './referenceTypes'; - -export const NameString = zod - .string() - .min(1, 'Name fields cannot be empty.') - .regex(RegExp('^[^.]+$'), 'Name fields cannot have `.` characters.'); -export type NameString = zod.infer; - -export const Integer = zod.number().int(); - -// Meta accepts as values only strings, numbers, booleans, arrays of numbers or arrays of strings -// Another Meta object can be nested inside a Meta property -export const DictionaryMetaValue = zod.union([ - zod.string(), - zod.number(), - zod.boolean(), - zod.array(zod.string()), - zod.array(zod.number()), -]); -export type DictionaryMetaValue = zod.infer; -export type DictionaryMeta = { [key: string]: DictionaryMetaValue | DictionaryMeta }; -export const DictionaryMeta: zod.ZodType = zod.record( - zod.union([DictionaryMetaValue, zod.lazy(() => DictionaryMeta)]), -); - -export const SchemaFieldValueType = zod.enum(['string', 'integer', 'number', 'boolean']); -export type SchemaFieldValueType = zod.infer; - -/* ************ * - * Restrictions * - * ************ */ -export const RestrictionScript = zod.array(zod.string().or(ReferenceTag)).min(1); //TODO: script formatting validation -export type RestrictionScript = zod.infer; -export const RestrictionNumberRange = zod - .object({ - exclusiveMax: zod.number().optional(), - exclusiveMin: zod.number().optional(), - max: zod.number().optional(), - min: zod.number().optional(), - }) - .refine( - (data) => - data.exclusiveMax !== undefined || - data.max !== undefined || - data.exclusiveMin !== undefined || - data.min !== undefined, - 'Range restriction requires one of `exclusiveMax`, `exclusiveMin`, `max` or `min`.', - ) - .refine( - (data) => !(data.exclusiveMin !== undefined && data.min !== undefined), - 'Range restriction cannot have both `exclusiveMin` and `min`.', - ) - .refine( - (data) => !(data.exclusiveMax !== undefined && data.max !== undefined), - 'Range restriction cannot have both `exclusiveMax` and `max`.', - ); - -export const RestrictionIntegerRange = zod - .object({ - exclusiveMax: Integer.optional(), - exclusiveMin: Integer.optional(), - max: Integer.optional(), - min: Integer.optional(), - }) - .refine( - (data) => - data.exclusiveMax !== undefined || - data.max !== undefined || - data.exclusiveMin !== undefined || - data.min !== undefined, - 'Range restriction requires one of `exclusiveMax`, `exclusiveMin`, `max` or `min`.', - ) - .refine( - (data) => !(data.exclusiveMin !== undefined && data.min !== undefined), - 'Range restriction cannot have both `exclusiveMin` and `min`.', - ) - .refine( - (data) => !(data.exclusiveMax !== undefined && data.max !== undefined), - 'Range restriction cannot have both `exclusiveMax` and `max`.', - ); -export type RestrictionRange = zod.infer; - -export const RestrictionRegex = zod.string().superRefine((data, context) => { - try { - // Attempt to build regexp from the value - RegExp(data); - } catch (e) { - // Thrown error creating regex, so we add validation issue. - const errorMessage = e instanceof Error ? e.message : `${e}`; - context.addIssue({ - code: zod.ZodIssueCode.custom, - message: `Error converting expression to Regex: ${errorMessage}`, - }); - } -}); -export type RestrictionRegex = zod.infer; - -/* ****************************** * - * Field Type Restriction Objects * - * ****************************** */ -export const StringFieldRestrictions = zod - .object({ - codeList: zod.union([zod.string(), ReferenceTag]).array().min(1).or(ReferenceTag), - required: zod.boolean(), - script: RestrictionScript.or(ReferenceTag), - regex: RestrictionRegex.or(ReferenceTag), - unique: zod.boolean(), - }) - .partial(); -export type StringFieldRestrictions = zod.infer; - -export const NumberFieldRestrictions = zod - .object({ - codeList: zod.array(zod.number()).min(1).or(ReferenceTag), - required: zod.boolean(), - script: RestrictionScript.or(ReferenceTag), - range: RestrictionNumberRange, - unique: zod.boolean(), - }) - .partial(); -export type NumberFieldRestrictions = zod.infer; - -export const IntegerFieldRestrictions = zod - .object({ - codeList: zod.array(Integer).min(1).or(ReferenceTag), - required: zod.boolean(), - script: RestrictionScript.or(ReferenceTag), - range: RestrictionIntegerRange, - unique: zod.boolean(), - }) - .partial(); -export type IntegerFieldRestrictions = zod.infer; - -export const BooleanFieldRestrictions = zod - .object({ required: zod.boolean(), script: RestrictionScript.or(ReferenceTag), unique: zod.boolean() }) - .partial(); -export type BooleanFieldRestrictions = zod.infer; - -/* ***************** * - * Field Definitions * - * ***************** */ -export const SchemaFieldBase = zod - .object({ - name: NameString, - description: zod.string().optional(), - isArray: zod.boolean().optional(), - meta: DictionaryMeta.optional(), - }) - .strict(); -export type SchemaFieldBase = zod.infer; - -export const SchemaStringField = SchemaFieldBase.merge( - zod.object({ - valueType: zod.literal(SchemaFieldValueType.Values.string), - restrictions: StringFieldRestrictions.optional(), - }), -).strict(); -export type SchemaStringField = zod.infer; - -export const SchemaNumberField = SchemaFieldBase.merge( - zod.object({ - valueType: zod.literal(SchemaFieldValueType.Values.number), - restrictions: NumberFieldRestrictions.optional(), - }), -).strict(); -export type SchemaNumberField = zod.infer; - -export const SchemaIntegerField = SchemaFieldBase.merge( - zod.object({ - valueType: zod.literal(SchemaFieldValueType.Values.integer), - restrictions: IntegerFieldRestrictions.optional(), - }), -).strict(); -export type SchemaIntegerField = zod.infer; - -export const SchemaBooleanField = SchemaFieldBase.merge( - zod.object({ - valueType: zod.literal(SchemaFieldValueType.Values.boolean), - restrictions: BooleanFieldRestrictions.optional(), - }), -).strict(); -export type SchemaBooleanField = zod.infer; - -export const SchemaField = zod.discriminatedUnion('valueType', [ - SchemaStringField, - SchemaNumberField, - SchemaIntegerField, - SchemaBooleanField, -]); -export type SchemaField = zod.infer; - -export type SchemaRestrictions = SchemaField['restrictions']; - -/* ****** * - * Schema * - * ****** */ -export const ForeignKeyRestrictionMapping = zod.object({ - schema: NameString, - mappings: zod.array( - zod.object({ - local: NameString, - foreign: NameString, - }), - ), -}); - -export const Schema = zod - .object({ - name: NameString, - description: zod.string().optional(), - fields: zod.array(SchemaField).min(1), - meta: DictionaryMeta.optional(), - restrictions: zod - .object({ - foreignKey: zod.array(ForeignKeyRestrictionMapping).min(1), - uniqueKey: zod.array(NameString).min(1), - }) - .partial() - .optional(), - }) - .strict() - .refine( - (schema) => allUnique(schema.fields.map((field) => field.name)), - 'All fields in the schema must have a unique name.', - ) - .refine((schema) => { - if (schema.restrictions && schema.restrictions.uniqueKey) { - return schema.restrictions.uniqueKey - .map((requiredField) => schema.fields.some((field) => field.name === requiredField)) - .every((requiredField) => requiredField); - } else { - return true; - } - }, "A field listed in schema restrictions.uniqueKey is not included the schema's fields") - .superRefine((schema, ctx) => { - const fieldNames = schema.fields.map((field) => field.name); - if (schema.restrictions?.foreignKey) { - schema.restrictions.foreignKey.forEach((fkRestriction) => { - if (fkRestriction.schema === schema.name) { - ctx.addIssue({ - code: zod.ZodIssueCode.custom, - message: `Schema '${schema.name}' has a foreignKey restriction that references itself. ForeignKey restrictions must reference another schema.`, - }); - } else { - fkRestriction.mappings.forEach((fkMapping) => { - if (!fieldNames.includes(fkMapping.local)) { - ctx.addIssue({ - code: zod.ZodIssueCode.custom, - message: `Schema '${schema.name}' has a foreign key restriction with local field '${fkMapping.local}' that does not exist in the schema's fields.`, - }); - } - }); - } - }); - } - }); -export type Schema = zod.infer; - -/* ********** * - * Dictionary * - * ********** */ - -export const VersionString = zod.string().regex(RegExp('^[0-9]+.[0-9]+$')); -// TODO: Semantic Versioning version string, reference: https://gist.github.com/jhorsman/62eeea161a13b80e39f5249281e17c39 -// update requires dictionary service changes, leaving like this for now -// .regex(RegExp('^([0-9]+).([0-9]+).([0-9]+)(?:-([0-9A-Za-z-]+(?:.[0-9A-Za-z-]+)*))?(?:+[0-9A-Za-z-]+)?$')); - -// Dictionary Base is the dictionary object types only, no refinements enforcing the restriction rules. -// This is needed to use for the base of the dbType DictionaryDocument -export const DictionaryBase = zod - .object({ - name: zod.string().min(1), - description: zod.string().optional(), - meta: DictionaryMeta.optional(), - references: References.optional(), - schemas: zod.array(Schema).min(1), - version: VersionString, - }) - .strict(); -export const Dictionary = DictionaryBase.refine( - (dictionary) => allUnique(dictionary.schemas.map((schema) => schema.name)), - // TODO: Can improve uniqueness check error by providing list of duplicate field names. - 'All schemas in the dictionary must have a unique name.', -).superRefine((dictionary, ctx) => { - /** - * Enforce schema foreignKey restrictions: - * 1. make sure that the foreign schema referenced in all provided foreignKey restrictions have matching schemas in the dictionary - * 2. make sure all foreign fields listed in foreignKey restrictions have matching fields in the specified foreign schemas - */ - - // List of all schemas and all their fields, for cross reference from the foreignKey checks - const schemaNames = dictionary.schemas.map((schema) => schema.name); - const schemaFieldMap: Record = dictionary.schemas.reduce>( - (acc, schema) => { - const fieldNames = schema.fields.map((field) => field.name); - return { ...acc, [schema.name]: fieldNames }; - }, - {}, - ); - - // Loop through each schema and apply checks if they have a foreignKey restriction - dictionary.schemas.forEach((schema) => { - if (schema.restrictions?.foreignKey) { - // We have a foreign key restriction, now we will now check its properties all reference existing schemas and fields - - schema.restrictions.foreignKey.forEach((fkRestriction) => { - if (schemaNames.includes(fkRestriction.schema)) { - fkRestriction.mappings.forEach((fkMapping) => { - if (!schemaFieldMap[fkRestriction.schema]?.includes(fkMapping.foreign)) { - // foreign field does not exist on foreign schema - ctx.addIssue({ - code: zod.ZodIssueCode.custom, - message: `Schema ForeignKey restriction references a foreign field that does not exist on the foreign schema. The schema '${schema.name}' references the foreign schema named '${fkRestriction.schema}' with foreign field '${fkMapping.foreign}'.`, - }); - } - - // ensure mapped fields are the same type - const localFieldType = schema.fields.find((field) => field.name === fkMapping.local)?.valueType; - const foreignFieldType = dictionary.schemas - .find((s) => s.name === fkRestriction.schema) - ?.fields.find((field) => field.name === fkMapping.foreign)?.valueType; - if (localFieldType && foreignFieldType && localFieldType !== foreignFieldType) { - //dont match undefined === undefined. Missing field validations are output elsewhere so don't check that here. - ctx.addIssue({ - code: zod.ZodIssueCode.custom, - message: `Schema ForeignKey restriction maps two fields of different types: the restriction in schema '${schema}' maps local field '${fkMapping.local}' of type '${localFieldType}' to foreign schema '${fkRestriction.schema}' and field '${fkMapping.foreign}' of type '${foreignFieldType}'.`, - }); - } - }); - } else { - // foreign schema name is not included in this dictionary, add error to context - ctx.addIssue({ - code: zod.ZodIssueCode.custom, - message: `Schema ForeignKey restriction references a schema name that is not included in the dictionary. The schema '${schema.name}' references foreign schema name '${fkRestriction.schema}'.`, - }); - } - }); - } - }); -}); -export type Dictionary = zod.infer; diff --git a/libraries/dictionary/src/types/diffTypes.ts b/libraries/dictionary/src/types/diffTypes.ts index 44ed828..7b6f6f5 100644 --- a/libraries/dictionary/src/types/diffTypes.ts +++ b/libraries/dictionary/src/types/diffTypes.ts @@ -1,5 +1,5 @@ import { z as zod } from 'zod'; -import { SchemaField } from './dictionaryTypes'; +import { SchemaField } from './dictionary'; export const ValueChangeTypeNames = { CREATED: 'created', diff --git a/libraries/dictionary/src/types/index.ts b/libraries/dictionary/src/types/index.ts index 055976f..05ac866 100644 --- a/libraries/dictionary/src/types/index.ts +++ b/libraries/dictionary/src/types/index.ts @@ -1,4 +1,3 @@ export * from './dbTypes'; -export * from './dictionaryTypes'; export * from './diffTypes'; -export * from './referenceTypes'; +export * from './dictionary'; diff --git a/libraries/dictionary/test/dictionaryTypes.spec.ts b/libraries/dictionary/test/dictionaryTypes.spec.ts deleted file mode 100644 index 571e803..0000000 --- a/libraries/dictionary/test/dictionaryTypes.spec.ts +++ /dev/null @@ -1,505 +0,0 @@ -/* - * Copyright (c) 2023 The Ontario Institute for Cancer Research. All rights reserved - * - * This program and the accompanying materials are made available under the terms of - * the GNU Affero General Public License v3.0. You should have received a copy of the - * GNU Affero General Public License along with this program. - * If not, see . - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT - * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER - * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -import { expect } from 'chai'; -import { - BooleanFieldRestrictions, - Dictionary, - DictionaryMeta, - Integer, - IntegerFieldRestrictions, - NameString, - NumberFieldRestrictions, - RestrictionIntegerRange, - RestrictionNumberRange, - Schema, - SchemaField, - StringFieldRestrictions, -} from '../src'; - -describe('Dictionary Types', () => { - describe('NameString', () => { - it("Can't be empty string", () => { - expect(NameString.safeParse('').success).false; - }); - it('Can be string', () => { - expect(NameString.safeParse('any').success).true; - expect(NameString.safeParse('123').success).true; - expect(NameString.safeParse('_').success).true; - // NOTE: if we want to limit the property names we should explicitly declare those reules, right now all characters are valid and the strings dont have to start with a letter - }); - it("Can't contain a `.`", () => { - expect(NameString.safeParse('asdf.asdf').success).false; - expect(NameString.safeParse('.').success).false; - expect(NameString.safeParse('.asdf').success).false; - expect(NameString.safeParse('adsf.').success).false; - expect(NameString.safeParse('\\.').success).false; - }); - }); - - describe('Integer', () => { - it("Can't be float", () => { - expect(Integer.safeParse(1.3).success).false; - expect(Integer.safeParse(2.0000001).success).false; - // Note: float precision issues, if the float resolves to a whole number the value will be accepted. - }); - it("Can't be string, boolean, object, array", () => { - expect(Integer.safeParse('1').success).false; - expect(Integer.safeParse(true).success).false; - expect(Integer.safeParse([1]).success).false; - expect(Integer.safeParse({}).success).false; - expect(Integer.safeParse({ thing: 1 }).success).false; - }); - it('Can be integer', () => { - expect(Integer.safeParse(1).success).true; - expect(Integer.safeParse(0).success).true; - expect(Integer.safeParse(-1).success).true; - expect(Integer.safeParse(1123).success).true; - }); - }); - describe('RangeRestriction', () => { - it("Integer Range Can't have exclusiveMin and Min", () => { - expect(RestrictionIntegerRange.safeParse({ exclusiveMin: 0, min: 0 }).success).false; - expect(RestrictionIntegerRange.safeParse({ min: 0 }).success).true; - expect(RestrictionIntegerRange.safeParse({ exclusiveMin: 0 }).success).true; - }); - it("Integer Range Can't have exclusiveMax and Max", () => { - expect(RestrictionIntegerRange.safeParse({ exclusiveMax: 0, max: 0 }).success).false; - expect(RestrictionIntegerRange.safeParse({ max: 0 }).success).true; - expect(RestrictionIntegerRange.safeParse({ exclusiveMax: 0 }).success).true; - }); - it("Number Range Can't have exclusiveMin and Min", () => { - expect(RestrictionNumberRange.safeParse({ exclusiveMin: 0, min: 0 }).success).false; - expect(RestrictionNumberRange.safeParse({ min: 0 }).success).true; - expect(RestrictionNumberRange.safeParse({ exclusiveMin: 0 }).success).true; - }); - it("Number Range Can't have exclusiveMax and Max", () => { - expect(RestrictionNumberRange.safeParse({ exclusiveMax: 0, max: 0 }).success).false; - expect(RestrictionNumberRange.safeParse({ max: 0 }).success).true; - expect(RestrictionNumberRange.safeParse({ exclusiveMax: 0 }).success).true; - }); - }); - describe('RegexRestriction', () => { - it('Accepts valid regex', () => { - expect(StringFieldRestrictions.safeParse({ regex: '[a-zA-Z]' }).success).true; - expect( - StringFieldRestrictions.safeParse({ - regex: '^({((([WUBRG]|([0-9]|[1-9][0-9]*))(/[WUBRG])?)|(X)|([WUBRG](/[WUBRG])?/[P]))})+$', - }).success, - ).true; - }); - it('Rejects invalid regex', () => { - expect(StringFieldRestrictions.safeParse({ regex: '[' }).success).false; - }); - }); - describe('UniqueRestriction', () => { - it('All fields accept unique restriction', () => { - expect(StringFieldRestrictions.safeParse({ unique: true }).success).true; - expect(NumberFieldRestrictions.safeParse({ unique: true }).success).true; - expect(IntegerFieldRestrictions.safeParse({ unique: true }).success).true; - expect(BooleanFieldRestrictions.safeParse({ unique: true }).success).true; - }); - }); - describe('ScriptRestriction', () => { - it('All fields accept script restriction', () => { - expect(StringFieldRestrictions.safeParse({ script: ['()=>true'] }).success).true; - expect(NumberFieldRestrictions.safeParse({ script: ['()=>true'] }).success).true; - expect(IntegerFieldRestrictions.safeParse({ script: ['()=>true'] }).success).true; - expect(BooleanFieldRestrictions.safeParse({ script: ['()=>true'] }).success).true; - }); - }); - describe('Schema', () => { - it("Can't have repeated field names", () => { - const sharedName = 'schemaName'; - const fieldA: SchemaField = { - name: sharedName, - valueType: 'boolean', - }; - const fieldB: SchemaField = { - name: sharedName, - valueType: 'string', - }; - const schema: Schema = { - name: 'asdf', - fields: [fieldA, fieldB], - }; - expect(Schema.safeParse(schema).success).false; - }); - it('Must have at least one field', () => { - const sharedName = 'schemaName'; - const fieldA: SchemaField = { - name: sharedName, - valueType: 'boolean', - }; - const schemaFail: Schema = { - name: 'asdf', - fields: [], - }; - const schemaPass: Schema = { - name: 'asdf', - fields: [fieldA], - }; - expect(Schema.safeParse(schemaFail).success).false; - expect(Schema.safeParse(schemaPass).success).true; - }); - describe('Schema Restrictions', () => { - it('uniqueKey - Can have restriction', () => { - const sharedName = 'schemaName'; - const fieldA: SchemaField = { - name: sharedName, - valueType: 'boolean', - }; - const schema: Schema = { - name: 'asdf', - fields: [fieldA], - restrictions: { - uniqueKey: [sharedName], - }, - }; - expect(Schema.safeParse(schema).success).true; - }); - it('uniqueKey - Schema must have all fields listed', () => { - const sharedNameA = 'sharedNameA'; - const sharedNameB = 'schemaNameB'; - const fieldPass: SchemaField = { - name: sharedNameA, - valueType: 'boolean', - }; - const fieldFail: SchemaField = { - name: 'qwerty', - valueType: 'string', - }; - - const schemaSharedA: Schema = { - name: 'asdf', - fields: [fieldPass], - restrictions: { - uniqueKey: [sharedNameA], - }, - }; - const schemaFailSingle: Schema = { - name: 'asdf', - fields: [fieldFail], - restrictions: { - uniqueKey: [sharedNameA], - }, - }; - const schemaFailMulti: Schema = { - name: 'asdf', - fields: [fieldPass], - restrictions: { - uniqueKey: [sharedNameA, sharedNameB], - }, - }; - const schemaPassMultipleFields: Schema = { - name: 'asdf', - fields: [fieldPass, fieldFail], - restrictions: { - uniqueKey: [sharedNameA], - }, - }; - expect(Schema.safeParse(schemaSharedA).success).true; - expect(Schema.safeParse(schemaFailSingle).success).false; - expect(Schema.safeParse(schemaFailMulti).success).false; - expect(Schema.safeParse(schemaPassMultipleFields).success).true; - }); - it('foreignKey - Can have restriction', () => { - const foreignFieldName = 'foreignField'; - const foreignSchemaName = 'foreignSchema'; - - const localFieldName = 'localField'; - const localField: SchemaField = { - name: localFieldName, - valueType: 'boolean', - }; - const schema: Schema = { - name: 'localSchema', - fields: [localField], - restrictions: { - foreignKey: [ - { - schema: foreignSchemaName, - mappings: [ - { - local: localFieldName, - foreign: foreignFieldName, - }, - ], - }, - ], - }, - }; - - expect(Schema.safeParse(schema).success).true; - }); - it('foreignKey - Rejects if uses same schema name', () => { - const foreignFieldName = 'foreignField'; - - const schemaName = 'reusedName'; - const localFieldName = 'localField'; - const localField: SchemaField = { - name: localFieldName, - valueType: 'boolean', - }; - const schema: Schema = { - name: schemaName, - fields: [localField], - restrictions: { - foreignKey: [ - { - schema: schemaName, - mappings: [ - { - local: localFieldName, - foreign: foreignFieldName, - }, - ], - }, - ], - }, - }; - expect(Schema.safeParse(schema).success).false; - }); - it('foreignKey - Rejects if local field does not exist', () => { - const foreignFieldName = 'foreignField'; - const foreignSchemaName = 'foreignSchema'; - - const localFieldName = 'localField'; - const localField: SchemaField = { - name: localFieldName, - valueType: 'boolean', - }; - const schema: Schema = { - name: 'localSchema', - fields: [localField], - restrictions: { - foreignKey: [ - { - schema: foreignSchemaName, - mappings: [ - { - local: 'unknownName', - foreign: foreignFieldName, - }, - ], - }, - ], - }, - }; - - expect(Schema.safeParse(schema).success).false; - }); - }); - }); - describe('Dictionary', () => { - it("Can't have repeated schema names", () => { - const sharedName = 'asdf'; - const schemaA: Schema = { - name: sharedName, - fields: [ - { - name: 'aName', - valueType: 'boolean', - }, - ], - }; - const schemaB: Schema = { - name: sharedName, - fields: [ - { - name: 'bName', - valueType: 'boolean', - }, - ], - }; - const dictionary: Dictionary = { - name: 'dictionaryName', - version: '1.0', - schemas: [schemaA, schemaB], - }; - expect(Dictionary.safeParse(dictionary).success).false; - }); - describe('ForeignKey Restrictions', () => { - it('Validates that foreign schema exists in dictionary', () => { - const foreignFieldName = 'foreignField'; - const foreignSchemaName = 'foreignSchema'; - const foreignField: SchemaField = { - name: foreignFieldName, - valueType: 'boolean', - }; - const foreignSchema: Schema = { - name: foreignSchemaName, - fields: [foreignField], - }; - - const localFieldName = 'localField'; - const localField: SchemaField = { - name: localFieldName, - valueType: 'boolean', - }; - const localSchema: Schema = { - name: 'localSchema', - fields: [localField], - restrictions: { - foreignKey: [ - { - schema: foreignSchemaName, - mappings: [ - { - local: localFieldName, - foreign: foreignFieldName, - }, - ], - }, - ], - }, - }; - - // should pass - const dictionaryWithForeignSchema: Dictionary = { - name: 'dictionaryName', - schemas: [localSchema, foreignSchema], - version: '1.0', - }; - expect(Dictionary.safeParse(dictionaryWithForeignSchema).success).true; - - // should fail - const dictionaryWithoutForeignSchema: Dictionary = { - name: 'dictionaryName', - schemas: [localSchema], - version: '1.0', - }; - expect(Dictionary.safeParse(dictionaryWithoutForeignSchema).success).false; - }); - it('Fails when foreign field does not exists in foreign schema', () => { - const foreignSchemaName = 'foreignSchema'; - const foreignField: SchemaField = { - name: 'foreignField', - valueType: 'boolean', - }; - const foreignSchema: Schema = { - name: foreignSchemaName, - fields: [foreignField], - }; - - const localFieldName = 'localField'; - const localField: SchemaField = { - name: localFieldName, - valueType: 'boolean', - }; - const localSchema: Schema = { - name: 'localSchema', - fields: [localField], - restrictions: { - foreignKey: [ - { - schema: foreignSchemaName, - mappings: [ - { - local: localFieldName, - foreign: 'invalidField', - }, - ], - }, - ], - }, - }; - - const dictionary: Dictionary = { - name: 'dictionaryName', - schemas: [localSchema, foreignSchema], - version: '1.0', - }; - expect(Dictionary.safeParse(dictionary).success).false; - }); - it('Fails when mapped between fields of different types', () => { - const foreignSchemaName = 'foreignSchema'; - const foreignFieldName = 'foreignField'; - const foreignField: SchemaField = { - name: foreignFieldName, - valueType: 'string', - }; - const foreignSchema: Schema = { - name: foreignSchemaName, - fields: [foreignField], - }; - - const localFieldName = 'localField'; - const localField: SchemaField = { - name: localFieldName, - valueType: 'boolean', - }; - const localSchema: Schema = { - name: 'localSchema', - fields: [localField], - restrictions: { - foreignKey: [ - { - schema: foreignSchemaName, - mappings: [ - { - local: localFieldName, - foreign: foreignFieldName, - }, - ], - }, - ], - }, - }; - - const dictionary: Dictionary = { - name: 'dictionaryName', - schemas: [localSchema, foreignSchema], - version: '1.0', - }; - expect(Dictionary.safeParse(dictionary).success).false; - }); - }); - }); - describe('Meta', () => { - it('Can accept non-nested values', () => { - const meta = { a: 'string', b: 123, c: true }; - expect(DictionaryMeta.safeParse(meta).success).true; - }); - it('Can accept nested values', () => { - const singleNested = { a: 'string', b: 123, c: true, nested: { d: 'asdf' } }; - const doubleNested = { a: 'string', b: 123, c: true, nested: { d: 'asdf', nested2: { e: 'asdf' } } }; - expect(DictionaryMeta.safeParse(singleNested).success).true; - expect(DictionaryMeta.safeParse(doubleNested).success).true; - }); - it('Can accept arrays of strings', () => { - const meta = { a: 'string', b: 123, c: true, array: ['asdf', 'qwerty'] }; - expect(DictionaryMeta.safeParse(meta).success).true; - }); - it('Can accept arrays of numbers', () => { - const meta = { a: 'string', b: 123, c: true, array: [123, 456, 789] }; - expect(DictionaryMeta.safeParse(meta).success).true; - }); - it('Cannot accept arrays of booleans', () => { - // This constraint feels a bit arbitrary but I can't imagine a clear use case for this. - // Could be changed with discussion - const meta = { a: 'string', b: 123, c: true, array: [true, false, true, false] }; - expect(DictionaryMeta.safeParse(meta).success).false; - }); - it('Cannot accept arrays of mixed types', () => { - const meta = { a: 'string', b: 123, c: true, array: ['asdf', 123] }; - expect(DictionaryMeta.safeParse(meta).success).false; - }); - }); -}); diff --git a/libraries/dictionary/test/types/commonDictionaryTypes.spec.ts b/libraries/dictionary/test/types/commonDictionaryTypes.spec.ts new file mode 100644 index 0000000..db3354c --- /dev/null +++ b/libraries/dictionary/test/types/commonDictionaryTypes.spec.ts @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { expect } from 'chai'; +import { Integer, NameString } from '../../src'; + +describe('Common Dictionary Types', () => { + describe('NameString', () => { + it("Can't be empty string", () => { + expect(NameString.safeParse('').success).false; + }); + it('Can be string', () => { + expect(NameString.safeParse('any').success).true; + expect(NameString.safeParse('123').success).true; + expect(NameString.safeParse('_').success).true; + // NOTE: if we want to limit the property names we should explicitly declare those reules, right now all characters are valid and the strings dont have to start with a letter + }); + it("Can't contain a `.`", () => { + expect(NameString.safeParse('asdf.asdf').success).false; + expect(NameString.safeParse('.').success).false; + expect(NameString.safeParse('.asdf').success).false; + expect(NameString.safeParse('adsf.').success).false; + expect(NameString.safeParse('\\.').success).false; + }); + }); + + describe('Integer', () => { + it("Can't be float", () => { + expect(Integer.safeParse(1.3).success).false; + expect(Integer.safeParse(2.0000001).success).false; + // Note: float precision issues, if the float resolves to a whole number the value will be accepted. + }); + it("Can't be string, boolean, object, array", () => { + expect(Integer.safeParse('1').success).false; + expect(Integer.safeParse(true).success).false; + expect(Integer.safeParse([1]).success).false; + expect(Integer.safeParse({}).success).false; + expect(Integer.safeParse({ thing: 1 }).success).false; + }); + it('Can be integer', () => { + expect(Integer.safeParse(1).success).true; + expect(Integer.safeParse(0).success).true; + expect(Integer.safeParse(-1).success).true; + expect(Integer.safeParse(1123).success).true; + }); + }); +}); diff --git a/libraries/dictionary/test/types/dictionaryTypes.spec.ts b/libraries/dictionary/test/types/dictionaryTypes.spec.ts new file mode 100644 index 0000000..5a060d9 --- /dev/null +++ b/libraries/dictionary/test/types/dictionaryTypes.spec.ts @@ -0,0 +1,188 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { expect } from 'chai'; + +import { Dictionary, Schema, SchemaField } from '../../src'; + +describe('Dictionary Types', () => { + it("Can't have repeated schema names", () => { + const sharedName = 'asdf'; + const schemaA: Schema = { + name: sharedName, + fields: [ + { + name: 'aName', + valueType: 'boolean', + }, + ], + }; + const schemaB: Schema = { + name: sharedName, + fields: [ + { + name: 'bName', + valueType: 'boolean', + }, + ], + }; + const dictionary: Dictionary = { + name: 'dictionaryName', + version: '1.0', + schemas: [schemaA, schemaB], + }; + expect(Dictionary.safeParse(dictionary).success).false; + }); + describe('ForeignKey Restrictions', () => { + it('Validates that foreign schema exists in dictionary', () => { + const foreignFieldName = 'foreignField'; + const foreignSchemaName = 'foreignSchema'; + const foreignField: SchemaField = { + name: foreignFieldName, + valueType: 'boolean', + }; + const foreignSchema: Schema = { + name: foreignSchemaName, + fields: [foreignField], + }; + + const localFieldName = 'localField'; + const localField: SchemaField = { + name: localFieldName, + valueType: 'boolean', + }; + const localSchema: Schema = { + name: 'localSchema', + fields: [localField], + restrictions: { + foreignKey: [ + { + schema: foreignSchemaName, + mappings: [ + { + local: localFieldName, + foreign: foreignFieldName, + }, + ], + }, + ], + }, + }; + + // should pass + const dictionaryWithForeignSchema: Dictionary = { + name: 'dictionaryName', + schemas: [localSchema, foreignSchema], + version: '1.0', + }; + expect(Dictionary.safeParse(dictionaryWithForeignSchema).success).true; + + // should fail + const dictionaryWithoutForeignSchema: Dictionary = { + name: 'dictionaryName', + schemas: [localSchema], + version: '1.0', + }; + expect(Dictionary.safeParse(dictionaryWithoutForeignSchema).success).false; + }); + it('Fails when foreign field does not exists in foreign schema', () => { + const foreignSchemaName = 'foreignSchema'; + const foreignField: SchemaField = { + name: 'foreignField', + valueType: 'boolean', + }; + const foreignSchema: Schema = { + name: foreignSchemaName, + fields: [foreignField], + }; + + const localFieldName = 'localField'; + const localField: SchemaField = { + name: localFieldName, + valueType: 'boolean', + }; + const localSchema: Schema = { + name: 'localSchema', + fields: [localField], + restrictions: { + foreignKey: [ + { + schema: foreignSchemaName, + mappings: [ + { + local: localFieldName, + foreign: 'invalidField', + }, + ], + }, + ], + }, + }; + + const dictionary: Dictionary = { + name: 'dictionaryName', + schemas: [localSchema, foreignSchema], + version: '1.0', + }; + expect(Dictionary.safeParse(dictionary).success).false; + }); + it('Fails when mapped between fields of different types', () => { + const foreignSchemaName = 'foreignSchema'; + const foreignFieldName = 'foreignField'; + const foreignField: SchemaField = { + name: foreignFieldName, + valueType: 'string', + }; + const foreignSchema: Schema = { + name: foreignSchemaName, + fields: [foreignField], + }; + + const localFieldName = 'localField'; + const localField: SchemaField = { + name: localFieldName, + valueType: 'boolean', + }; + const localSchema: Schema = { + name: 'localSchema', + fields: [localField], + restrictions: { + foreignKey: [ + { + schema: foreignSchemaName, + mappings: [ + { + local: localFieldName, + foreign: foreignFieldName, + }, + ], + }, + ], + }, + }; + + const dictionary: Dictionary = { + name: 'dictionaryName', + schemas: [localSchema, foreignSchema], + version: '1.0', + }; + expect(Dictionary.safeParse(dictionary).success).false; + }); + }); +}); diff --git a/libraries/dictionary/test/types/metaTypes.spec.ts b/libraries/dictionary/test/types/metaTypes.spec.ts new file mode 100644 index 0000000..8c3089a --- /dev/null +++ b/libraries/dictionary/test/types/metaTypes.spec.ts @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { expect } from 'chai'; + +import { DictionaryMeta } from '../../src'; + +describe('Meta', () => { + it('Can accept non-nested values', () => { + const meta = { a: 'string', b: 123, c: true }; + expect(DictionaryMeta.safeParse(meta).success).true; + }); + it('Can accept nested values', () => { + const singleNested = { a: 'string', b: 123, c: true, nested: { d: 'asdf' } }; + const doubleNested = { a: 'string', b: 123, c: true, nested: { d: 'asdf', nested2: { e: 'asdf' } } }; + expect(DictionaryMeta.safeParse(singleNested).success).true; + expect(DictionaryMeta.safeParse(doubleNested).success).true; + }); + it('Can accept arrays of strings', () => { + const meta = { a: 'string', b: 123, c: true, array: ['asdf', 'qwerty'] }; + expect(DictionaryMeta.safeParse(meta).success).true; + }); + it('Can accept arrays of numbers', () => { + const meta = { a: 'string', b: 123, c: true, array: [123, 456, 789] }; + expect(DictionaryMeta.safeParse(meta).success).true; + }); + it('Cannot accept arrays of booleans', () => { + // This constraint feels a bit arbitrary but I can't imagine a clear use case for this. + // Could be changed with discussion + const meta = { a: 'string', b: 123, c: true, array: [true, false, true, false] }; + expect(DictionaryMeta.safeParse(meta).success).false; + }); + it('Cannot accept arrays of mixed types', () => { + const meta = { a: 'string', b: 123, c: true, array: ['asdf', 123] }; + expect(DictionaryMeta.safeParse(meta).success).false; + }); +}); diff --git a/libraries/dictionary/test/references.spec.ts b/libraries/dictionary/test/types/references.spec.ts similarity index 68% rename from libraries/dictionary/test/references.spec.ts rename to libraries/dictionary/test/types/references.spec.ts index 823d8ba..41ca57d 100644 --- a/libraries/dictionary/test/references.spec.ts +++ b/libraries/dictionary/test/types/references.spec.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 The Ontario Institute for Cancer Research. All rights reserved + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved * * This program and the accompanying materials are made available under the terms of * the GNU Affero General Public License v3.0. You should have received a copy of the @@ -18,26 +18,26 @@ */ import { expect } from 'chai'; -import { replaceReferences } from '../src/references'; +import { replaceReferences } from '../../src/references'; -import noReferencesSectionInput from './fixtures/references/no_references_section/input'; -import noReferencesSectionOutput from './fixtures/references/no_references_section/output'; -import emptyReferencesInput from './fixtures/references/empty_references_section/input'; -import emptyReferencesOutput from './fixtures/references/empty_references_section/output'; -import simpleReferencesInput from './fixtures/references/simple_references/input'; -import simpleReferencesOutput from './fixtures/references/simple_references/output'; -import codeListReferencesInput from './fixtures/references/codeList_references/input'; -import codeListReferencesOutput from './fixtures/references/codeList_references/output'; -import referencesWithinReferencesInput from './fixtures/references/references_within_references/input'; -import referencesWithinReferencesOutput from './fixtures/references/references_within_references/output'; -import scriptReferencesInput from './fixtures/references/script_references/input'; -import scriptReferencesOutput from './fixtures/references/script_references/output'; -import regexReferencesInput from './fixtures/references/regex_reference/input'; -import regexReferencesOutput from './fixtures/references/regex_reference/output'; -import regexArrayReferencesInput from './fixtures/references/regex_reference/input_with_array'; -import nonExistingReferencesInput from './fixtures/references/non_existing_references/input'; -import cyclicReferencesInput from './fixtures/references/cyclic_references/input'; -import selfReferencesInput from './fixtures/references/self_references/input'; +import noReferencesSectionInput from '../fixtures/references/no_references_section/input'; +import noReferencesSectionOutput from '../fixtures/references/no_references_section/output'; +import emptyReferencesInput from '../fixtures/references/empty_references_section/input'; +import emptyReferencesOutput from '../fixtures/references/empty_references_section/output'; +import simpleReferencesInput from '../fixtures/references/simple_references/input'; +import simpleReferencesOutput from '../fixtures/references/simple_references/output'; +import codeListReferencesInput from '../fixtures/references/codeList_references/input'; +import codeListReferencesOutput from '../fixtures/references/codeList_references/output'; +import referencesWithinReferencesInput from '../fixtures/references/references_within_references/input'; +import referencesWithinReferencesOutput from '../fixtures/references/references_within_references/output'; +import scriptReferencesInput from '../fixtures/references/script_references/input'; +import scriptReferencesOutput from '../fixtures/references/script_references/output'; +import regexReferencesInput from '../fixtures/references/regex_reference/input'; +import regexReferencesOutput from '../fixtures/references/regex_reference/output'; +import regexArrayReferencesInput from '../fixtures/references/regex_reference/input_with_array'; +import nonExistingReferencesInput from '../fixtures/references/non_existing_references/input'; +import cyclicReferencesInput from '../fixtures/references/cyclic_references/input'; +import selfReferencesInput from '../fixtures/references/self_references/input'; describe('Replace References', () => { it('Should return the same original schema if dictionary does not contain a references section', () => { diff --git a/libraries/dictionary/test/types/schemaFieldTypes.spec.ts b/libraries/dictionary/test/types/schemaFieldTypes.spec.ts new file mode 100644 index 0000000..319ad9e --- /dev/null +++ b/libraries/dictionary/test/types/schemaFieldTypes.spec.ts @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { expect } from 'chai'; +import { + BooleanFieldRestrictions, + IntegerFieldRestrictions, + NumberFieldRestrictions, + RestrictionIntegerRange, + RestrictionNumberRange, + StringFieldRestrictions, +} from '../../src'; + +describe('SchemaField Types', () => { + describe('RangeRestriction', () => { + it("Integer Range Can't have exclusiveMin and Min", () => { + expect(RestrictionIntegerRange.safeParse({ exclusiveMin: 0, min: 0 }).success).false; + expect(RestrictionIntegerRange.safeParse({ min: 0 }).success).true; + expect(RestrictionIntegerRange.safeParse({ exclusiveMin: 0 }).success).true; + }); + it("Integer Range Can't have exclusiveMax and Max", () => { + expect(RestrictionIntegerRange.safeParse({ exclusiveMax: 0, max: 0 }).success).false; + expect(RestrictionIntegerRange.safeParse({ max: 0 }).success).true; + expect(RestrictionIntegerRange.safeParse({ exclusiveMax: 0 }).success).true; + }); + it("Number Range Can't have exclusiveMin and Min", () => { + expect(RestrictionNumberRange.safeParse({ exclusiveMin: 0, min: 0 }).success).false; + expect(RestrictionNumberRange.safeParse({ min: 0 }).success).true; + expect(RestrictionNumberRange.safeParse({ exclusiveMin: 0 }).success).true; + }); + it("Number Range Can't have exclusiveMax and Max", () => { + expect(RestrictionNumberRange.safeParse({ exclusiveMax: 0, max: 0 }).success).false; + expect(RestrictionNumberRange.safeParse({ max: 0 }).success).true; + expect(RestrictionNumberRange.safeParse({ exclusiveMax: 0 }).success).true; + }); + }); + describe('RegexRestriction', () => { + it('Accepts valid regex', () => { + expect(StringFieldRestrictions.safeParse({ regex: '[a-zA-Z]' }).success).true; + expect( + StringFieldRestrictions.safeParse({ + regex: '^({((([WUBRG]|([0-9]|[1-9][0-9]*))(/[WUBRG])?)|(X)|([WUBRG](/[WUBRG])?/[P]))})+$', + }).success, + ).true; + }); + it('Rejects invalid regex', () => { + expect(StringFieldRestrictions.safeParse({ regex: '[' }).success).false; + }); + }); + describe('UniqueRestriction', () => { + it('All fields accept unique restriction', () => { + expect(StringFieldRestrictions.safeParse({ unique: true }).success).true; + expect(NumberFieldRestrictions.safeParse({ unique: true }).success).true; + expect(IntegerFieldRestrictions.safeParse({ unique: true }).success).true; + expect(BooleanFieldRestrictions.safeParse({ unique: true }).success).true; + }); + }); + describe('ScriptRestriction', () => { + it('All fields accept script restriction', () => { + expect(StringFieldRestrictions.safeParse({ script: ['()=>true'] }).success).true; + expect(NumberFieldRestrictions.safeParse({ script: ['()=>true'] }).success).true; + expect(IntegerFieldRestrictions.safeParse({ script: ['()=>true'] }).success).true; + expect(BooleanFieldRestrictions.safeParse({ script: ['()=>true'] }).success).true; + }); + }); +}); diff --git a/libraries/dictionary/test/types/schemaTypes.spec.ts b/libraries/dictionary/test/types/schemaTypes.spec.ts new file mode 100644 index 0000000..5d5a3b8 --- /dev/null +++ b/libraries/dictionary/test/types/schemaTypes.spec.ts @@ -0,0 +1,209 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { expect } from 'chai'; +import { Schema, SchemaField } from '../../src'; + +describe('Schema Types', () => { + it("Can't have repeated field names", () => { + const sharedName = 'schemaName'; + const fieldA: SchemaField = { + name: sharedName, + valueType: 'boolean', + }; + const fieldB: SchemaField = { + name: sharedName, + valueType: 'string', + }; + const schema: Schema = { + name: 'asdf', + fields: [fieldA, fieldB], + }; + expect(Schema.safeParse(schema).success).false; + }); + it('Must have at least one field', () => { + const sharedName = 'schemaName'; + const fieldA: SchemaField = { + name: sharedName, + valueType: 'boolean', + }; + const schemaFail: Schema = { + name: 'asdf', + fields: [], + }; + const schemaPass: Schema = { + name: 'asdf', + fields: [fieldA], + }; + expect(Schema.safeParse(schemaFail).success).false; + expect(Schema.safeParse(schemaPass).success).true; + }); + describe('Schema Restrictions', () => { + describe('uniqueKey', () => { + it('Can have restriction', () => { + const sharedName = 'schemaName'; + const fieldA: SchemaField = { + name: sharedName, + valueType: 'boolean', + }; + const schema: Schema = { + name: 'asdf', + fields: [fieldA], + restrictions: { + uniqueKey: [sharedName], + }, + }; + expect(Schema.safeParse(schema).success).true; + }); + it('Schema must have all fields listed', () => { + const sharedNameA = 'sharedNameA'; + const sharedNameB = 'schemaNameB'; + const fieldPass: SchemaField = { + name: sharedNameA, + valueType: 'boolean', + }; + const fieldFail: SchemaField = { + name: 'qwerty', + valueType: 'string', + }; + + const schemaSharedA: Schema = { + name: 'asdf', + fields: [fieldPass], + restrictions: { + uniqueKey: [sharedNameA], + }, + }; + const schemaFailSingle: Schema = { + name: 'asdf', + fields: [fieldFail], + restrictions: { + uniqueKey: [sharedNameA], + }, + }; + const schemaFailMulti: Schema = { + name: 'asdf', + fields: [fieldPass], + restrictions: { + uniqueKey: [sharedNameA, sharedNameB], + }, + }; + const schemaPassMultipleFields: Schema = { + name: 'asdf', + fields: [fieldPass, fieldFail], + restrictions: { + uniqueKey: [sharedNameA], + }, + }; + expect(Schema.safeParse(schemaSharedA).success).true; + expect(Schema.safeParse(schemaFailSingle).success).false; + expect(Schema.safeParse(schemaFailMulti).success).false; + expect(Schema.safeParse(schemaPassMultipleFields).success).true; + }); + }); + describe('foreignKey', () => { + it('Can have restriction', () => { + const foreignFieldName = 'foreignField'; + const foreignSchemaName = 'foreignSchema'; + + const localFieldName = 'localField'; + const localField: SchemaField = { + name: localFieldName, + valueType: 'boolean', + }; + const schema: Schema = { + name: 'localSchema', + fields: [localField], + restrictions: { + foreignKey: [ + { + schema: foreignSchemaName, + mappings: [ + { + local: localFieldName, + foreign: foreignFieldName, + }, + ], + }, + ], + }, + }; + + expect(Schema.safeParse(schema).success).true; + }); + it('Rejects if uses same schema name', () => { + const foreignFieldName = 'foreignField'; + + const schemaName = 'reusedName'; + const localFieldName = 'localField'; + const localField: SchemaField = { + name: localFieldName, + valueType: 'boolean', + }; + const schema: Schema = { + name: schemaName, + fields: [localField], + restrictions: { + foreignKey: [ + { + schema: schemaName, + mappings: [ + { + local: localFieldName, + foreign: foreignFieldName, + }, + ], + }, + ], + }, + }; + expect(Schema.safeParse(schema).success).false; + }); + it('Rejects if local field does not exist', () => { + const foreignFieldName = 'foreignField'; + const foreignSchemaName = 'foreignSchema'; + + const localFieldName = 'localField'; + const localField: SchemaField = { + name: localFieldName, + valueType: 'boolean', + }; + const schema: Schema = { + name: 'localSchema', + fields: [localField], + restrictions: { + foreignKey: [ + { + schema: foreignSchemaName, + mappings: [ + { + local: 'unknownName', + foreign: foreignFieldName, + }, + ], + }, + ], + }, + }; + + expect(Schema.safeParse(schema).success).false; + }); + }); + }); +}); diff --git a/libraries/dictionary/tsconfig.json b/libraries/dictionary/tsconfig.json index 38783ea..c480d7e 100644 --- a/libraries/dictionary/tsconfig.json +++ b/libraries/dictionary/tsconfig.json @@ -1,6 +1,6 @@ { "compilerOptions": { - "target": "ES5", + "target": "ESNext", "lib": ["ESNext"], "module": "CommonJS", "moduleResolution": "node", diff --git a/packages/client/README.md b/packages/client/README.md new file mode 100644 index 0000000..e6ac7ed --- /dev/null +++ b/packages/client/README.md @@ -0,0 +1,12 @@ +# Lectern TypeScript Client + +## Features: +- Runs different restrictions validations: regex, range, scripts, required fields, type checks, etc. +- Transforms the data from string to their proper type. +- Report validation errors. +- Fetch dictionaries from the configured lectern service. +- Provide typed definitions for the dictionary object. +- Analyze dictionary versions diff. + +## Usage examples: +- icgc-argo/argo-clinical [https://github.com/icgc-argo/argo-clinical] \ No newline at end of file diff --git a/packages/client/package.json b/packages/client/package.json new file mode 100644 index 0000000..6f22327 --- /dev/null +++ b/packages/client/package.json @@ -0,0 +1,52 @@ +{ + "name": "@overturebio-stack/lectern-client", + "version": "1.5.0", + "files": [ + "dist/" + ], + "main": "./dist/index.js", + "typings": "./dist/index.d.ts", + "description": "TypeScript client to interact with Lectern servers and perform data validation versus Lectern dictionaries.", + "scripts": { + "build": "rimraf dist && tsc", + "test": "nyc mocha --exit --timeout 5000 -r ts-node/register test/**.spec.ts", + "lint": "tslint -c tslint.json -e node_modules -p tsconfig.json" + }, + "repository": { + "type": "git", + "url": "https://github.com/overture-stack/lectern.git" + }, + "publishConfig": { + "access": "public" + }, + "license": "AGPL-3.0", + "devDependencies": { + "@types/chai": "^4.2.16", + "@types/deep-freeze": "^0.1.2", + "@types/lodash": "^4.14.195", + "@types/mocha": "^8.2.2", + "@types/node": "^12.0.10", + "@types/node-fetch": "^2.5.10", + "chai": "^4.3.4", + "husky": "^6.0.0", + "mocha": "^8.3.2", + "prettier": "^2.2.1", + "pretty-quick": "^3.1.0", + "rimraf": "^3.0.2", + "ts-node": "^9.1.1", + "tslint": "^6.1.3", + "typedoc": "^0.17.7", + "typescript": "^5.1.6" + }, + "dependencies": { + "cd": "^0.3.3", + "common": "workspace:^", + "deep-freeze": "^0.0.1", + "lodash": "^4.17.21", + "node-fetch": "^2.6.1", + "node-worker-threads-pool": "^1.4.3", + "promise-tools": "^2.1.0", + "winston": "^3.3.3" + }, + "author": "Ontario Institute for Cancer Research" +} diff --git a/packages/client/src/change-analyzer.ts b/packages/client/src/change-analyzer.ts new file mode 100644 index 0000000..02e2c08 --- /dev/null +++ b/packages/client/src/change-analyzer.ts @@ -0,0 +1,254 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { restClient } from './schema-rest-client'; +import { + SchemasDictionaryDiffs, + FieldChanges, + FieldDiff, + Change, + ChangeAnalysis, + ChangeTypeName, + RestrictionChanges, + FieldDefinition, +} from './schema-entities'; + +const isFieldChange = (obj: any): obj is Change => { + return obj.type !== undefined; +}; + +const isNestedChange = (obj: any): obj is { [field: string]: FieldChanges } => { + return obj.type === undefined; +}; + +const isRestrictionChange = (obj: any): obj is { [field: string]: FieldChanges } => { + return obj.type === undefined; +}; + +export const fetchDiffAndAnalyze = async (serviceUrl: string, name: string, fromVersion: string, toVersion: string) => { + const changes = await restClient.fetchDiff(serviceUrl, name, fromVersion, toVersion); + return analyzeChanges(changes); +}; + +export const analyzeChanges = (schemasDiff: SchemasDictionaryDiffs): ChangeAnalysis => { + const analysis: ChangeAnalysis = { + fields: { + addedFields: [], + renamedFields: [], + deletedFields: [], + }, + isArrayDesignationChanges: [], + restrictionsChanges: { + codeList: { + created: [], + deleted: [], + updated: [], + }, + regex: { + updated: [], + created: [], + deleted: [], + }, + required: { + updated: [], + created: [], + deleted: [], + }, + script: { + updated: [], + created: [], + deleted: [], + }, + range: { + updated: [], + created: [], + deleted: [], + }, + }, + metaChanges: { + core: { + changedToCore: [], + changedFromCore: [], + }, + }, + valueTypeChanges: [], + }; + + for (const field of Object.keys(schemasDiff)) { + const fieldChange: FieldDiff = schemasDiff[field]; + if (fieldChange) { + const fieldDiff = fieldChange.diff; + // if we have type at first level then it's a field add/delete + if (isFieldChange(fieldDiff)) { + categorizeFieldChanges(analysis, field, fieldDiff); + } + + if (isNestedChange(fieldDiff)) { + if (fieldDiff.meta) { + categorizeMetaChagnes(analysis, field, fieldDiff.meta); + } + + if (fieldDiff.restrictions) { + categorizeRestrictionChanges(analysis, field, fieldDiff.restrictions, fieldChange.after); + } + + if (fieldDiff.isArray) { + categorizeFieldArrayDesignationChange(analysis, field, fieldDiff.isArray); + } + + if (fieldDiff.valueType) { + categorizerValueTypeChange(analysis, field, fieldDiff.valueType); + } + } + } + } + + return analysis; +}; + +const categorizeFieldArrayDesignationChange = ( + analysis: ChangeAnalysis, + field: string, + changes: { [field: string]: FieldChanges } | Change, +) => { + // changing isArray designation is a relevant change for all cases except if it is created and set to false + if (!(changes.type === 'created' && changes.data === false)) { + analysis.isArrayDesignationChanges.push(field); + } +}; + +const categorizerValueTypeChange = ( + analysis: ChangeAnalysis, + field: string, + changes: { [field: string]: FieldChanges } | Change, +) => { + analysis.valueTypeChanges.push(field); +}; + +const categorizeRestrictionChanges = ( + analysis: ChangeAnalysis, + field: string, + restrictionsChange: { [field: string]: FieldChanges } | Change, + fieldDefinitionAfter?: FieldDefinition, +) => { + const restrictionsToCheck = ['regex', 'script', 'required', 'codeList', 'range']; + + // additions or deletions of a restriction object as whole (i.e. contains 1 or many restrictions within the 'data') + if (restrictionsChange.type) { + const createOrAddChange = restrictionsChange as Change; + const restrictionsData = createOrAddChange.data as any; + + for (const k of restrictionsToCheck) { + if (restrictionsData[k]) { + analysis.restrictionsChanges[k as keyof RestrictionChanges][restrictionsChange.type as ChangeTypeName].push({ + field: field, + definition: restrictionsData[k], + } as any); + } + } + return; + } + + // in case 'restrictions' key was already there but we modified its contents + const restrictionUpdate = restrictionsChange as { [field: string]: FieldChanges }; + for (const k of restrictionsToCheck) { + if (restrictionUpdate[k]) { + const change = restrictionUpdate[k] as Change; + // we need the '|| change' in case of nested attributes like ranges + /* + "diff": { + "restrictions": { + "range": { + "exclusiveMin": { + "type": "deleted", + "data": 0 + }, + "max": { + "type": "updated", + "data": 200000 + }, + "min": { + "type": "created", + "data": 0 + } + } + } + } + */ + if (k == 'range' && !change.type) { + // if the change is nested (type is at min max level) then the boundries were updated only : ex: + /* + change = { + "max" : { + type: "updated" + data: "..." + }, + "exclusiveMin": { + type: "deleted" + data .. + } + } + */ + const def: any = {}; + if (Object.keys(change).some((k) => k == 'max' || k == 'min' || k == 'exclusiveMin' || k == 'exclusiveMax')) { + analysis.restrictionsChanges[k]['updated'].push({ + field: field, + // we push the whole range definition since it doesnt make sense to just + // push one boundary. + definition: fieldDefinitionAfter?.restrictions?.range, + }); + } + return; + } + const definition = change.data || change; + analysis.restrictionsChanges[k as keyof RestrictionChanges][change.type as ChangeTypeName].push({ + field: field, + definition, + } as any); + } + } +}; + +const categorizeFieldChanges = (analysis: ChangeAnalysis, field: string, changes: Change) => { + const changeType = changes.type; + if (changeType == 'created') { + analysis.fields.addedFields.push({ + name: field, + definition: changes.data, + }); + } else if (changeType == 'deleted') { + analysis.fields.deletedFields.push(field); + } +}; + +const categorizeMetaChagnes = ( + analysis: ChangeAnalysis, + field: string, + metaChanges: { [field: string]: FieldChanges } | Change, +) => { + // **** meta changes - core *** + if (metaChanges?.data?.core === true) { + const changeType = metaChanges.type; + if (changeType === 'created' || changeType === 'updated') { + analysis.metaChanges?.core.changedToCore.push(field); + } else if (changeType === 'deleted') { + analysis.metaChanges?.core.changedFromCore.push(field); + } + } +}; diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts new file mode 100644 index 0000000..e7cf786 --- /dev/null +++ b/packages/client/src/index.ts @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import * as entities from './schema-entities'; +import * as analyzer from './change-analyzer'; +import * as functions from './schema-functions'; +import * as parallel from './parallel'; + +import { restClient } from './schema-rest-client'; +export { entities, analyzer, functions, parallel, restClient }; diff --git a/packages/client/src/logger.ts b/packages/client/src/logger.ts new file mode 100644 index 0000000..2724ace --- /dev/null +++ b/packages/client/src/logger.ts @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import winston from 'winston'; +const { createLogger, format, transports } = winston; +const { combine, timestamp, label, prettyPrint, json, align, simple } = format; + +// read the log level from the env directly since this is a very high priority value. +const LOG_LEVEL = process.env.LOG_LEVEL || 'info'; +console.log('log level configured: ', LOG_LEVEL); + +// Logger configuration +const logConfiguration = { + level: LOG_LEVEL, + format: combine(json(), simple(), timestamp()), + transports: [new winston.transports.Console()], +}; + +export interface Logger { + error(msg: string, err?: Error): void; + info(msg: string): void; + debug(msg: string): void; + profile(s: string): void; +} + +const winstonLogger = winston.createLogger(logConfiguration); +if (process.env.LOG_LEVEL == 'debug') { + console.log('logger configured: ', winstonLogger); +} +export const loggerFor = (fileName: string): Logger => { + if (process.env.LOG_LEVEL == 'debug') { + console.debug('creating logger for', fileName); + } + const source = fileName.substring(fileName.indexOf('argo-clinical')); + return { + error: (msg: string, err: Error): void => { + winstonLogger.error(msg, err, { source }); + }, + debug: (msg: string): void => { + winstonLogger.debug(msg, { source }); + }, + info: (msg: string): void => { + winstonLogger.info(msg, { source }); + }, + profile: (id: string): void => { + winstonLogger.profile(id); + }, + }; +}; diff --git a/packages/client/src/parallel.ts b/packages/client/src/parallel.ts new file mode 100644 index 0000000..603d57f --- /dev/null +++ b/packages/client/src/parallel.ts @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { DataRecord, SchemasDictionary, SchemaProcessingResult } from './schema-entities'; +import { StaticPool } from 'node-worker-threads-pool'; +import * as os from 'os'; +import { loggerFor } from './logger'; +const L = loggerFor(__filename); + +// check allowed cpus or use available +const cpuCount = os.cpus().length; +L.info(`available cpus: ${cpuCount}`); +const availableCpus = Number(process.env.ALLOWED_CPUS) || cpuCount; +L.info(`using ${availableCpus} cpus`); + +const pool = new StaticPool({ + size: availableCpus, + task: __dirname + '/schema-worker.js', +}); + +export const processRecord = async ( + dictionary: SchemasDictionary, + schemaName: string, + record: Readonly, + index: number, +): Promise => { + return (await pool.exec({ + dictionary, + schemaName, + record, + index, + })) as Promise; +}; diff --git a/packages/client/src/records-operations.ts b/packages/client/src/records-operations.ts new file mode 100644 index 0000000..7b964f3 --- /dev/null +++ b/packages/client/src/records-operations.ts @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2023 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { differenceWith, isEqual } from 'lodash'; + +/** + * Renames properties in a record using a mapping between current and new names. + * @param record The record whose properties should be renamed. + * @param fieldsMapping A mapping of current property names to new property names. + * @returns A new record with the properties' names changed according to the mapping. + */ +const renameProperties = ( + record: Record, + fieldsMapping: Map, +): Record => { + const renamed: Record = {}; + Object.entries(record).forEach(([propertyName, propertyValue]) => { + const newName = fieldsMapping.get(propertyName) ?? propertyName; + renamed[newName] = propertyValue; + }); + return renamed; +}; + +/** + * Returns a string representation of a record. The record is sorted by its properties so + * 2 records which have the same properties and values (even if in different order) will produce the + * same string with this function. + * @param record Record to be processed. + * @returns String representation of the record sorted by its properties. + */ +const getSortedRecordKey = (record: Record): string => { + const sortedKeys = Object.keys(record).sort(); + const sortedRecord: Record = {}; + for (const key of sortedKeys) { + sortedRecord[key] = record[key]; + } + return JSON.stringify(sortedRecord); +}; + +/** + * Find missing foreign keys by calculating the difference between 2 dataset keys (similar to a set difference). + * Returns rows in `dataKeysA` which are not present in `dataKeysB`. + * @param datasetKeysA Keys of the dataset A. The returned value of this function is a subset of this array. + * @param datasetKeysB Keys of the dataset B. Elements to be substracted from `datasetKeysA`. + * @param fieldsMapping Mapping of the field names so the keys can be compared correctly. + */ +export const findMissingForeignKeys = ( + datasetKeysA: [number, Record][], + datasetKeysB: [number, Record][], + fieldsMapping: Map, +): [number, Record][] => { + const diff = differenceWith(datasetKeysA, datasetKeysB, (a, b) => + isEqual(a[1], renameProperties(b[1], fieldsMapping)), + ); + return diff; +}; + +/** + * Find duplicate keys in a dataset. + * @param datasetKeys Array with the keys to evaluate. + * @returns An Array with all the values that appear more than once in the dataset. + */ +export const findDuplicateKeys = ( + datasetKeys: [number, Record][], +): [number, Record][] => { + const duplicateKeys: [number, Record][] = []; + const recordKeysMap: Map<[number, Record], string> = new Map(); + const keyCount: Map = new Map(); + + // Calculate a key per record, which is a string representation that allows to compare records even if their properties + // are in different order + datasetKeys.forEach((row) => { + const recordKey = getSortedRecordKey(row[1]); + const count = keyCount.get(recordKey) || 0; + keyCount.set(recordKey, count + 1); + recordKeysMap.set(row, recordKey); + }); + + // Find duplicates by checking the count of they key on each record + recordKeysMap.forEach((value, key) => { + const count = keyCount.get(value) ?? 0; + if (count > 1) { + duplicateKeys.push(key); + } + }); + return duplicateKeys; +}; diff --git a/packages/client/src/schema-entities.ts b/packages/client/src/schema-entities.ts new file mode 100644 index 0000000..810c9c8 --- /dev/null +++ b/packages/client/src/schema-entities.ts @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { loggerFor } from './logger'; +import { DeepReadonly } from 'deep-freeze'; +const L = loggerFor(__filename); + +export class DataRecord { + readonly [k: string]: string | string[]; +} + +export class TypedDataRecord { + readonly [k: string]: SchemaTypes; +} + +export type SchemaTypes = string | string[] | boolean | boolean[] | number | number[] | undefined; + +export interface SchemasDictionary { + version: string; + name: string; + schemas: Array; +} + +export interface SchemaDefinition { + readonly name: string; + readonly description: string; + readonly restrictions: SchemaRestriction; + readonly fields: ReadonlyArray; +} + +export interface SchemasDictionaryDiffs { + [fieldName: string]: FieldDiff; +} + +export interface FieldDiff { + before?: FieldDefinition; + after?: FieldDefinition; + diff: FieldChanges; +} + +export type SchemaData = ReadonlyArray; + +// changes can be nested +// in case of created/delete field we get Change +// in case of simple field change we get {"fieldName": {"data":.., "type": ..}} +// in case of nested fields: {"fieldName1": {"fieldName2": {"data":.., "type": ..}}} +export type FieldChanges = { [field: string]: FieldChanges } | Change; + +export enum ChangeTypeName { + CREATED = 'created', + DELETED = 'deleted', + UPDATED = 'updated', +} + +export interface Change { + type: ChangeTypeName; + data: any; +} + +export interface SchemaRestriction { + foreignKey?: { + schema: string; + mappings: { + local: string; + foreign: string; + }[]; + }[]; + uniqueKey?: string[]; +} + +export interface FieldDefinition { + name: string; + valueType: ValueType; + description: string; + meta?: { key?: boolean; default?: SchemaTypes; core?: boolean; examples?: string }; + restrictions?: { + codeList?: CodeListRestriction; + regex?: string; + script?: Array | string; + required?: boolean; + unique?: boolean; + range?: RangeRestriction; + }; + isArray?: boolean; +} + +export type CodeListRestriction = Array; + +export type RangeRestriction = { + min?: number; + max?: number; + exclusiveMin?: number; + exclusiveMax?: number; +}; + +export enum ValueType { + STRING = 'string', + INTEGER = 'integer', + NUMBER = 'number', + BOOLEAN = 'boolean', +} + +export type SchemaProcessingResult = DeepReadonly<{ + validationErrors: SchemaValidationError[]; + processedRecord: TypedDataRecord; +}>; + +export type BatchProcessingResult = DeepReadonly<{ + validationErrors: SchemaValidationError[]; + processedRecords: TypedDataRecord[]; +}>; + +export enum SchemaValidationErrorTypes { + MISSING_REQUIRED_FIELD = 'MISSING_REQUIRED_FIELD', + INVALID_FIELD_VALUE_TYPE = 'INVALID_FIELD_VALUE_TYPE', + INVALID_BY_REGEX = 'INVALID_BY_REGEX', + INVALID_BY_RANGE = 'INVALID_BY_RANGE', + INVALID_BY_SCRIPT = 'INVALID_BY_SCRIPT', + INVALID_ENUM_VALUE = 'INVALID_ENUM_VALUE', + UNRECOGNIZED_FIELD = 'UNRECOGNIZED_FIELD', + INVALID_BY_UNIQUE = 'INVALID_BY_UNIQUE', + INVALID_BY_FOREIGN_KEY = 'INVALID_BY_FOREIGN_KEY', + INVALID_BY_UNIQUE_KEY = 'INVALID_BY_UNIQUE_KEY', +} + +export interface SchemaValidationError { + readonly errorType: SchemaValidationErrorTypes; + readonly index: number; + readonly fieldName: string; + readonly info: Record; + readonly message: string; +} + +export interface FieldNamesByPriorityMap { + required: string[]; + optional: string[]; +} + +export interface ChangeAnalysis { + fields: { + addedFields: AddedFieldChange[]; + renamedFields: string[]; + deletedFields: string[]; + }; + isArrayDesignationChanges: string[]; + restrictionsChanges: RestrictionChanges; + metaChanges?: MetaChanges; + valueTypeChanges: string[]; +} + +export type RestrictionChanges = { + range: { + [key in ChangeTypeName]: ObjectChange[]; + }; + codeList: { + [key in ChangeTypeName]: ObjectChange[]; + }; + regex: RegexChanges; + required: RequiredChanges; + script: ScriptChanges; +}; + +export type MetaChanges = { + core: { + changedToCore: string[]; // fields that are core now + changedFromCore: string[]; // fields that are not core now + }; +}; + +export type RegexChanges = { + [key in ChangeTypeName]: StringAttributeChange[]; +}; + +export type RequiredChanges = { + [key in ChangeTypeName]: BooleanAttributeChange[]; +}; + +export type ScriptChanges = { + [key in ChangeTypeName]: StringAttributeChange[]; +}; + +export interface AddedFieldChange { + name: string; + definition: FieldDefinition; +} + +export interface ObjectChange { + field: string; + definition: any; +} + +export interface CodeListChange { + field: string; + definition: any; +} + +export interface StringAttributeChange { + field: string; + definition: string; +} + +export interface BooleanAttributeChange { + field: string; + definition: boolean; +} diff --git a/packages/client/src/schema-error-messages.ts b/packages/client/src/schema-error-messages.ts new file mode 100644 index 0000000..3ed08ab --- /dev/null +++ b/packages/client/src/schema-error-messages.ts @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { isArray } from 'lodash'; +import { RangeRestriction } from './schema-entities'; + +function getForeignKeyErrorMsg(errorData: any) { + const valueEntries = Object.entries(errorData.info.value); + const formattedKeyValues: string[] = valueEntries.map(([key, value]) => { + if (isArray(value)) { + return `${key}: [${value.join(', ')}]`; + } else { + return `${key}: ${value}`; + } + }); + const valuesAsString = formattedKeyValues.join(', '); + const detail = `Key ${valuesAsString} is not present in schema ${errorData.info.foreignSchema}`; + const msg = `Record violates foreign key restriction defined for field(s) ${errorData.fieldName}. ${detail}.`; + return msg; +} + +function getUniqueKeyErrorMsg(errorData: any) { + const uniqueKeyFields: string[] = errorData.info.uniqueKeyFields; + const formattedKeyValues: string[] = uniqueKeyFields.map((fieldName) => { + const value = errorData.info.value[fieldName]; + if (isArray(value)) { + return `${fieldName}: [${value.join(', ')}]`; + } else { + return `${fieldName}: ${value === '' ? 'null' : value}`; + } + }); + const valuesAsString = formattedKeyValues.join(', '); + const msg = `Key ${valuesAsString} must be unique.`; + return msg; +} + +const INVALID_VALUE_ERROR_MESSAGE = 'The value is not permissible for this field.'; +const ERROR_MESSAGES: { [key: string]: (errorData: any) => string } = { + INVALID_FIELD_VALUE_TYPE: () => INVALID_VALUE_ERROR_MESSAGE, + INVALID_BY_REGEX: (errData) => getRegexErrorMsg(errData.info), + INVALID_BY_RANGE: (errorData) => `Value is out of permissible range, value must be ${rangeToSymbol(errorData.info)}.`, + INVALID_BY_SCRIPT: (error) => error.info.message, + INVALID_ENUM_VALUE: () => INVALID_VALUE_ERROR_MESSAGE, + MISSING_REQUIRED_FIELD: (errorData) => `${errorData.fieldName} is a required field.`, + INVALID_BY_UNIQUE: (errorData) => `Value for ${errorData.fieldName} must be unique.`, + INVALID_BY_FOREIGN_KEY: (errorData) => getForeignKeyErrorMsg(errorData), + INVALID_BY_UNIQUE_KEY: (errorData) => getUniqueKeyErrorMsg(errorData), +}; + +// Returns the formatted message for the given error key, taking any required properties from the info object +// Default value is the errorType itself (so we can identify errorTypes that we are missing messages for and the user could look up the error meaning in our docs) +const schemaErrorMessage = (errorType: string, errorData: any = {}): string => { + return errorType && Object.keys(ERROR_MESSAGES).includes(errorType) + ? ERROR_MESSAGES[errorType](errorData) + : errorType; +}; + +const rangeToSymbol = (range: RangeRestriction): string => { + let minString = ''; + let maxString = ''; + + const hasBothRange = + (range.min !== undefined || range.exclusiveMin !== undefined) && + (range.max != undefined || range.exclusiveMax !== undefined); + + if (range.min !== undefined) { + minString = `>= ${range.min}`; + } + + if (range.exclusiveMin !== undefined) { + minString = `> ${range.exclusiveMin}`; + } + + if (range.max !== undefined) { + maxString = `<= ${range.max}`; + } + + if (range.exclusiveMax !== undefined) { + maxString = `< ${range.exclusiveMax}`; + } + + return hasBothRange ? `${minString} and ${maxString}` : `${minString}${maxString}`; +}; + +function getRegexErrorMsg(info: any) { + let msg = `The value is not a permissible for this field, it must meet the regular expression: "${info.regex}".`; + if (info.examples) { + msg = msg + ` Examples: ${info.examples}`; + } + return msg; +} + +export default schemaErrorMessage; diff --git a/packages/client/src/schema-functions.ts b/packages/client/src/schema-functions.ts new file mode 100644 index 0000000..52c0d7f --- /dev/null +++ b/packages/client/src/schema-functions.ts @@ -0,0 +1,855 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { + SchemaValidationError, + TypedDataRecord, + SchemaTypes, + SchemaProcessingResult, + FieldNamesByPriorityMap, + BatchProcessingResult, + CodeListRestriction, + RangeRestriction, + SchemaData, +} from './schema-entities'; +import vm from 'vm'; +import { + SchemasDictionary, + SchemaDefinition, + FieldDefinition, + ValueType, + DataRecord, + SchemaValidationErrorTypes, +} from './schema-entities'; + +import { + Checks, + notEmpty, + isEmptyString, + isAbsent, + F, + isNotAbsent, + isStringArray, + isString, + isEmpty, + convertToArray, + isNumberArray, +} from './utils'; +import schemaErrorMessage from './schema-error-messages'; +import { loggerFor } from './logger'; +import { DeepReadonly } from 'deep-freeze'; +import _, { isArray } from 'lodash'; +import { findDuplicateKeys, findMissingForeignKeys } from './records-operations'; +const L = loggerFor(__filename); + +export const getSchemaFieldNamesWithPriority = ( + schema: SchemasDictionary, + definition: string, +): FieldNamesByPriorityMap => { + const schemaDef: SchemaDefinition | undefined = schema.schemas.find((schema) => schema.name === definition); + if (!schemaDef) { + throw new Error(`no schema found for : ${definition}`); + } + const fieldNamesMapped: FieldNamesByPriorityMap = { required: [], optional: [] }; + schemaDef.fields.forEach((field) => { + if (field.restrictions && field.restrictions.required) { + fieldNamesMapped.required.push(field.name); + } else { + fieldNamesMapped.optional.push(field.name); + } + }); + return fieldNamesMapped; +}; + +const getNotNullSchemaDefinitionFromDictionary = ( + dictionary: SchemasDictionary, + schemaName: string, +): SchemaDefinition => { + const schemaDef: SchemaDefinition | undefined = dictionary.schemas.find((e) => e.name === schemaName); + if (!schemaDef) { + throw new Error(`no schema found for : ${schemaName}`); + } + return schemaDef; +}; + +export const processSchemas = ( + dictionary: SchemasDictionary, + schemasData: Record, +): Record => { + Checks.checkNotNull('dictionary', dictionary); + Checks.checkNotNull('schemasData', schemasData); + + const results: Record = {}; + + Object.keys(schemasData).forEach((schemaName) => { + // Run validations at the record level + const recordLevelValidationResults = processRecords(dictionary, schemaName, schemasData[schemaName]); + + // Run cross-schema validations + const schemaDef: SchemaDefinition = getNotNullSchemaDefinitionFromDictionary(dictionary, schemaName); + const crossSchemaLevelValidationResults = validation + .runCrossSchemaValidationPipeline(schemaDef, schemasData, [validation.validateForeignKey]) + .filter(notEmpty); + + const recordLevelErrors = recordLevelValidationResults.validationErrors.map((x) => { + return { + errorType: x.errorType, + index: x.index, + fieldName: x.fieldName, + info: x.info, + message: x.message, + }; + }); + + const crossSchemaLevelErrors = crossSchemaLevelValidationResults.map((x) => { + return { + errorType: x.errorType, + index: x.index, + fieldName: x.fieldName, + info: x.info, + message: x.message, + }; + }); + + const allErrorsBySchema = [...recordLevelErrors, ...crossSchemaLevelErrors]; + + results[schemaName] = F({ + validationErrors: allErrorsBySchema, + processedRecords: recordLevelValidationResults.processedRecords, + }); + }); + + return results; +}; + +export const processRecords = ( + dataSchema: SchemasDictionary, + definition: string, + records: ReadonlyArray, +): BatchProcessingResult => { + Checks.checkNotNull('records', records); + Checks.checkNotNull('dataSchema', dataSchema); + Checks.checkNotNull('definition', definition); + + const schemaDef: SchemaDefinition = getNotNullSchemaDefinitionFromDictionary(dataSchema, definition); + + let validationErrors: SchemaValidationError[] = []; + const processedRecords: TypedDataRecord[] = []; + + records.forEach((r, i) => { + const result = process(dataSchema, definition, r, i); + validationErrors = validationErrors.concat(result.validationErrors); + processedRecords.push(_.cloneDeep(result.processedRecord) as TypedDataRecord); + }); + // Record set level validations + const newErrors = validateRecordsSet(schemaDef, processedRecords); + validationErrors.push(...newErrors); + L.debug( + `done processing all rows, validationErrors: ${validationErrors.length}, validRecords: ${processedRecords.length}`, + ); + + return F({ + validationErrors, + processedRecords, + }); +}; + +export const process = ( + dataSchema: SchemasDictionary, + definition: string, + rec: Readonly, + index: number, +): SchemaProcessingResult => { + Checks.checkNotNull('records', rec); + Checks.checkNotNull('dataSchema', dataSchema); + Checks.checkNotNull('definition', definition); + + const schemaDef: SchemaDefinition | undefined = dataSchema.schemas.find((e) => e.name === definition); + + if (!schemaDef) { + throw new Error(`no schema found for : ${definition}`); + } + + let validationErrors: SchemaValidationError[] = []; + + const defaultedRecord: DataRecord = populateDefaults(schemaDef, F(rec), index); + L.debug(`done populating defaults for record #${index}`); + const result = validate(schemaDef, defaultedRecord, index); + L.debug(`done validation for record #${index}`); + if (result && result.length > 0) { + L.debug(`${result.length} validation errors for record #${index}`); + validationErrors = validationErrors.concat(result); + } + const convertedRecord = convertFromRawStrings(schemaDef, defaultedRecord, index, result); + L.debug(`converted row #${index} from raw strings`); + const postTypeConversionValidationResult = validateAfterTypeConversion( + schemaDef, + _.cloneDeep(convertedRecord) as DataRecord, + index, + ); + + if (postTypeConversionValidationResult && postTypeConversionValidationResult.length > 0) { + validationErrors = validationErrors.concat(postTypeConversionValidationResult); + } + + L.debug(`done processing all rows, validationErrors: ${validationErrors.length}, validRecords: ${convertedRecord}`); + + return F({ + validationErrors, + processedRecord: convertedRecord, + }); +}; + +/** + * Populate the passed records with the default value based on the field name if the field is + * missing from the records it will NOT be added. + * @param definition the name of the schema definition to use for these records + * @param records the list of records to populate with the default values. + */ +const populateDefaults = ( + schemaDef: Readonly, + record: DeepReadonly, + index: number, +): DataRecord => { + Checks.checkNotNull('records', record); + L.debug(`in populateDefaults ${schemaDef.name}, ${record}`); + const mutableRecord: RawMutableRecord = _.cloneDeep(record) as RawMutableRecord; + const x: SchemaDefinition = schemaDef; + schemaDef.fields.forEach((field) => { + const defaultValue = field.meta && field.meta.default; + if (isEmpty(defaultValue)) return undefined; + + const value = record[field.name]; + + // data record value is (or is expected to be) just one string + if (isString(value) && !field.isArray) { + if (isNotAbsent(value) && value.trim() === '') { + L.debug(`populating Default: ${defaultValue} for ${field.name} in record : ${record}`); + mutableRecord[field.name] = `${defaultValue}`; + } + return undefined; + } + + // data record value is (or is expected to be) array of string + if (isStringArray(value) && field.isArray) { + if (notEmpty(value) && value.every((v) => v.trim() === '')) { + L.debug(`populating Default: ${defaultValue} for ${field.name} in record : ${record}`); + const arrayDefaultValue = convertToArray(defaultValue); + mutableRecord[field.name] = arrayDefaultValue.map((v) => `${v}`); + } + return undefined; + } + }); + + return _.cloneDeep(mutableRecord); +}; + +const convertFromRawStrings = ( + schemaDef: SchemaDefinition, + record: DataRecord, + index: number, + recordErrors: ReadonlyArray, +): DeepReadonly => { + const mutableRecord: MutableRecord = { ...record }; + schemaDef.fields.forEach((field) => { + // if there was an error for this field don't convert it. this means a string was passed instead of number or boolean + // this allows us to continue other validations without hiding possible errors down. + if ( + recordErrors.find( + (er) => er.errorType == SchemaValidationErrorTypes.INVALID_FIELD_VALUE_TYPE && er.fieldName == field.name, + ) + ) { + return undefined; + } + + /* + * if the field is missing from the records don't set it to undefined + */ + if (!_.has(record, field.name)) { + return; + } + + // need to check how it behaves for record[field.name] == "" + if (isEmpty(record[field.name])) { + mutableRecord[field.name] = undefined; + return; + } + + const valueType = field.valueType; + const rawValue = record[field.name]; + + if (field.isArray) { + const rawValues = convertToArray(rawValue); + mutableRecord[field.name] = rawValues.map( + (rv) => getTypedValue(field, valueType, rv) as any, // fix type here + ); + } else { + mutableRecord[field.name] = getTypedValue(field, valueType, rawValue as string); + } + }); + return F(mutableRecord); +}; + +const getTypedValue = (field: FieldDefinition, valueType: ValueType, rawValue: string) => { + let formattedFieldValue = rawValue; + // convert field to match corresponding enum from codelist, if possible + if (field.restrictions && field.restrictions.codeList && valueType === ValueType.STRING) { + const formattedField = field.restrictions.codeList.find( + (e) => e.toString().toLowerCase() === rawValue.toString().toLowerCase(), + ); + if (formattedField) { + formattedFieldValue = formattedField as string; + } + } + + let typedValue: SchemaTypes = rawValue; + switch (valueType) { + case ValueType.STRING: + typedValue = formattedFieldValue; + break; + case ValueType.INTEGER: + typedValue = Number(rawValue); + break; + case ValueType.NUMBER: + typedValue = Number(rawValue); + break; + case ValueType.BOOLEAN: + // we have to lower case in case of inconsistent letters (boolean requires all small letters). + typedValue = Boolean(rawValue.toLowerCase()); + break; + } + + return typedValue; +}; + +/** + * A "select" function that retrieves specific fields from the dataset as a record, as well as the numeric position of each row in the dataset. + * @param dataset Dataset to select fields from. + * @param fields Array with names of the fields to select. + * @returns A tuple array. In each tuple, the first element is the index of the row in the dataset, and the second value is the record with the + * selected values. + */ +const selectFieldsFromDataset = ( + dataset: SchemaData, + fields: string[], +): [number, Record][] => { + const records: [number, Record][] = []; + dataset.forEach((row, index) => { + const values: Record = {}; + fields.forEach((field) => { + values[field] = row[field] || ''; + }); + records.push([index, values]); + }); + return records; +}; + +/** + * Run schema validation pipeline for a schema defintion on the list of records provided. + * @param definition the schema definition name. + * @param record the records to validate. + */ +const validate = ( + schemaDef: SchemaDefinition, + record: DataRecord, + index: number, +): ReadonlyArray => { + const majorErrors = validation + .runValidationPipeline(record, index, schemaDef.fields, [ + validation.validateFieldNames, + validation.validateNonArrayFields, + validation.validateRequiredFields, + validation.validateValueTypes, + ]) + .filter(notEmpty); + return [...majorErrors]; +}; + +const validateAfterTypeConversion = ( + schemaDef: SchemaDefinition, + record: TypedDataRecord, + index: number, +): ReadonlyArray => { + const validationErrors = validation + .runValidationPipeline(record, index, schemaDef.fields, [ + validation.validateRegex, + validation.validateRange, + validation.validateEnum, + validation.validateScript, + ]) + .filter(notEmpty); + + return [...validationErrors]; +}; +export type ProcessingFunction = (schema: SchemaDefinition, rec: Readonly, index: number) => any; + +type MutableRecord = { [key: string]: SchemaTypes }; +type RawMutableRecord = { [key: string]: string | string[] }; + +namespace validation { + // these validation functions run AFTER the record has been converted to the correct types from raw strings + export type TypedValidationFunction = ( + rec: TypedDataRecord, + index: number, + fields: Array, + ) => Array; + + // these validation functions run BEFORE the record has been converted to the correct types from raw strings + export type ValidationFunction = ( + rec: DataRecord, + index: number, + fields: Array, + ) => Array; + + // these validation functions run AFTER the records has been converted to the correct types from raw strings, and apply to a dataset instead of + // individual records + export type TypedDatasetValidationFunction = ( + dataset: Array, + schemaDef: SchemaDefinition, + ) => Array; + + export type CrossSchemaValidationFunction = ( + schemaDef: SchemaDefinition, + schemasData: Record, + ) => Array; + + export const runValidationPipeline = ( + rec: DataRecord | TypedDataRecord, + index: number, + fields: ReadonlyArray, + funs: Array, + ) => { + let result: Array = []; + for (const fun of funs) { + if (rec instanceof DataRecord) { + const typedFunc = fun as ValidationFunction; + result = result.concat(typedFunc(rec as DataRecord, index, getValidFields(result, fields))); + } else { + const typedFunc = fun as TypedValidationFunction; + result = result.concat(typedFunc(rec as TypedDataRecord, index, getValidFields(result, fields))); + } + } + return result; + }; + + export const runDatasetValidationPipeline = ( + dataset: Array, + schemaDef: SchemaDefinition, + funs: Array, + ) => { + let result: Array = []; + for (const fun of funs) { + const typedFunc = fun as TypedDatasetValidationFunction; + result = result.concat(typedFunc(dataset, schemaDef)); + } + return result; + }; + + export const runCrossSchemaValidationPipeline = ( + schemaDef: SchemaDefinition, + schemasData: Record, + funs: Array, + ) => { + let result: Array = []; + for (const fun of funs) { + const typedFunc = fun as CrossSchemaValidationFunction; + result = result.concat(typedFunc(schemaDef, schemasData)); + } + return result; + }; + + export const validateRegex: TypedValidationFunction = ( + rec: TypedDataRecord, + index: number, + fields: ReadonlyArray, + ) => { + return fields + .map((field) => { + const recordFieldValues = convertToArray(rec[field.name]); + if (!isStringArray(recordFieldValues)) return undefined; + + const regex = field.restrictions?.regex; + if (isEmpty(regex)) return undefined; + + const invalidValues = recordFieldValues.filter((v) => isInvalidRegexValue(regex, v)); + if (invalidValues.length !== 0) { + const examples = field.meta?.examples; + const info = { value: invalidValues, regex, examples }; + return buildError(SchemaValidationErrorTypes.INVALID_BY_REGEX, field.name, index, info); + } + return undefined; + }) + .filter(notEmpty); + }; + + export const validateRange: TypedValidationFunction = ( + rec: TypedDataRecord, + index: number, + fields: ReadonlyArray, + ) => { + return fields + .map((field) => { + const recordFieldValues = convertToArray(rec[field.name]); + if (!isNumberArray(recordFieldValues)) return undefined; + + const range = field.restrictions?.range; + if (isEmpty(range)) return undefined; + + const invalidValues = recordFieldValues.filter((v) => isOutOfRange(range, v)); + if (invalidValues.length !== 0) { + const info = { value: invalidValues, ...range }; + return buildError(SchemaValidationErrorTypes.INVALID_BY_RANGE, field.name, index, info); + } + return undefined; + }) + .filter(notEmpty); + }; + + export const validateScript: TypedValidationFunction = ( + rec: TypedDataRecord, + index: number, + fields: Array, + ) => { + return fields + .map((field) => { + if (field.restrictions && field.restrictions.script) { + const scriptResult = validateWithScript(field, rec); + if (!scriptResult.valid) { + return buildError(SchemaValidationErrorTypes.INVALID_BY_SCRIPT, field.name, index, { + message: scriptResult.message, + value: rec[field.name], + }); + } + } + return undefined; + }) + .filter(notEmpty); + }; + + export const validateEnum: TypedValidationFunction = ( + rec: TypedDataRecord, + index: number, + fields: Array, + ) => { + return fields + .map((field) => { + const codeList = field.restrictions?.codeList || undefined; + if (isEmpty(codeList)) return undefined; + + const recordFieldValues = convertToArray(rec[field.name]); // put all values into array for easier validation + const invalidValues = recordFieldValues.filter((val) => isInvalidEnumValue(codeList, val)); + + if (invalidValues.length !== 0) { + const info = { value: invalidValues }; + return buildError(SchemaValidationErrorTypes.INVALID_ENUM_VALUE, field.name, index, info); + } + return undefined; + }) + .filter(notEmpty); + }; + + export const validateUnique: TypedDatasetValidationFunction = ( + dataset: Array, + schemaDef: SchemaDefinition, + ) => { + const errors: Array = []; + schemaDef.fields.forEach((field) => { + const unique = field.restrictions?.unique || undefined; + if (!unique) return undefined; + const keysToValidate = selectFieldsFromDataset(dataset as DataRecord[], [field.name]); + const duplicateKeys = findDuplicateKeys(keysToValidate); + + duplicateKeys.forEach(([index, record]) => { + const info = { value: record[field.name] }; + errors.push(buildError(SchemaValidationErrorTypes.INVALID_BY_UNIQUE, field.name, index, info)); + }); + }); + return errors; + }; + + export const validateUniqueKey: TypedDatasetValidationFunction = ( + dataset: Array, + schemaDef: SchemaDefinition, + ) => { + const errors: Array = []; + const uniqueKeyRestriction = schemaDef?.restrictions?.uniqueKey; + if (uniqueKeyRestriction) { + const uniqueKeyFields: string[] = uniqueKeyRestriction; + const keysToValidate = selectFieldsFromDataset(dataset as SchemaData, uniqueKeyFields); + const duplicateKeys = findDuplicateKeys(keysToValidate); + + duplicateKeys.forEach(([index, record]) => { + const info = { value: record, uniqueKeyFields: uniqueKeyFields }; + errors.push( + buildError(SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, uniqueKeyFields.join(', '), index, info), + ); + }); + } + return errors; + }; + + export const validateValueTypes: ValidationFunction = ( + rec: DataRecord, + index: number, + fields: Array, + ) => { + return fields + .map((field) => { + if (isEmpty(rec[field.name])) return undefined; + + const recordFieldValues = convertToArray(rec[field.name]); // put all values into array + const invalidValues = recordFieldValues.filter((v) => isInvalidFieldType(field.valueType, v)); + const info = { value: invalidValues }; + + if (invalidValues.length !== 0) { + return buildError(SchemaValidationErrorTypes.INVALID_FIELD_VALUE_TYPE, field.name, index, info); + } + return undefined; + }) + .filter(notEmpty); + }; + + export const validateRequiredFields = (rec: DataRecord, index: number, fields: Array) => { + return fields + .map((field) => { + if (isRequiredMissing(field, rec)) { + return buildError(SchemaValidationErrorTypes.MISSING_REQUIRED_FIELD, field.name, index); + } + return undefined; + }) + .filter(notEmpty); + }; + + export const validateFieldNames: ValidationFunction = ( + record: Readonly, + index: number, + fields: Array, + ) => { + const expectedFields = new Set(fields.map((field) => field.name)); + return Object.keys(record) + .map((recFieldName) => { + if (!expectedFields.has(recFieldName)) { + return buildError(SchemaValidationErrorTypes.UNRECOGNIZED_FIELD, recFieldName, index); + } + return undefined; + }) + .filter(notEmpty); + }; + + export const validateNonArrayFields: ValidationFunction = ( + record: Readonly, + index: number, + fields: Array, + ) => { + return fields + .map((field) => { + if (!field.isArray && isStringArray(record[field.name])) { + return buildError(SchemaValidationErrorTypes.INVALID_FIELD_VALUE_TYPE, field.name, index); + } + return undefined; + }) + .filter(notEmpty); + }; + + export const validateForeignKey: CrossSchemaValidationFunction = ( + schemaDef: SchemaDefinition, + schemasData: Record, + ) => { + const errors: Array = []; + const foreignKeyDefinitions = schemaDef?.restrictions?.foreignKey; + if (foreignKeyDefinitions) { + foreignKeyDefinitions.forEach((foreignKeyDefinition) => { + const localSchemaData = schemasData[schemaDef.name] || []; + const foreignSchemaData = schemasData[foreignKeyDefinition.schema] || []; + + // A foreign key can have more than one field, in which case is a composite foreign key. + const localFields = foreignKeyDefinition.mappings.map((x) => x.local); + const foreignFields = foreignKeyDefinition.mappings.map((x) => x.foreign); + + const fieldsMappings = new Map(foreignKeyDefinition.mappings.map((x) => [x.foreign, x.local])); + + // Select the keys of the datasets to compare. The keys are records to support the scenario where the fk is composite. + const localValues: [number, Record][] = selectFieldsFromDataset( + localSchemaData, + localFields, + ); + const foreignValues: [number, Record][] = selectFieldsFromDataset( + foreignSchemaData, + foreignFields, + ); + + // This artificial record in foreignValues allows null references in localValues to be valid. + const emptyRow: Record = {}; + foreignFields.forEach((field) => (emptyRow[field] = '')); + foreignValues.push([-1, emptyRow]); + + const missingForeignKeys = findMissingForeignKeys(localValues, foreignValues, fieldsMappings); + + missingForeignKeys.forEach((record) => { + const index = record[0]; + const info = { + value: record[1], + foreignSchema: foreignKeyDefinition.schema, + }; + + errors.push( + buildError(SchemaValidationErrorTypes.INVALID_BY_FOREIGN_KEY, localFields.join(', '), index, info), + ); + }); + }); + } + return errors; + }; + + export const getValidFields = ( + errs: ReadonlyArray, + fields: ReadonlyArray, + ) => { + return fields.filter((field) => { + return !errs.find((e) => e.fieldName == field.name); + }); + }; + + // return false if the record value is a valid type + export const isInvalidFieldType = (valueType: ValueType, value: string) => { + // optional field if the value is absent at this point + if (isAbsent(value) || isEmptyString(value)) return false; + switch (valueType) { + case ValueType.STRING: + return false; + case ValueType.INTEGER: + return isNaN(Number(value)) || !Number.isInteger(Number(value)); + case ValueType.NUMBER: + return isNaN(Number(value)); + case ValueType.BOOLEAN: + return !(value.toLowerCase() === 'true' || value.toLowerCase() === 'false'); + } + }; + + export const isRequiredMissing = (field: FieldDefinition, record: DataRecord) => { + const isRequired = field.restrictions && field.restrictions.required; + if (!isRequired) return false; + + const recordFieldValues = convertToArray(record[field.name]); + return recordFieldValues.every(isEmptyString); + }; + + const isOutOfRange = (range: RangeRestriction, value: number | undefined) => { + if (value == undefined) return false; + const invalidRange = + // less than the min if defined ? + (range.min !== undefined && value < range.min) || + (range.exclusiveMin !== undefined && value <= range.exclusiveMin) || + // bigger than max if defined ? + (range.max !== undefined && value > range.max) || + (range.exclusiveMax !== undefined && value >= range.exclusiveMax); + return invalidRange; + }; + + const isInvalidEnumValue = (codeList: CodeListRestriction, value: string | boolean | number | undefined) => { + // optional field if the value is absent at this point + if (isAbsent(value) || isEmptyString(value as string)) return false; + return !codeList.find((e) => e === value); + }; + + const isInvalidRegexValue = (regex: string, value: string) => { + // optional field if the value is absent at this point + if (isAbsent(value) || isEmptyString(value)) return false; + const regexPattern = new RegExp(regex); + return !regexPattern.test(value); + }; + + const ctx = vm.createContext(); + + const validateWithScript = ( + field: FieldDefinition, + record: TypedDataRecord, + ): { + valid: boolean; + message: string; + } => { + try { + const args = { + $row: record, + $field: record[field.name], + $name: field.name, + }; + + if (!field.restrictions || !field.restrictions.script) { + throw new Error('called validation by script without script provided'); + } + + // scripts should already be strings inside arrays, but ensure that they are to help transition between lectern versions + // checking for this can be removed in future versions of lectern (feb 2020) + const scripts = + typeof field.restrictions.script === 'string' ? [field.restrictions.script] : field.restrictions.script; + + let result: { + valid: boolean; + message: string; + } = { + valid: false, + message: '', + }; + + for (const scriptString of scripts) { + const script = getScript(scriptString); + const valFunc = script.runInContext(ctx); + if (!valFunc) throw new Error('Invalid script'); + result = valFunc(args); + /* Return the first script that's invalid. Otherwise result will be valid with message: 'ok'*/ + if (!result.valid) break; + } + + return result; + } catch (err) { + console.error( + `failed running validation script ${field.name} for record: ${JSON.stringify(record)}. Error message: ${err}`, + ); + return { + valid: false, + message: 'failed to run script validation, check script and the input', + }; + } + }; + + const getScript = (scriptString: string) => { + const script = new vm.Script(scriptString); + return script; + }; + + const buildError = ( + errorType: SchemaValidationErrorTypes, + fieldName: string, + index: number, + info: object = {}, + ): SchemaValidationError => { + const errorData = { errorType, fieldName, index, info }; + return { ...errorData, message: schemaErrorMessage(errorType, errorData) }; + }; +} +function validateRecordsSet(schemaDef: SchemaDefinition, processedRecords: TypedDataRecord[]) { + const validationErrors = validation + .runDatasetValidationPipeline(processedRecords, schemaDef, [ + validation.validateUnique, + validation.validateUniqueKey, + ]) + .filter(notEmpty); + return validationErrors; +} diff --git a/packages/client/src/schema-rest-client.ts b/packages/client/src/schema-rest-client.ts new file mode 100644 index 0000000..a9a944c --- /dev/null +++ b/packages/client/src/schema-rest-client.ts @@ -0,0 +1,154 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { loggerFor } from './logger'; +import fetch from 'node-fetch'; +import { SchemasDictionary, SchemasDictionaryDiffs, FieldChanges, FieldDiff } from './schema-entities'; +import promiseTools from 'promise-tools'; +import { unknownToString } from 'common'; +const L = loggerFor(__filename); + +export interface SchemaServiceRestClient { + fetchSchema(schemaSvcUrl: string, name: string, version: string): Promise; + fetchDiff( + schemaSvcUrl: string, + name: string, + fromVersion: string, + toVersion: string, + ): Promise; +} + +export const restClient: SchemaServiceRestClient = { + fetchSchema: async (schemaSvcUrl: string, name: string, version: string): Promise => { + // for testing where we need to work against stub schema + if (schemaSvcUrl.startsWith('file://')) { + return await loadSchemaFromFile(version, schemaSvcUrl, name); + } + + if (!schemaSvcUrl) { + throw new Error('please configure a valid url to get schema from'); + } + const url = `${schemaSvcUrl}/dictionaries?name=${name}&version=${version}`; + try { + L.debug(`in fetch live schema ${version}`); + const schemaDictionary = await doRequest(url); + // todo validate response and map it to a schema + return schemaDictionary[0] as SchemasDictionary; + } catch (error: unknown) { + L.error(`failed to fetch schema at url: ${url} - ${unknownToString(error)}`); + throw error; + } + }, + fetchDiff: async ( + schemaSvcBaseUrl: string, + name: string, + fromVersion: string, + toVersion: string, + ): Promise => { + // for testing where we need to work against stub schema + let diffResponse: any; + if (schemaSvcBaseUrl.startsWith('file://')) { + diffResponse = await loadDiffFromFile(schemaSvcBaseUrl, name, fromVersion, toVersion); + } else { + const url = `${schemaSvcBaseUrl}/diff?name=${name}&left=${fromVersion}&right=${toVersion}`; + diffResponse = (await doRequest(url)) as any[]; + } + const result: SchemasDictionaryDiffs = {}; + for (const entry of diffResponse) { + const fieldName = entry[0] as string; + if (entry[1]) { + const fieldDiff: FieldDiff = { + before: entry[1].left, + after: entry[1].right, + diff: entry[1].diff, + }; + result[fieldName] = fieldDiff; + } + } + return result; + }, +}; + +const doRequest = async (url: string) => { + let response: any; + try { + const retryAttempt = 1; + response = await promiseTools.retry({ times: 5, interval: 1000 }, async () => { + L.debug(`fetching schema attempt #${retryAttempt}`); + return promiseTools.timeout(fetch(url), 5000); + }); + return await response.json(); + } catch (error: unknown) { + L.error(`failed to fetch schema at url: ${url} - ${unknownToString(error)}`); + throw response.status == 404 ? new Error('Not Found') : new Error('Request Failed'); + } +}; + +async function loadSchemaFromFile(version: string, schemaSvcUrl: string, name: string) { + L.debug(`in fetch stub schema ${version}`); + const result = delay(1000); + const dictionary = await result(() => { + const dictionaries: SchemasDictionary[] = require(schemaSvcUrl.substring(7, schemaSvcUrl.length)) + .dictionaries as SchemasDictionary[]; + if (!dictionaries) { + throw new Error('your mock json is not structured correctly, see sampleFiles/sample-schema.json'); + } + const dic = dictionaries.find((d: any) => d.version == version && d.name == name); + if (!dic) { + return undefined; + } + return dic; + }); + if (dictionary == undefined) { + throw new Error("couldn't load stub dictionary with the criteria specified"); + } + L.debug(`schema found ${dictionary.version}`); + return dictionary; +} + +async function loadDiffFromFile(schemaSvcBaseUrl: string, name: string, fromVersion: string, toVersion: string) { + L.debug(`in fetch stub diffs ${name} ${fromVersion} ${toVersion}`); + const result = delay(1000); + const diff = await result(() => { + const diffResponse = require(schemaSvcBaseUrl.substring(7, schemaSvcBaseUrl.length)).diffs as any[]; + if (!diffResponse) { + throw new Error('your mock json is not structured correctly, see sampleFiles/sample-schema.json'); + } + + const diff = diffResponse.find( + (d: any) => d.fromVersion == fromVersion && d.toVersion == toVersion && d.name == name, + ); + if (!diff) { + return undefined; + } + return diff; + }); + if (diff == undefined) { + throw new Error("couldn't load stub diff with the criteria specified, check your stub file"); + } + return diff.data; +} + +function delay(milliseconds: number) { + return async (result: () => T | undefined) => { + return new Promise((resolve, reject) => { + setTimeout(() => resolve(result()), milliseconds); + }); + }; +} diff --git a/packages/client/src/schema-worker.js b/packages/client/src/schema-worker.js new file mode 100644 index 0000000..cdb44c0 --- /dev/null +++ b/packages/client/src/schema-worker.js @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2020 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +const workerThreads = require('worker_threads'); +const parentPort = workerThreads.parentPort; +// ts node is required to import ts modules like "schema-functions.ts" in this case since +// worker threads run in their own V8 instance +const tsNode = require('ts-node'); + +/** + * when we run the app with node directly like this: + * node -r ts-node/register server.ts + * we will have a registered ts node instance, registering another one will result in wierd behaviour + * however when we run with mocha: + * mocha -r ts-node/register .ts + * (same applies if we run with ts node directly: ts-node server.ts) + * the worker thread won't have an instance of ts node transpiler + * unlike node for some reason which seem to attach the isntance to the worker thread process. + * + * so we had to add this work around to avoid double registry in different run modes. + * root cause of why mocha acts different than node is not found yet. + */ +if (!process[tsNode.REGISTER_INSTANCE]) { + tsNode.register(); +} +const service = require('./schema-functions'); + +function processProxy(args) { + return service.process(args.dictionary, args.schemaName, args.record, args.index); +} + +parentPort.on('message', (args) => { + const result = processProxy(args); + parentPort.postMessage(result); +}); diff --git a/packages/client/src/utils.ts b/packages/client/src/utils.ts new file mode 100644 index 0000000..391b266 --- /dev/null +++ b/packages/client/src/utils.ts @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import fs from 'fs'; +import deepFreeze from 'deep-freeze'; +import _ from 'lodash'; +import { isArray } from 'util'; + +const fsPromises = fs.promises; + +export namespace Checks { + export const checkNotNull = (argName: string, arg: any) => { + if (!arg) { + throw new Errors.InvalidArgument(argName); + } + }; +} + +export namespace Errors { + export class InvalidArgument extends Error { + constructor(argumentName: string) { + super(`Invalid argument : ${argumentName}`); + } + } + + export class NotFound extends Error { + constructor(msg: string) { + super(msg); + } + } + + export class StateConflict extends Error { + constructor(msg: string) { + super(msg); + } + } +} + +// type gaurd to filter out undefined and null +// https://stackoverflow.com/questions/43118692/typescript-filter-out-nulls-from-an-array +export function notEmpty(value: TValue | null | undefined): value is TValue { + // lodash 4.14 behavior note, these are all evaluated to true: + // _.isEmpty(null) _.isEmpty(undefined) _.isEmpty([]) + // _.isEmpty({}) _.isEmpty('') _.isEmpty(12) & _.isEmpty(NaN) + + // so check number seperately since it will evaluate to isEmpty=true + return (isNumber(value) && !isNaN(value)) || !_.isEmpty(value); +} + +export function isEmpty(value: TValue | null | undefined): value is undefined { + return !notEmpty(value); +} + +export const convertToArray = (val: T | T[]): T[] => { + if (Array.isArray(val)) { + return val; + } else { + return [val]; + } +}; + +export function isString(value: any): value is string { + return typeof value === 'string' || value instanceof String; +} + +export function isStringArray(value: any | undefined | null): value is string[] { + return Array.isArray(value) && value.every(isString); +} + +export function isNumber(value: any): value is number { + return typeof value === 'number'; +} + +export function isNumberArray(values: any): values is number[] { + return Array.isArray(values) && values.every(isNumber); +} + +// returns true if value matches at least one of the expressions +export const isStringMatchRegex = (expressions: RegExp[], value: string) => { + return expressions.filter((exp) => RegExp(exp).test(value)).length >= 1; +}; + +export const isNotEmptyString = (value: string | undefined): value is string => { + return isNotAbsent(value) && value.trim() !== ''; +}; + +export const isEmptyString = (value: string) => { + return !isNotEmptyString(value); +}; + +export const isAbsent = (value: string | number | boolean | undefined): value is undefined => { + return !isNotAbsent(value); +}; + +export const isNotAbsent = (value: string | number | boolean | undefined): value is string | number | boolean => { + return value !== null && value !== undefined; +}; + +export const sleep = async (milliSeconds: number = 2000) => { + return new Promise((resolve) => setTimeout(resolve, milliSeconds)); +}; + +export function toString(obj: any) { + if (!obj) { + return undefined; + } + Object.keys(obj).forEach((k) => { + if (typeof obj[k] === 'object') { + return toString(obj[k]); + } + obj[k] = `${obj[k]}`; + }); + + return obj; +} + +export function isValueEqual(value: any, other: any) { + if (isArray(value) && isArray(other)) { + return _.difference(value, other).length === 0; // check equal, ignore order + } + + return _.isEqual(value, other); +} + +export function isValueNotEqual(value: any, other: any) { + return !isValueEqual(value, other); +} + +export function convertToTrimmedString(val: unknown | undefined | string | number | boolean | null) { + return val === undefined || val === null ? '' : String(val).trim(); +} + +export const F = deepFreeze; diff --git a/packages/client/test/change-analyzer.spec.ts b/packages/client/test/change-analyzer.spec.ts new file mode 100644 index 0000000..b0fb319 --- /dev/null +++ b/packages/client/test/change-analyzer.spec.ts @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import chai from 'chai'; +import * as analyzer from '../src/change-analyzer'; +import { SchemasDictionaryDiffs, FieldDiff, ChangeAnalysis } from '../src/schema-entities'; +import _ from 'lodash'; +chai.should(); +const diffResponse: any = require('./schema-diff.json'); +const schemaDiff: SchemasDictionaryDiffs = {}; +for (const entry of diffResponse) { + const fieldName = entry[0] as string; + if (entry[1]) { + const fieldDiff: FieldDiff = { + before: entry[1].left, + after: entry[1].right, + diff: entry[1].diff, + }; + schemaDiff[fieldName] = fieldDiff; + } +} + +const expectedResult: ChangeAnalysis = { + fields: { + addedFields: [], + renamedFields: [], + deletedFields: ['primary_diagnosis.menopause_status'], + }, + metaChanges: { + core: { + changedToCore: [], + changedFromCore: [], + }, + }, + restrictionsChanges: { + codeList: { + created: [], + deleted: [ + { + field: 'donor.vital_status', + definition: ['Alive', 'Deceased', 'Not reported', 'Unknown'], + }, + ], + updated: [ + { + field: 'donor.cause_of_death', + definition: { + added: ['N/A'], + deleted: ['Died of cancer', 'Unknown'], + }, + }, + ], + }, + regex: { + updated: [ + { + field: 'donor.submitter_donor_id', + definition: '[A-Za-z0-9\\-\\._]{3,64}', + }, + { + field: 'primary_diagnosis.cancer_type_code', + definition: '[A-Z]{1}[0-9]{2}.[0-9]{0,3}[A-Z]{2,3}$', + }, + ], + created: [ + { + field: 'donor.vital_status', + definition: '[A-Z]{3,100}', + }, + ], + deleted: [], + }, + required: { + updated: [], + created: [], + deleted: [], + }, + script: { + updated: [], + created: [ + { + field: 'donor.survival_time', + definition: ' $field / 2 == 0 ', + }, + ], + deleted: [], + }, + range: { + updated: [ + { + definition: { + max: 1, + }, + field: 'specimen.percent_stromal_cells', + }, + ], + created: [ + { + field: 'donor.survival_time', + definition: { + min: 0, + max: 200000, + }, + }, + ], + deleted: [], + }, + }, + isArrayDesignationChanges: ['primary_diagnosis.presenting_symptoms'], + valueTypeChanges: ['sample_registration.program_id'], +}; + +describe('change-analyzer', () => { + it('categorize changes correctly', () => { + const result = analyzer.analyzeChanges(schemaDiff); + result.should.deep.eq(expectedResult); + }); +}); diff --git a/packages/client/test/schema-diff.json b/packages/client/test/schema-diff.json new file mode 100644 index 0000000..e4f7a4d --- /dev/null +++ b/packages/client/test/schema-diff.json @@ -0,0 +1,312 @@ +[ + [ + "donor.submitter_donor_id", + { + "left": { + "description": "Unique identifier of the donor, assigned by the data provider.", + "name": "submitter_donor_id", + "restrictions": { + "required": true, + "regex": "[A-Za-z0-9\\-\\._]{1,64}" + }, + "valueType": "string" + }, + "right": { + "description": "Unique identifier of the donor, assigned by the data provider.", + "name": "submitter_donor_id", + "restrictions": { + "required": true, + "regex": "[A-Za-z0-9\\-\\._]{3,64}" + }, + "valueType": "string" + }, + "diff": { + "restrictions": { + "regex": { + "type": "updated", + "data": "[A-Za-z0-9\\-\\._]{3,64}" + } + } + } + } + ], + [ + "donor.vital_status", + { + "left": { + "description": "Donors last known state of living or deceased.", + "name": "vital_status", + "restrictions": { + "codeList": ["Alive", "Deceased", "Not reported", "Unknown"], + "required": true + }, + "valueType": "string" + }, + "right": { + "description": "Donors last known state of living or deceased.", + "name": "vital_status", + "restrictions": { + "regex": "[A-Z]{3,100}", + "required": true + }, + "valueType": "string" + }, + "diff": { + "restrictions": { + "codeList": { + "type": "deleted", + "data": ["Alive", "Deceased", "Not reported", "Unknown"] + }, + "regex": { + "type": "created", + "data": "[A-Z]{3,100}" + } + } + } + } + ], + [ + "donor.cause_of_death", + { + "left": { + "description": "Description of the cause of a donor's death.", + "name": "cause_of_death", + "restrictions": { + "codeList": ["Died of cancer", "Died of other reasons", "Not reported", "Unknown"] + }, + "valueType": "string" + }, + "right": { + "description": "Description of the cause of a donor's death.", + "name": "cause_of_death", + "restrictions": { + "codeList": ["Died of other reasons", "Not reported", "N/A"] + }, + "valueType": "string" + }, + "diff": { + "restrictions": { + "codeList": { + "type": "updated", + "data": { + "added": ["N/A"], + "deleted": ["Died of cancer", "Unknown"] + } + } + } + } + } + ], + [ + "donor.survival_time", + { + "left": { + "description": "Interval of how long the donor has survived since primary diagnosis, in days.", + "meta": { + "units": "days" + }, + "name": "survival_time", + "valueType": "integer" + }, + "right": { + "description": "Interval of how long the donor has survived since primary diagnosis, in days.", + "meta": { + "units": "days" + }, + "name": "survival_time", + "valueType": "integer", + "restrictions": { + "script": " $field / 2 == 0 ", + "range": { + "min": 0, + "max": 200000 + } + } + }, + "diff": { + "restrictions": { + "type": "created", + "data": { + "range": { + "min": 0, + "max": 200000 + }, + "script": " $field / 2 == 0 " + } + } + } + } + ], + [ + "primary_diagnosis.cancer_type_code", + { + "left": { + "name": "cancer_type_code", + "valueType": "string", + "description": "The code to represent the cancer type using the WHO ICD-10 code (https://icd.who.int/browse10/2016/en#/) classification.", + "restrictions": { + "required": true, + "regex": "[A-Z]{1}[0-9]{2}.[0-9]{0,3}[A-Z]{0,1}$" + } + }, + "right": { + "name": "cancer_type_code", + "valueType": "string", + "description": "The code to represent the cancer type using the WHO ICD-10 code (https://icd.who.int/browse10/2016/en#/) classification.", + "restrictions": { + "required": true, + "regex": "[A-Z]{1}[0-9]{2}.[0-9]{0,3}[A-Z]{2,3}$" + } + }, + "diff": { + "restrictions": { + "regex": { + "type": "updated", + "data": "[A-Z]{1}[0-9]{2}.[0-9]{0,3}[A-Z]{2,3}$" + } + } + } + } + ], + [ + "primary_diagnosis.menopause_status", + { + "left": { + "name": "menopause_status", + "description": "Indicate the menopause status of the patient at the time of primary diagnosis.", + "valueType": "string", + "restrictions": { + "codeList": ["Perimenopausal", "Postmenopausal", "Premenopausal", "Unknown"] + } + }, + "diff": { + "type": "deleted", + "data": { + "name": "menopause_status", + "description": "Indicate the menopause status of the patient at the time of primary diagnosis.", + "valueType": "string", + "restrictions": { + "codeList": ["Perimenopausal", "Postmenopausal", "Premenopausal", "Unknown"] + } + } + } + } + ], + [ + "primary_diagnosis.presenting_symptoms", + { + "left": { + "name": "presenting_symptoms", + "description": "Indicate presenting symptoms at time of primary diagnosis.", + "valueType": "string", + "restrictions": { + "codeList": ["Abdominal Pain", "Anemia", "Diabetes", "Diarrhea", "Nausea", "None"] + } + }, + "right": { + "name": "presenting_symptoms", + "description": "Indicate presenting symptoms at time of primary diagnosis.", + "valueType": "string", + "isArray": true, + "restrictions": { + "codeList": ["Abdominal Pain", "Anemia", "Diabetes", "Diarrhea", "Nausea", "None"] + } + }, + "diff": { + "isArray": { + "type": "created", + "data": true + } + } + } + ], + [ + "sample_registration.program_id", + { + "left": { + "name": "program_id", + "valueType": "string", + "description": "Unique identifier of the ARGO program.", + "meta": { + "validationDependency": true, + "primaryId": true, + "examples": "PACA-AU,BR-CA", + "displayName": "Program ID" + }, + "restrictions": { + "required": true + } + }, + "right": { + "name": "program_id", + "valueType": "integer", + "description": "Unique identifier of the ARGO program.", + "meta": { + "validationDependency": true, + "primaryId": true, + "examples": "PACA-AU,BR-CA", + "displayName": "Program ID" + }, + "restrictions": { + "required": true + } + }, + "diff": { + "valueType": { + "type": "updated", + "data": "integer" + } + } + } + ], + [ + "specimen.percent_stromal_cells", + { + "left": { + "name": "percent_stromal_cells", + "description": "Indicate a value, in decimals, that represents the percentage of reactive cells that are present in a malignant tumour specimen but are not malignant such as fibroblasts, vascular structures, etc.", + "valueType": "number", + "meta": { + "dependsOn": "sample_registration.tumour_normal_designation", + "notes": "", + "displayName": "Percent Stromal Cells" + }, + "restrictions": { + "range": { + "min": 0, + "max": 0.1 + } + } + }, + "right": { + "name": "percent_stromal_cells", + "description": "Indicate a value, in decimals, that represents the percentage of reactive cells that are present in a malignant tumour specimen but are not malignant such as fibroblasts, vascular structures, etc.", + "valueType": "number", + "meta": { + "dependsOn": "sample_registration.tumour_normal_designation", + "notes": "", + "displayName": "Percent Stromal Cells" + }, + "restrictions": { + "range": { + "max": 1 + } + } + }, + "diff": { + "restrictions": { + "range": { + "min": { + "type": "deleted", + "data": 0 + }, + "max": { + "type": "updated", + "data": 1 + } + } + } + } + } + ] +] diff --git a/packages/client/test/schema-functions.spec.ts b/packages/client/test/schema-functions.spec.ts new file mode 100644 index 0000000..315fa81 --- /dev/null +++ b/packages/client/test/schema-functions.spec.ts @@ -0,0 +1,9295 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import chai from 'chai'; +import * as schemaService from '../src/schema-functions'; +import { SchemasDictionary, SchemaValidationErrorTypes } from '../src/schema-entities'; +import schemaErrorMessage from '../src/schema-error-messages'; +import { loggerFor } from '../src/logger'; +const L = loggerFor(__filename); + +chai.should(); +const schema: SchemasDictionary = require('./schema.json')[0]; + +const VALUE_NOT_ALLOWED = 'The value is not permissible for this field.'; +const PROGRAM_ID_REQ = 'program_id is a required field.'; + +describe('schema-functions', () => { + it('should populate records based on default value ', () => { + const result = schemaService.processRecords(schema, 'registration', [ + { + program_id: 'PEME-CA', + submitter_donor_id: 'OD1234', + gender: '', + submitter_specimen_id: '87813', + specimen_type: 'Skin', + tumour_normal_designation: 'Normal', + submitter_sample_id: 'MAS123', + sample_type: 'ctDNA', + }, + { + program_id: 'PEME-CA', + submitter_donor_id: 'OD1234', + gender: '', + submitter_specimen_id: '87812', + specimen_type: 'Skin', + tumour_normal_designation: 'Normal', + submitter_sample_id: 'MAS1234', + sample_type: 'ctDNA', + }, + ]); + chai.expect(result.processedRecords[0].gender).to.eq('Other'); + chai.expect(result.processedRecords[1].gender).to.eq('Other'); + }); + + it('should NOT populate missing columns based on default value ', () => { + const result = schemaService.processRecords(schema, 'registration', [ + { + program_id: 'PEME-CA', + submitter_donor_id: 'OD1234', + gendr: '', + submitter_specimen_id: '87813', + specimen_type: 'Skin', + tumour_normal_designation: 'Normal', + submitter_sample_id: 'MAS123', + sample_type: 'ctDNA', + }, + { + program_id: 'PEME-CA', + submitter_donor_id: 'OD1234', + gender: '', + submitter_specimen_id: '87812', + specimen_type: 'Skin', + tumour_normal_designation: 'Normal', + submitter_sample_id: 'MAS1234', + sample_type: 'ctDNA', + }, + ]); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.MISSING_REQUIRED_FIELD, + fieldName: 'gender', + index: 0, + info: {}, + message: 'gender is a required field.', + }); + }); + + it('should validate required', () => { + const result = schemaService.processRecords(schema, 'registration', [ + { + submitter_donor_id: 'OD1234', + gender: 'Female', + submitter_specimen_id: '87813', + specimen_type: 'Skin', + tumour_normal_designation: 'Normal', + submitter_sample_id: 'MAS123', + sample_type: 'ctDNA', + }, + ]); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.MISSING_REQUIRED_FIELD, + fieldName: 'program_id', + index: 0, + info: {}, + message: PROGRAM_ID_REQ, + }); + }); + + it('should validate value types', () => { + const result = schemaService.processRecords(schema, 'address', [ + { + country: 'US', + unit_number: 'abc', + postal_code: '12345', + }, + ]); + + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_FIELD_VALUE_TYPE, + fieldName: 'unit_number', + index: 0, + info: { value: ['abc'] }, + message: VALUE_NOT_ALLOWED, + }); + }); + + it('should convert string to integer after processing', () => { + const result = schemaService.processRecords(schema, 'address', [ + { + country: 'US', + unit_number: '123', + postal_code: '12345', + }, + ]); + chai.expect(result.processedRecords).to.deep.include({ + country: 'US', + unit_number: 123, + postal_code: '12345', + }); + }); + + it('should validate regex', () => { + const result = schemaService.processRecords(schema, 'registration', [ + { + program_id: 'PEME-CAA', + submitter_donor_id: 'OD1234', + gender: 'Female', + submitter_specimen_id: '87813', + specimen_type: 'Skin', + tumour_normal_designation: 'Normal', + submitter_sample_id: 'MAS123', + sample_type: 'ctDNA', + }, + ]); + chai.expect(result.validationErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_REGEX, + fieldName: 'program_id', + index: 0, + info: { + examples: 'PACA-CA, BASHAR-LA', + regex: '^[A-Z1-9][-_A-Z1-9]{2,7}(-[A-Z][A-Z])$', + value: ['PEME-CAA'], + }, + message: + 'The value is not a permissible for this field, it must meet the regular expression: "^[A-Z1-9][-_A-Z1-9]{2,7}(-[A-Z][A-Z])$". Examples: PACA-CA, BASHAR-LA', + }); + }); + + it('should validate range', () => { + const result = schemaService.processRecords(schema, 'address', [ + { + country: 'US', + postal_code: '12345', + unit_number: '-1', + }, + { + country: 'US', + postal_code: '12345', + unit_number: '223', + }, + { + country: 'US', + postal_code: '12345', + unit_number: '500000', + }, + ]); + + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_BY_RANGE, + fieldName: 'unit_number', + index: 0, + info: { + exclusiveMax: 999, + min: 0, + value: [-1], + }, + message: schemaErrorMessage(SchemaValidationErrorTypes.INVALID_BY_RANGE, { + info: { + exclusiveMax: 999, + min: 0, + }, + }), + }); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_BY_RANGE, + fieldName: 'unit_number', + index: 2, + info: { + exclusiveMax: 999, + min: 0, + value: [500000], + }, + message: schemaErrorMessage(SchemaValidationErrorTypes.INVALID_BY_RANGE, { + info: { + exclusiveMax: 999, + min: 0, + }, + }), + }); + }); + + it('should validate script', () => { + const result = schemaService.processRecords(schema, 'address', [ + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + ]); + + chai.expect(result.validationErrors.length).to.eq(2); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_BY_SCRIPT, + fieldName: 'postal_code', + index: 0, + info: { message: 'invalid postal code for US', value: '12' }, + message: 'invalid postal code for US', + }); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_BY_SCRIPT, + fieldName: 'postal_code', + index: 1, + info: { message: 'invalid postal code for CANADA', value: 'ABC' }, + message: 'invalid postal code for CANADA', + }); + }); + + it('should validate if non-required feilds are not provided', () => { + const result = schemaService.processRecords(schema, 'donor', [ + // optional enum field not provided + { + program_id: 'PACA-AU', + submitter_donor_id: 'ICGC_0004', + gender: 'Female', + ethnicity: 'black or african american', + vital_status: 'alive', + }, + // optional enum field provided with proper value + { + program_id: 'PACA-AU', + submitter_donor_id: 'ICGC_0002', + gender: 'Male', + ethnicity: 'asian', + vital_status: 'deceased', + cause_of_death: 'died of cancer', + survival_time: '124', + }, + // optional enum field provided with no value + { + program_id: 'PACA-AU', + submitter_donor_id: 'ICGC_0002', + gender: 'Male', + ethnicity: 'asian', + vital_status: 'deceased', + cause_of_death: '', + survival_time: '124', + }, + ]); + chai.expect(result.validationErrors.length).to.eq(0); + }); + + it('should error if integer fields are not valid', () => { + const result = schemaService.processRecords(schema, 'donor', [ + { + program_id: 'PACA-AU', + submitter_donor_id: 'ICGC_0002', + gender: 'Other', + ethnicity: 'asian', + vital_status: 'deceased', + cause_of_death: 'died of cancer', + survival_time: '0.5', + }, + ]); + + chai.expect(result.validationErrors.length).to.eq(1); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_FIELD_VALUE_TYPE, + fieldName: 'survival_time', + index: 0, + info: { value: ['0.5'] }, + message: VALUE_NOT_ALLOWED, + }); + }); + + it('should validate case insensitive enums, return proper format', () => { + const result = schemaService.processRecords(schema, 'registration', [ + { + program_id: 'PACA-AU', + submitter_donor_id: 'OD1234', + gender: 'feMale', + submitter_specimen_id: '87813', + specimen_type: 'sKiN', + tumour_normal_designation: 'Normal', + submitter_sample_id: 'MAS123', + sample_type: 'CTdna', + }, + ]); + chai.expect(result.validationErrors.length).to.eq(0); + chai.expect(result.processedRecords[0]).to.deep.eq({ + program_id: 'PACA-AU', + submitter_donor_id: 'OD1234', + gender: 'Female', + submitter_specimen_id: '87813', + specimen_type: 'Skin', + tumour_normal_designation: 'Normal', + submitter_sample_id: 'MAS123', + sample_type: 'ctDNA', + }); + }); + + it('should not validate if unrecognized fields are provided', () => { + const result = schemaService.processRecords(schema, 'donor', [ + { + program_id: 'PACA-AU', + submitter_donor_id: 'ICGC_0002', + gender: 'Other', + ethnicity: 'asian', + vital_status: 'deceased', + cause_of_death: 'died of cancer', + survival_time: '5', + hackField: 'muchHack', + }, + ]); + chai.expect(result.validationErrors.length).to.eq(1); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.UNRECOGNIZED_FIELD, + message: SchemaValidationErrorTypes.UNRECOGNIZED_FIELD, + fieldName: 'hackField', + index: 0, + info: {}, + }); + }); + + it('should validate number/integer array with field defined ranges', () => { + const result = schemaService.processRecords(schema, 'favorite_things', [ + { + id: 'TH-ING', + fraction: ['0.2', '2', '3'], + integers: ['-100', '-2'], + }, + ]); + + chai.expect(result.validationErrors.length).to.eq(2); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_BY_RANGE, + message: 'Value is out of permissible range, value must be > 0 and <= 1.', + index: 0, + fieldName: 'fraction', + info: { value: [2, 3], max: 1, exclusiveMin: 0 }, + }); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_BY_RANGE, + message: 'Value is out of permissible range, value must be >= -10 and <= 10.', + index: 0, + fieldName: 'integers', + info: { value: [-100], max: 10, min: -10 }, + }); + }); + + it('should validate string array with field defined codelist', () => { + const result = schemaService.processRecords(schema, 'favorite_things', [ + { + id: 'TH-ING', + fruit: ['Mango', '2'], + }, + ]); + chai.expect(result.validationErrors.length).to.eq(1); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_ENUM_VALUE, + message: 'The value is not permissible for this field.', + fieldName: 'fruit', + index: 0, + info: { value: ['2'] }, + }); + }); + + it('should validate string with field defined codelist', () => { + const result = schemaService.processRecords(schema, 'favorite_things', [ + { + id: 'TH-ING', + fruit_single_value: 'Banana', + }, + ]); + chai.expect(result.validationErrors.length).to.eq(1); + chai.expect(result.validationErrors).to.deep.include({ + errorType: SchemaValidationErrorTypes.INVALID_ENUM_VALUE, + message: 'The value is not permissible for this field.', + fieldName: 'fruit_single_value', + index: 0, + info: { value: ['Banana'] }, + }); + }); + + it('should validate string array with field defined regex', () => { + const result = schemaService.processRecords(schema, 'favorite_things', [ + { + id: 'TH-ING', + qWords: ['que', 'not_q'], + }, + ]); + chai.expect(result.validationErrors.length).to.eq(1); + chai.expect(result.validationErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_REGEX, + message: 'The value is not a permissible for this field, it must meet the regular expression: "^q.*$".', + fieldName: 'qWords', + index: 0, + info: { value: ['not_q'], regex: '^q.*$', examples: undefined }, + }); + }); + + it('should validate string with field defined regex', () => { + const result = schemaService.processRecords(schema, 'favorite_things', [ + { + id: 'TH-ING', + qWord: 'not_q', + }, + ]); + chai.expect(result.validationErrors.length).to.eq(1); + chai.expect(result.validationErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_REGEX, + message: 'The value is not a permissible for this field, it must meet the regular expression: "^q.*$".', + fieldName: 'qWord', + index: 0, + info: { value: ['not_q'], regex: '^q.*$', examples: undefined }, + }); + }); + + it('should pass unique restriction validation when only null values exists', () => { + const result = schemaService.processRecords(schema, 'favorite_things', [ + { + id: 'TH-ING', + unique_value: '', + }, + ]); + chai.expect(result.validationErrors.length).to.eq(0); + }); + + it('should pass unique restriction validation when only one record exists', () => { + const result = schemaService.processRecords(schema, 'favorite_things', [ + { + id: 'TH-ING', + unique_value: 'unique_value_1', + }, + ]); + chai.expect(result.validationErrors.length).to.eq(0); + }); + + it('should fail unique restriction validation when duplicate values exist (scalar)', () => { + const result = schemaService.processRecords(schema, 'favorite_things', [ + { + id: 'ID-1', + unique_value: 'unique_value_1', + }, + { + id: 'ID-2', + unique_value: 'unique_value_1', + }, + ]); + + chai.expect(result.validationErrors.length).to.eq(2); + chai.expect(result.validationErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE, + message: 'Value for unique_value must be unique.', + fieldName: 'unique_value', + index: 0, + info: { value: ['unique_value_1'] }, + }); + chai.expect(result.validationErrors[1]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE, + message: 'Value for unique_value must be unique.', + fieldName: 'unique_value', + index: 1, + info: { value: ['unique_value_1'] }, + }); + }); + it('should fail unique restriction validation when duplicate values exist (array)', () => { + const result = schemaService.processRecords(schema, 'favorite_things', [ + { + id: 'ID-1', + unique_value: ['unique_value_1', 'unique_value_2'], + }, + { + id: 'ID-2', + unique_value: ['unique_value_1', 'unique_value_2'], + }, + ]); + + chai.expect(result.validationErrors.length).to.eq(2); + chai.expect(result.validationErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE, + message: 'Value for unique_value must be unique.', + fieldName: 'unique_value', + index: 0, + info: { value: ['unique_value_1', 'unique_value_2'] }, + }); + chai.expect(result.validationErrors[1]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE, + message: 'Value for unique_value must be unique.', + fieldName: 'unique_value', + index: 1, + info: { value: ['unique_value_1', 'unique_value_2'] }, + }); + }); + + it('should pass foreignKey restriction validation when values exist in foreign schema', () => { + const parent_schema_1_data = [ + { + id: 'parent_schema_1_id_1', + name: 'parent_schema_1_name_1', + }, + { + id: 'parent_schema_1_id_2', + name: 'parent_schema_1_name_2', + }, + ]; + + const child_schema_simple_fk_data = [ + { + id: '1', + parent_schema_1_id: 'parent_schema_1_id_1', + }, + { + id: '2', + parent_schema_1_id: 'parent_schema_1_id_2', + }, + ]; + const schemaData = { + parent_schema_1: parent_schema_1_data, + child_schema_simple_fk: child_schema_simple_fk_data, + }; + + const result = schemaService.processSchemas(schema, schemaData); + + chai.expect(result['parent_schema_1'].validationErrors.length).to.eq(0); + chai.expect(result['child_schema_simple_fk'].validationErrors.length).to.eq(0); + }); + + it('should pass foreignKey restriction validation when local schema has null values', () => { + const parent_schema_1_data = [ + { + id: 'parent_schema_1_id_1', + name: 'parent_schema_1_name_1', + }, + { + id: 'parent_schema_1_id_2', + name: 'parent_schema_1_name_2', + }, + ]; + + const child_schema_simple_fk_data = [ + { + id: '1', + parent_schema_1_id: 'parent_schema_1_id_1', + }, + { + id: '2', + parent_schema_1_id: '', + }, + ]; + const schemaData = { + parent_schema_1: parent_schema_1_data, + child_schema_simple_fk: child_schema_simple_fk_data, + }; + + const result = schemaService.processSchemas(schema, schemaData); + + chai.expect(result['parent_schema_1'].validationErrors.length).to.eq(0); + chai.expect(result['child_schema_simple_fk'].validationErrors.length).to.eq(0); + }); + + it('should pass foreignKey restriction validation when values exist in foreign schema (composite fk)', () => { + const parent_schema_1_data = [ + { + id: 'parent_schema_1_id_1', + external_id: 'parent_schema_1_external_id_1', + name: 'parent_schema_1_name_1', + }, + { + id: 'parent_schema_1_id_2', + external_id: 'parent_schema_1_external_id_2', + name: 'parent_schema_1_name_2', + }, + ]; + + const child_schema_composite_fk_data = [ + { + id: '1', + parent_schema_1_id: 'parent_schema_1_id_1', + parent_schema_1_external_id: 'parent_schema_1_external_id_1', + }, + { + id: '2', + parent_schema_1_id: 'parent_schema_1_id_2', + parent_schema_1_external_id: 'parent_schema_1_external_id_2', + }, + ]; + const schemaData = { + parent_schema_1: parent_schema_1_data, + child_schema_composite_fk: child_schema_composite_fk_data, + }; + + const result = schemaService.processSchemas(schema, schemaData); + + chai.expect(result['parent_schema_1'].validationErrors.length).to.eq(0); + chai.expect(result['child_schema_composite_fk'].validationErrors.length).to.eq(0); + }); + + it('should fail foreignKey restriction validation when value does not exist in foreign schema', () => { + const parent_schema_1_data = [ + { + id: 'parent_schema_1_id_1', + name: 'parent_schema_1_name_1', + }, + { + id: 'parent_schema_1_id_2', + name: 'parent_schema_1_name_2', + }, + ]; + + const child_schema_simple_fk_data = [ + { + id: '1', + parent_schema_1_id: 'parent_schema_1_id_1', + }, + { + id: '2', + parent_schema_1_id: 'non_existing_value_in_foreign_schema', + }, + ]; + const schemaData = { + parent_schema_1: parent_schema_1_data, + child_schema_simple_fk: child_schema_simple_fk_data, + }; + + const result = schemaService.processSchemas(schema, schemaData); + const childSchemaErrors = result['child_schema_simple_fk'].validationErrors; + + chai.expect(childSchemaErrors.length).to.eq(1); + chai.expect(childSchemaErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_FOREIGN_KEY, + message: + 'Record violates foreign key restriction defined for field(s) parent_schema_1_id. Key parent_schema_1_id: non_existing_value_in_foreign_schema is not present in schema parent_schema_1.', + fieldName: 'parent_schema_1_id', + index: 1, + info: { foreignSchema: 'parent_schema_1', value: { parent_schema_1_id: 'non_existing_value_in_foreign_schema' } }, + }); + }); + + it('should fail foreignKey restriction validation when values do not exist in foreign schema (composite fk)', () => { + const parent_schema_1_data = [ + { + id: 'parent_schema_1_id_1', + external_id: 'parent_schema_1_external_id_1', + name: 'parent_schema_1_name_1', + }, + { + id: 'parent_schema_1_id_2', + external_id: 'parent_schema_1_external_id_2', + name: 'parent_schema_1_name_2', + }, + ]; + + const child_schema_composite_fk_data = [ + { + id: '1', + parent_schema_1_id: 'parent_schema_1_id_1', + parent_schema_1_external_id: 'parent_schema_1_external_id_1', + }, + { + id: '2', + parent_schema_1_id: 'parent_schema_1_id_2', + parent_schema_1_external_id: 'non_existing_value_in_foreign_schema', + }, + ]; + const schemaData = { + parent_schema_1: parent_schema_1_data, + child_schema_composite_fk: child_schema_composite_fk_data, + }; + + const result = schemaService.processSchemas(schema, schemaData); + const childSchemaErrors = result['child_schema_composite_fk'].validationErrors; + + chai.expect(childSchemaErrors.length).to.eq(1); + chai.expect(childSchemaErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_FOREIGN_KEY, + message: + 'Record violates foreign key restriction defined for field(s) parent_schema_1_id, parent_schema_1_external_id. Key parent_schema_1_id: parent_schema_1_id_2, parent_schema_1_external_id: non_existing_value_in_foreign_schema is not present in schema parent_schema_1.', + fieldName: 'parent_schema_1_id, parent_schema_1_external_id', + index: 1, + info: { + foreignSchema: 'parent_schema_1', + value: { + parent_schema_1_external_id: 'non_existing_value_in_foreign_schema', + parent_schema_1_id: 'parent_schema_1_id_2', + }, + }, + }); + }); + + it('should fail foreignKey restriction validation when values (array) do not match in foreign schema (composite fk)', () => { + const parent_schema_2_data = [ + { + id1: ['id1_1', 'id1_2'], + id2: ['id2_1'], + }, + ]; + + const child_schema_composite_array_values_fk_data = [ + { + id: '1', + parent_schema_2_id1: ['id1_1'], + parent_schema_2_id12: ['id1_2', 'id2_1'], + }, + ]; + const schemaData = { + parent_schema_2: parent_schema_2_data, + child_schema_composite_array_values_fk: child_schema_composite_array_values_fk_data, + }; + + const result = schemaService.processSchemas(schema, schemaData); + const childSchemaErrors = result['child_schema_composite_array_values_fk'].validationErrors; + + chai.expect(childSchemaErrors.length).to.eq(1); + chai.expect(childSchemaErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_FOREIGN_KEY, + message: + 'Record violates foreign key restriction defined for field(s) parent_schema_2_id1, parent_schema_2_id12. Key parent_schema_2_id1: [id1_1], parent_schema_2_id12: [id1_2, id2_1] is not present in schema parent_schema_2.', + fieldName: 'parent_schema_2_id1, parent_schema_2_id12', + index: 0, + info: { + foreignSchema: 'parent_schema_2', + value: { + parent_schema_2_id1: ['id1_1'], + parent_schema_2_id12: ['id1_2', 'id2_1'], + }, + }, + }); + }); + + it('should pass uniqueKey restriction validation when only a record exists', () => { + const result = schemaService.processRecords(schema, 'unique_key_schema', [ + { + numeric_id_1: '1', + string_id_2: 'string_value', + array_string_id_3: ['array_element_1', 'array_element_2'], + }, + ]); + + chai.expect(result.validationErrors.length).to.eq(0); + }); + + it('should pass uniqueKey restriction validation when values are unique', () => { + const result = schemaService.processRecords(schema, 'unique_key_schema', [ + { + numeric_id_1: '1', + string_id_2: 'string_value', + array_string_id_3: ['array_element_1', 'array_element_2'], + }, + { + numeric_id_1: '1', + string_id_2: 'string_value', + array_string_id_3: ['array_element_1', 'array_element_x'], + }, + ]); + + chai.expect(result.validationErrors.length).to.eq(0); + }); + + it('should fail uniqueKey restriction validation when missing values are part of the key and they are not unique', () => { + const result = schemaService.processRecords(schema, 'unique_key_schema', [ + { + numeric_id_1: '', + string_id_2: '', + array_string_id_3: [], + }, + { + numeric_id_1: '', + string_id_2: '', + array_string_id_3: [], + }, + ]); + + chai.expect(result.validationErrors.length).to.eq(2); + chai.expect(result.validationErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, + message: 'Key numeric_id_1: null, string_id_2: null, array_string_id_3: null must be unique.', + fieldName: 'numeric_id_1, string_id_2, array_string_id_3', + index: 0, + info: { + uniqueKeyFields: ['numeric_id_1', 'string_id_2', 'array_string_id_3'], + value: { + numeric_id_1: '', + string_id_2: '', + array_string_id_3: '', + }, + }, + }); + chai.expect(result.validationErrors[1]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, + message: 'Key numeric_id_1: null, string_id_2: null, array_string_id_3: null must be unique.', + fieldName: 'numeric_id_1, string_id_2, array_string_id_3', + index: 1, + info: { + uniqueKeyFields: ['numeric_id_1', 'string_id_2', 'array_string_id_3'], + value: { + numeric_id_1: '', + string_id_2: '', + array_string_id_3: '', + }, + }, + }); + }); + + it('should fail uniqueKey restriction validation when values are not unique', () => { + const result = schemaService.processRecords(schema, 'unique_key_schema', [ + { + numeric_id_1: '1', + string_id_2: 'string_value', + array_string_id_3: ['array_element_1', 'array_element_2'], + }, + { + numeric_id_1: '1', + string_id_2: 'string_value', + array_string_id_3: ['array_element_1', 'array_element_2'], + }, + ]); + + chai.expect(result.validationErrors.length).to.eq(2); + chai.expect(result.validationErrors[0]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, + message: + 'Key numeric_id_1: 1, string_id_2: string_value, array_string_id_3: [array_element_1, array_element_2] must be unique.', + fieldName: 'numeric_id_1, string_id_2, array_string_id_3', + index: 0, + info: { + uniqueKeyFields: ['numeric_id_1', 'string_id_2', 'array_string_id_3'], + value: { + numeric_id_1: 1, + string_id_2: 'string_value', + array_string_id_3: ['array_element_1', 'array_element_2'], + }, + }, + }); + chai.expect(result.validationErrors[1]).to.deep.eq({ + errorType: SchemaValidationErrorTypes.INVALID_BY_UNIQUE_KEY, + message: + 'Key numeric_id_1: 1, string_id_2: string_value, array_string_id_3: [array_element_1, array_element_2] must be unique.', + fieldName: 'numeric_id_1, string_id_2, array_string_id_3', + index: 1, + info: { + uniqueKeyFields: ['numeric_id_1', 'string_id_2', 'array_string_id_3'], + value: { + numeric_id_1: 1, + string_id_2: 'string_value', + array_string_id_3: ['array_element_1', 'array_element_2'], + }, + }, + }); + }); +}); + +const records = [ + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, + { + country: 'US', + postal_code: '12', + }, + { + country: 'CANADA', + postal_code: 'ABC', + }, + { + country: 'US', + postal_code: '15523', + }, +]; diff --git a/packages/client/test/schema.json b/packages/client/test/schema.json new file mode 100644 index 0000000..19411a6 --- /dev/null +++ b/packages/client/test/schema.json @@ -0,0 +1,522 @@ +[ + { + "schemas": [ + { + "name": "registration", + "description": "TSV for Registration of Donor-Specimen-Sample", + "key": "submitter_donor_id", + "fields": [ + { + "name": "program_id", + "valueType": "string", + "description": "Unique identifier for program", + "meta": { + "key": true, + "examples": "PACA-CA, BASHAR-LA" + }, + "restrictions": { + "required": true, + "regex": "^[A-Z1-9][-_A-Z1-9]{2,7}(-[A-Z][A-Z])$" + } + }, + { + "name": "submitter_donor_id", + "valueType": "string", + "description": "Unique identifier for donor, assigned by the data provider.", + "meta": { + "key": true + }, + "restrictions": { + "required": true, + "regex": "^(?!(DO|do)).+" + } + }, + { + "name": "gender", + "valueType": "string", + "description": "The gender of the patient", + "meta": { + "default": "Other" + }, + "restrictions": { + "required": true, + "codeList": ["Male", "Female", "Other"] + } + }, + { + "name": "submitter_specimen_id", + "valueType": "string", + "description": "Submitter assigned specimen id", + "meta": { + "key": true + }, + "restrictions": { + "required": true, + "regex": "^(?!(SP|sp)).+" + } + }, + { + "name": "specimen_type", + "valueType": "string", + "description": "Indicate the tissue source of the biospecimen", + "meta": { + "default": "Other" + }, + "restrictions": { + "required": true, + "codeList": [ + "Blood derived", + "Blood derived - bone marrow", + "Blood derived - peripheral blood", + "Bone marrow", + "Buccal cell", + "Lymph node", + "Solid tissue", + "Plasma", + "Serum", + "Urine", + "Cerebrospinal fluid", + "Sputum", + "NOS (Not otherwise specified)", + "Other", + "FFPE", + "Pleural effusion", + "Mononuclear cells from bone marrow", + "Saliva", + "Skin" + ] + } + }, + { + "name": "tumour_normal_designation", + "valueType": "string", + "description": "Indicate whether specimen is tumour or normal type", + "restrictions": { + "required": true, + "codeList": [ + "Normal", + "Normal - tissue adjacent to primary tumour", + "Primary tumour", + "Primary tumour - adjacent to normal", + "Primary tumour - additional new primary", + "Recurrent tumour", + "Metastatic tumour", + "Metastatic tumour - metastasis local to lymph node", + "Metastatic tumour - metastasis to distant location", + "Metastatic tumour - additional metastatic", + "Xenograft - derived from primary tumour", + "Xenograft - derived from tumour cell line", + "Cell line - derived from xenograft tissue", + "Cell line - derived from tumour", + "Cell line - derived from normal" + ] + } + }, + { + "name": "submitter_sample_id", + "valueType": "string", + "description": "Submitter assigned sample id", + "restrictions": { + "required": true, + "regex": "^(?!(SA|sa)).+" + } + }, + { + "name": "sample_type", + "valueType": "string", + "description": "Specimen Type", + "restrictions": { + "required": true, + "codeList": [ + "Total DNA", + "Amplified DNA", + "ctDNA", + "other DNA enrichments", + "Total RNA", + "Ribo-Zero RNA", + "polyA+ RNA", + "other RNA fractions" + ] + } + } + ] + }, + { + "name": "address", + "description": "adderss schema", + "fields": [ + { + "name": "postal_code", + "valueType": "string", + "description": "postal code", + "restrictions": { + "required": true, + "script": "/** important to return the result object here here */\r\n(function validate(inputs) {\r\n const {$row, $field, $name} = inputs; var person = $row;\r\nvar postalCode = $field; var result = { valid: true, message: \"ok\"};\r\n\r\n /* custom logic start */\r\n if (person.country === \"US\") {\r\n var valid = /^[0-9]{5}(?:-[0-9]{4})?$/.test(postalCode);\r\n if (!valid) {\r\n result.valid = false;\r\n result.message = \"invalid postal code for US\";\r\n }\r\n } else if (person.country === \"CANADA\") {\r\n var valid = /^[A-Za-z]\\d[A-Za-z][ -]?\\d[A-Za-z]\\d$/.test(postalCode);\r\n if (!valid) {\r\n result.valid = false;\r\n result.message = \"invalid postal code for CANADA\";\r\n }\r\n }\r\n /* custom logic end */\r\n\r\n return result;\r\n})\r\n\r\n" + } + }, + { + "name": "unit_number", + "valueType": "integer", + "description": "unit number", + "restrictions": { + "range": { + "min": 0, + "exclusiveMax": 999 + } + } + }, + { + "name": "country", + "valueType": "string", + "description": "Country", + "restrictions": { + "required": true, + "codeList": ["US", "CANADA"] + } + } + ] + }, + { + "name": "donor", + "description": "TSV for donor", + "key": "submitter_donor_id", + "fields": [ + { + "name": "program_id", + "valueType": "string", + "description": "Unique identifier for program", + "meta": { + "key": true + }, + "restrictions": { + "required": true, + "regex": "^[A-Z1-9][-_A-Z1-9]{2,7}(-[A-Z][A-Z])$" + } + }, + { + "name": "submitter_donor_id", + "valueType": "string", + "description": "Unique identifier for donor, assigned by the data provider.", + "meta": { + "key": true + }, + "restrictions": { + "required": true, + "regex": "^(?!(DO|do)).+" + } + }, + { + "name": "gender", + "valueType": "string", + "description": "The gender of the patient", + "meta": { + "default": "Other" + }, + "restrictions": { + "required": true, + "codeList": ["Male", "Female", "Other"] + } + }, + { + "name": "ethnicity", + "valueType": "string", + "description": "The ethnicity of the patient", + "restrictions": { + "required": true, + "codeList": ["asian", "black or african american", "caucasian", "not reported"] + } + }, + { + "name": "vital_status", + "valueType": "string", + "description": "Indicate the vital status of the patient", + "restrictions": { + "required": true, + "codeList": ["alive", "deceased"] + } + }, + { + "name": "cause_of_death", + "valueType": "string", + "description": "Indicate the cause of death of patient", + "restrictions": { + "required": false, + "codeList": ["died of cancer", "died of other reasons", "N/A"] + } + }, + { + "name": "survival_time", + "valueType": "integer", + "description": "Survival time", + "restrictions": { + "required": false + } + } + ] + }, + { + "name": "favorite_things", + "description": "favorite things listed", + "fields": [ + { + "name": "id", + "valueType": "string", + "description": "Favourite id values", + "restrictions": { + "required": true, + "regex": "^[A-Z1-9][-_A-Z1-9]{2,7}$" + } + }, + { + "name": "qWords", + "valueType": "string", + "description": "Words starting with q", + "restrictions": { + "required": false, + "regex": "^q.*$" + }, + "isArray": true + }, + { + "name": "qWord", + "valueType": "string", + "description": "Word starting with q", + "restrictions": { + "required": false, + "regex": "^q.*$" + }, + "isArray": false + }, + { + "name": "fruit", + "valueType": "string", + "description": "fruit", + "restrictions": { + "required": false, + "codeList": ["Mango", "Orange", "None"] + }, + "isArray": true + }, + { + "name": "fruit_single_value", + "valueType": "string", + "description": "fruit", + "restrictions": { + "required": false, + "codeList": ["Mango", "Orange", "None"] + }, + "isArray": false + }, + { + "name": "animal", + "valueType": "string", + "description": "animal", + "restrictions": { + "required": false, + "codeList": ["Dog", "Cat", "None"] + }, + "isArray": true + }, + { + "name": "fraction", + "valueType": "number", + "description": "numbers between 0 and 1 exclusive", + "restrictions": { "required": false, "range": { "max": 1, "exclusiveMin": 0 } }, + "isArray": true + }, + { + "name": "integers", + "valueType": "integer", + "description": "integers between -10 and 10", + "restrictions": { "required": false, "range": { "max": 10, "min": -10 } }, + "isArray": true + }, + { + "name": "unique_value", + "valueType": "string", + "description": "unique value", + "restrictions": { "required": false, "unique": true }, + "isArray": true + } + ] + }, + { + "name": "parent_schema_1", + "description": "Parent schema 1. Used to test relational validations", + "fields": [ + { + "name": "id", + "valueType": "string", + "description": "Id" + }, + { + "name": "external_id", + "valueType": "string", + "description": "External Id" + }, + { + "name": "name", + "valueType": "string", + "description": "Name" + } + ] + }, + { + "name": "parent_schema_2", + "description": "Parent schema 1. Used to test relational validations", + "fields": [ + { + "name": "id1", + "valueType": "string", + "description": "Id 1", + "isArray": true + }, + { + "name": "id2", + "valueType": "string", + "description": "Id 2", + "isArray": true + } + ] + }, + { + "name": "child_schema_simple_fk", + "description": "Child schema referencing a field in a foreign schema", + "restrictions": { + "foreignKey": [ + { + "schema": "parent_schema_1", + "mappings": [ + { + "local": "parent_schema_1_id", + "foreign": "id" + } + ] + } + ] + }, + "fields": [ + { + "name": "id", + "valueType": "number", + "description": "Id" + }, + { + "name": "parent_schema_1_id", + "valueType": "string", + "description": "Reference to id in schema parent_schema_1" + } + ] + }, + { + "name": "child_schema_composite_fk", + "description": "Child schema referencing several fields in a foreign schema", + "restrictions": { + "foreignKey": [ + { + "schema": "parent_schema_1", + "mappings": [ + { + "local": "parent_schema_1_id", + "foreign": "id" + }, + { + "local": "parent_schema_1_external_id", + "foreign": "external_id" + } + ] + } + ] + }, + "fields": [ + { + "name": "id", + "valueType": "number", + "description": "Id" + }, + { + "name": "parent_schema_1_id", + "valueType": "string", + "description": "Reference to id in schema parent_schema_1" + }, + { + "name": "parent_schema_1_external_id", + "valueType": "string", + "description": "Reference to external id in schema parent_schema_1" + } + ] + }, + { + "name": "child_schema_composite_array_values_fk", + "description": "Child schema referencing several fields in a foreign schema", + "restrictions": { + "foreignKey": [ + { + "schema": "parent_schema_2", + "mappings": [ + { + "local": "parent_schema_2_id1", + "foreign": "id1" + }, + { + "local": "parent_schema_2_id12", + "foreign": "id2" + } + ] + } + ] + }, + "fields": [ + { + "name": "id", + "valueType": "number", + "description": "Id" + }, + { + "name": "parent_schema_2_id1", + "valueType": "string", + "description": "Reference to id1 in schema parent_schema_2", + "isArray": true + }, + { + "name": "parent_schema_2_id12", + "valueType": "string", + "description": "Reference to external id2 in schema parent_schema_2", + "isArray": true + } + ] + }, + { + "name": "unique_key_schema", + "description": "Schema to test uniqueKey restriction", + "restrictions": { + "uniqueKey": ["numeric_id_1", "string_id_2", "array_string_id_3"] + }, + "fields": [ + { + "name": "numeric_id_1", + "valueType": "number", + "description": "Id 1. Numeric value as part of a composite unique key" + }, + { + "name": "string_id_2", + "valueType": "string", + "description": "Id 2. String value as part of a composite unique key" + }, + { + "name": "array_string_id_3", + "valueType": "string", + "description": "Id 3. String array as part of a composite unique key", + "isArray": true + } + ] + } + ], + "_id": "5d250369f38d1f0d9376fd38", + "name": "ARGO Clinical Submission", + "version": "1.0", + "createdAt": "2019-07-09T21:13:13.683Z", + "updatedAt": "2019-07-09T21:13:13.683Z", + "__v": 0 + } +] diff --git a/packages/client/tsconfig.json b/packages/client/tsconfig.json new file mode 100644 index 0000000..1ff397e --- /dev/null +++ b/packages/client/tsconfig.json @@ -0,0 +1,22 @@ +{ + "compilerOptions": { + "module": "CommonJS", + "target": "ESNext", + "lib": ["ESNext", "DOM"], + "strict": true, + "esModuleInterop": true, + "allowSyntheticDefaultImports": true, + "strictNullChecks": true, + "noImplicitAny": true, + "allowJs": true, + "moduleResolution": "node", + "experimentalDecorators": true, + "sourceMap": true, + "declaration": true, + "outDir": "dist", + "baseUrl": "./src", + "skipLibCheck": true + }, + "include": ["./src/**/*.ts"], + "exclude": ["node_modules/"] +} diff --git a/packages/client/tslint.json b/packages/client/tslint.json new file mode 100644 index 0000000..afe5fb9 --- /dev/null +++ b/packages/client/tslint.json @@ -0,0 +1,43 @@ +{ + "rules": { + "class-name": true, + "comment-format": [true, "check-space"], + "no-async-without-await": true, + "indent": [true, "tabs"], + "one-line": [true, "check-open-brace", "check-whitespace"], + "no-var-keyword": true, + "quotemark": [true, "single", "avoid-escape"], + "semicolon": [true, "always", "ignore-bound-class-methods"], + "whitespace": [ + true, + "check-branch", + "check-decl", + "check-operator", + "check-module", + "check-separator", + "check-type" + ], + "typedef-whitespace": [ + true, + { + "call-signature": "nospace", + "index-signature": "nospace", + "parameter": "nospace", + "property-declaration": "nospace", + "variable-declaration": "nospace" + }, + { + "call-signature": "onespace", + "index-signature": "onespace", + "parameter": "onespace", + "property-declaration": "onespace", + "variable-declaration": "onespace" + } + ], + "no-internal-module": true, + "no-trailing-whitespace": true, + "no-null-keyword": true, + "prefer-const": true, + "jsdoc-format": true + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d077607..48ca307 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -158,7 +158,11 @@ importers: specifier: ^3.21.3 version: 3.21.3(zod@3.21.4) - libraries/common: {} + libraries/common: + devDependencies: + rimraf: + specifier: ^3.0.2 + version: 3.0.2 libraries/dictionary: dependencies: @@ -178,6 +182,85 @@ importers: '@types/lodash': specifier: ^4.14.195 version: 4.14.195 + rimraf: + specifier: ^3.0.2 + version: 3.0.2 + + packages/client: + dependencies: + cd: + specifier: ^0.3.3 + version: 0.3.3 + common: + specifier: workspace:^ + version: link:../../libraries/common + deep-freeze: + specifier: ^0.0.1 + version: 0.0.1 + lodash: + specifier: ^4.17.21 + version: 4.17.21 + node-fetch: + specifier: ^2.6.1 + version: 2.7.0 + node-worker-threads-pool: + specifier: ^1.4.3 + version: 1.5.1 + promise-tools: + specifier: ^2.1.0 + version: 2.1.0 + winston: + specifier: ^3.3.3 + version: 3.9.0 + devDependencies: + '@types/chai': + specifier: ^4.2.16 + version: 4.3.5 + '@types/deep-freeze': + specifier: ^0.1.2 + version: 0.1.5 + '@types/lodash': + specifier: ^4.14.195 + version: 4.14.195 + '@types/mocha': + specifier: ^8.2.2 + version: 8.2.3 + '@types/node': + specifier: ^12.0.10 + version: 12.20.55 + '@types/node-fetch': + specifier: ^2.5.10 + version: 2.6.11 + chai: + specifier: ^4.3.4 + version: 4.3.7 + husky: + specifier: ^6.0.0 + version: 6.0.0 + mocha: + specifier: ^8.3.2 + version: 8.4.0 + prettier: + specifier: ^2.2.1 + version: 2.8.8 + pretty-quick: + specifier: ^3.1.0 + version: 3.3.1(prettier@2.8.8) + rimraf: + specifier: ^3.0.2 + version: 3.0.2 + ts-node: + specifier: ^9.1.1 + version: 9.1.1(typescript@5.1.6) + tslint: + specifier: ^6.1.3 + version: 6.1.3(typescript@5.1.6) + typedoc: + specifier: ^0.17.7 + version: 0.17.8(typescript@5.1.6) + typescript: + specifier: ^5.1.6 + version: 5.1.6 packages: @@ -186,7 +269,7 @@ packages: engines: {node: '>=6.0.0'} dependencies: '@jridgewell/gen-mapping': 0.3.3 - '@jridgewell/trace-mapping': 0.3.9 + '@jridgewell/trace-mapping': 0.3.18 dev: true /@babel/code-frame@7.22.5: @@ -444,6 +527,7 @@ packages: /@jridgewell/resolve-uri@3.1.1: resolution: {integrity: sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA==} engines: {node: '>=6.0.0'} + dev: false /@jridgewell/set-array@1.1.2: resolution: {integrity: sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==} @@ -469,6 +553,7 @@ packages: dependencies: '@jridgewell/resolve-uri': 3.1.1 '@jridgewell/sourcemap-codec': 1.4.15 + dev: false /@nicolo-ribaudo/semver-v6@6.3.3: resolution: {integrity: sha512-3Yc1fUTs69MG/uZbJlLSI3JISMn2UV2rg+1D/vROUqZyh3l6iYHCs7GMp+M40ZD7yOdDbYjJcU1oTJhrc+dGKg==} @@ -679,6 +764,10 @@ packages: resolution: {integrity: sha512-t73xJJrvdTjXrn4jLS9VSGRbz0nUY3cl2DMGDU48lKl+HR9dbbjW2A9r3g40VA++mQpy6uuHg33gy7du2BKpog==} dev: true + /@types/deep-freeze@0.1.5: + resolution: {integrity: sha512-KZtR+jtmgkCpgE0f+We/QEI2Fi0towBV/tTkvHVhMzx+qhUVGXMx7pWvAtDp6vEWIjdKLTKpqbI/sORRCo8TKg==} + dev: true + /@types/errorhandler@0.0.32: resolution: {integrity: sha512-wC9CfwPMIzklPd5lEYC8HnQdlMC1PswlohWmEDMWlw+E/rMYuz5eSqKBc72Earb29KptKJrRl77qVRJzrZndww==} dependencies: @@ -733,10 +822,25 @@ packages: resolution: {integrity: sha512-rADY+HtTOA52l9VZWtgQfn4p+UDVM2eDVkMZT1I6syp0YKxW2F9v+0pbRZLsvskhQv/vMb6ZfCay81GHbz5SHg==} dev: true + /@types/mocha@8.2.3: + resolution: {integrity: sha512-ekGvFhFgrc2zYQoX4JeZPmVzZxw6Dtllga7iGHzfbYIYkAMUx/sAFP2GdFpLff+vdHXu5fl7WX9AT+TtqYcsyw==} + dev: true + /@types/ms@0.7.31: resolution: {integrity: sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==} dev: true + /@types/node-fetch@2.6.11: + resolution: {integrity: sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g==} + dependencies: + '@types/node': 12.20.55 + form-data: 4.0.0 + dev: true + + /@types/node@12.20.55: + resolution: {integrity: sha512-J8xLz7q2OFulZ2cyGTLE1TbbZcjpno7FaN6zdJNrgAdrJ+DZzh/uFR6YrTb4C+nXakvud8Q4+rbhoIWlYQbUFQ==} + dev: true + /@types/node@20.4.1: resolution: {integrity: sha512-JIzsAvJeA/5iY6Y/OxZbv1lUcc8dNSE77lb2gnBH+/PJ3lFR1Ccvgwl5JWnHAkNHcRsT0TbpVOsiMKZ1F/yyJg==} @@ -903,6 +1007,11 @@ packages: engines: {node: '>=6'} dev: true + /ansi-regex@3.0.1: + resolution: {integrity: sha512-+O9Jct8wf++lXxxFc4hc8LsjaSq0HFzzL7cVsw8pRDIPdjKD2mT4ytDZlLuSBZ4cLKZFXIrMGO7DbQCtMJJMKw==} + engines: {node: '>=4'} + dev: true + /ansi-regex@4.1.1: resolution: {integrity: sha512-ILlv4k/3f6vfQ4OoP2AGvirOktlQ98ZEL1k9FaQjxa3L1abBgbuTDAdPOpvbGncC0BTVQrl+OM8xZGK6tWXt7g==} engines: {node: '>=6'} @@ -952,7 +1061,6 @@ packages: /arg@4.1.3: resolution: {integrity: sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==} - dev: false /argparse@1.0.10: resolution: {integrity: sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==} @@ -1155,6 +1263,11 @@ packages: ieee754: 1.2.1 dev: true + /builtin-modules@1.1.1: + resolution: {integrity: sha512-wxXCdllwGhI2kCC0MnvTGYTMvnVZTvqgypkiTI8Pa5tcz2i6VqsqwYGgqwXji+4RgCzms6EajE4IxiUH6HH8nQ==} + engines: {node: '>=0.10.0'} + dev: true + /byline@5.0.0: resolution: {integrity: sha512-s6webAy+R4SR8XVuJWt2V2rGvhnrhxN+9S15GNuTK3wKPOXFF6RNc+8ug2XhH+2s4f+uudG4kUVYmYOQWL2g0Q==} engines: {node: '>=0.10.0'} @@ -1218,6 +1331,10 @@ packages: resolution: {integrity: sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==} dev: false + /cd@0.3.3: + resolution: {integrity: sha512-X2y0Ssu48ucdkrNgCdg6k3EZWjWVy/dsEywUUTeZEIW31f3bQfq65Svm+TzU1Hz+qqhdmyCdjGhUvRsSKHl/mw==} + dev: false + /chai-as-promised@7.1.1(chai@4.3.7): resolution: {integrity: sha512-azL6xMoi+uxu6z4rhWQ1jbdUhOMhis2PvscD/xjLqNMkv3BPPp2JyyuTHOrf9BOosGpNQ11v6BKv/g57RXbiaA==} peerDependencies: @@ -1282,6 +1399,21 @@ packages: resolution: {integrity: sha512-BrgHpW9NURQgzoNyjfq0Wu6VFO6D7IZEmJNdtgNqpzGG8RuNFHt2jQxWlAs4HMe119chBnv+34syEZtc6IhLtA==} dev: true + /chokidar@3.5.1: + resolution: {integrity: sha512-9+s+Od+W0VJJzawDma/gvBNQqkTiqYTWLuZoyAsivsI4AaWTCzHG06/TMjsf1cYe9Cb97UCEhjz7HvnPk2p/tw==} + engines: {node: '>= 8.10.0'} + dependencies: + anymatch: 3.1.3 + braces: 3.0.2 + glob-parent: 5.1.2 + is-binary-path: 2.1.0 + is-glob: 4.0.3 + normalize-path: 3.0.0 + readdirp: 3.5.0 + optionalDependencies: + fsevents: 2.3.2 + dev: true + /chokidar@3.5.3: resolution: {integrity: sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==} engines: {node: '>= 8.10.0'} @@ -1400,6 +1532,10 @@ packages: dependencies: delayed-stream: 1.0.0 + /commander@2.20.3: + resolution: {integrity: sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==} + dev: true + /commondir@1.0.1: resolution: {integrity: sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==} dev: true @@ -1487,7 +1623,6 @@ packages: /create-require@1.1.1: resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==} - dev: false /cross-spawn@6.0.5: resolution: {integrity: sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==} @@ -1564,6 +1699,19 @@ packages: supports-color: 5.5.0 dev: true + /debug@4.3.1(supports-color@8.1.1): + resolution: {integrity: sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + dependencies: + ms: 2.1.2 + supports-color: 8.1.1 + dev: true + /debug@4.3.4(supports-color@8.1.1): resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==} engines: {node: '>=6.0'} @@ -1593,6 +1741,10 @@ packages: type-detect: 4.0.8 dev: true + /deep-freeze@0.0.1: + resolution: {integrity: sha512-Z+z8HiAvsGwmjqlphnHW5oz6yWlOwu6EQfFTjmeTWlDeda3FS2yv3jhq35TX/ewmsnqB+RX2IdsIOyjJCQN5tg==} + dev: false + /default-require-extensions@3.0.1: resolution: {integrity: sha512-eXTJmRbm2TIt9MgWTsOH1wEuhew6XGZcMeGKCtLedIg/NCsg1iBePXkceTdK4Fii7pzmN9tGsZhKzZ4h7O/fxw==} engines: {node: '>=8'} @@ -1629,7 +1781,6 @@ packages: /diff@4.0.2: resolution: {integrity: sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==} engines: {node: '>=0.3.1'} - dev: false /diff@5.0.0: resolution: {integrity: sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w==} @@ -1831,6 +1982,21 @@ packages: strip-eof: 1.0.0 dev: true + /execa@4.1.0: + resolution: {integrity: sha512-j5W0//W7f8UxAn8hXVnwG8tLwdiUy4FJLcSupCg6maBYZDpyBvTApK7KyuI4bKj8KOh1r2YH+6ucuYtJv1bTZA==} + engines: {node: '>=10'} + dependencies: + cross-spawn: 7.0.3 + get-stream: 5.2.0 + human-signals: 1.1.1 + is-stream: 2.0.1 + merge-stream: 2.0.0 + npm-run-path: 4.0.1 + onetime: 5.1.2 + signal-exit: 3.0.7 + strip-final-newline: 2.0.0 + dev: true + /express@4.18.2: resolution: {integrity: sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==} engines: {node: '>= 0.10.0'} @@ -2062,6 +2228,15 @@ packages: universalify: 2.0.0 dev: true + /fs-extra@8.1.0: + resolution: {integrity: sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==} + engines: {node: '>=6 <7 || >=8'} + dependencies: + graceful-fs: 4.2.11 + jsonfile: 4.0.0 + universalify: 0.1.2 + dev: true + /fs.realpath@1.0.0: resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==} dev: true @@ -2121,6 +2296,13 @@ packages: pump: 3.0.0 dev: true + /get-stream@5.2.0: + resolution: {integrity: sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==} + engines: {node: '>=8'} + dependencies: + pump: 3.0.0 + dev: true + /getpass@0.1.7: resolution: {integrity: sha512-0fzj9JxOLfJ+XGLhR8ze3unN0KZCgZwiSSDz168VERjK8Wl8kVSdcu2kspd4s4wtAa1y/qrVRiAA0WclVsu0ng==} dependencies: @@ -2145,8 +2327,20 @@ packages: path-is-absolute: 1.0.1 dev: true + /glob@7.1.6: + resolution: {integrity: sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==} + dependencies: + fs.realpath: 1.0.0 + inflight: 1.0.6 + inherits: 2.0.4 + minimatch: 3.1.2 + once: 1.4.0 + path-is-absolute: 1.0.1 + dev: true + /glob@7.2.0: resolution: {integrity: sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==} + deprecated: Glob versions prior to v9 are no longer supported dependencies: fs.realpath: 1.0.0 inflight: 1.0.6 @@ -2165,6 +2359,24 @@ packages: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} dev: true + /growl@1.10.5: + resolution: {integrity: sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==} + engines: {node: '>=4.x'} + dev: true + + /handlebars@4.7.8: + resolution: {integrity: sha512-vafaFqs8MZkRrSX7sFVUdo3ap/eNiLnb4IakshzvP56X5Nr1iGKAIqdX6tMlm6HcNRIkr6AxO5jFEoJzzpT8aQ==} + engines: {node: '>=0.4.7'} + hasBin: true + dependencies: + minimist: 1.2.8 + neo-async: 2.6.2 + source-map: 0.6.1 + wordwrap: 1.0.0 + optionalDependencies: + uglify-js: 3.17.4 + dev: true + /har-schema@2.0.0: resolution: {integrity: sha512-Oqluz6zhGX8cyRaTQlFMPw80bSJVG2x/cFb8ZPhUILGgHka9SsokCCOQgpveePerqidZOrT14ipqfJb7ILcW5Q==} engines: {node: '>=4'} @@ -2220,6 +2432,10 @@ packages: engines: {node: '>=8'} dev: true + /highlight.js@10.7.3: + resolution: {integrity: sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A==} + dev: true + /hosted-git-info@2.8.9: resolution: {integrity: sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==} dev: true @@ -2248,6 +2464,11 @@ packages: sshpk: 1.17.0 dev: false + /human-signals@1.1.1: + resolution: {integrity: sha512-SEQu7vl8KjNL2eoGBLF3+wAjpsNfA9XMlXAYj/3EdaNfAlxKthD1xjEQfGOUhllCGGJVNY34bRr6lPINhNjyZw==} + engines: {node: '>=8.12.0'} + dev: true + /husky@3.1.0: resolution: {integrity: sha512-FJkPoHHB+6s4a+jwPqBudBDvYZsoQW5/HBuMSehC8qDiCe50kpcxeqFoDSlow+9I6wg47YxBoT3WxaURlrDIIQ==} engines: {node: '>=8.6.0'} @@ -2267,6 +2488,11 @@ packages: slash: 3.0.0 dev: true + /husky@6.0.0: + resolution: {integrity: sha512-SQS2gDTB7tBN486QSoKPKQItZw97BMOd+Kdb6ghfpBc0yXyzrddI0oDV5MkDAbuB4X2mO3/nj60TRMcYxwzZeQ==} + hasBin: true + dev: true + /iconv-lite@0.4.24: resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==} engines: {node: '>=0.10.0'} @@ -2287,6 +2513,11 @@ packages: engines: {node: '>= 4'} dev: true + /ignore@5.3.1: + resolution: {integrity: sha512-5Fytz/IraMjqpwfd34ke28PTVMjZjJG2MPn5t7OE4eUCUNf8BAa7b5WUS9/Qvr6mwOQS7Mk6vdsMno5he+T8Xw==} + engines: {node: '>= 4'} + dev: true + /immer@10.0.2: resolution: {integrity: sha512-Rx3CqeqQ19sxUtYV9CU911Vhy8/721wRFnJv3REVGWUmoAcIwzifTsdmJte/MV+0/XpM35LZdQMBGkRIoLPwQA==} dev: false @@ -2311,6 +2542,7 @@ packages: /inflight@1.0.6: resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==} + deprecated: This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful. dependencies: once: 1.4.0 wrappy: 1.0.2 @@ -2319,6 +2551,11 @@ packages: /inherits@2.0.4: resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} + /interpret@1.4.0: + resolution: {integrity: sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==} + engines: {node: '>= 0.10'} + dev: true + /ip-regex@2.1.0: resolution: {integrity: sha512-58yWmlHpp7VYfcdTwMTvwMmqx/Elfxjd9RXTDyMsbL7lLWmhMylLEqiYVLKuLzOZqVgiWXD9MfR62Vv89VRxkw==} engines: {node: '>=4'} @@ -2529,6 +2766,13 @@ packages: esprima: 4.0.1 dev: true + /js-yaml@4.0.0: + resolution: {integrity: sha512-pqon0s+4ScYUvX30wxQi3PogGFAlUyH0awepWvwkj4jD4v+ova3RiYw8bmA6x2rDrEaj8i/oWKoRxpVNW+Re8Q==} + hasBin: true + dependencies: + argparse: 2.0.1 + dev: true + /js-yaml@4.1.0: resolution: {integrity: sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==} hasBin: true @@ -2580,6 +2824,12 @@ packages: resolution: {integrity: sha512-gfFQZrcTc8CnKXp6Y4/CBT3fTc0OVuDofpre4aEeEpSBPV5X5v4+Vmx+8snU7RLPrNHPKSgLxGo9YuQzz20o+w==} dev: true + /jsonfile@4.0.0: + resolution: {integrity: sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==} + optionalDependencies: + graceful-fs: 4.2.11 + dev: true + /jsonfile@6.1.0: resolution: {integrity: sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==} dependencies: @@ -2717,6 +2967,13 @@ packages: /lodash@4.17.21: resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==} + /log-symbols@4.0.0: + resolution: {integrity: sha512-FN8JBzLx6CzeMrB0tg6pqlGU1wCrXW+ZXGH481kfsBqer0hToTIiHdjH4Mq8xJUbvATujKCvaREGWpGUionraA==} + engines: {node: '>=10'} + dependencies: + chalk: 4.1.2 + dev: true + /log-symbols@4.1.0: resolution: {integrity: sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==} engines: {node: '>=10'} @@ -2761,6 +3018,10 @@ packages: es5-ext: 0.10.62 dev: false + /lunr@2.3.9: + resolution: {integrity: sha512-zTU3DaZaF3Rt9rhN3uBMGQD3dD2/vFQqnvZCDv4dl5iOzq2IZQqTxu90r4E5J+nP70J3ilqVCrbho2eWaeW8Ow==} + dev: true + /make-dir@3.1.0: resolution: {integrity: sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==} engines: {node: '>=8'} @@ -2770,7 +3031,12 @@ packages: /make-error@1.3.6: resolution: {integrity: sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==} - dev: false + + /marked@1.0.0: + resolution: {integrity: sha512-Wo+L1pWTVibfrSr+TTtMuiMfNzmZWiOPeO7rZsQUY5bgsxpHesBEcIWJloWVTFnrMXnf/TL30eTFSGJddmQAng==} + engines: {node: '>= 8.16.2'} + hasBin: true + dev: true /media-typer@0.3.0: resolution: {integrity: sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=} @@ -2792,6 +3058,7 @@ packages: /memory-pager@1.5.0: resolution: {integrity: sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==} + requiresBuild: true dev: false optional: true @@ -2799,6 +3066,10 @@ packages: resolution: {integrity: sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=} dev: false + /merge-stream@2.0.0: + resolution: {integrity: sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==} + dev: true + /merge2@1.4.1: resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==} engines: {node: '>= 8'} @@ -2843,6 +3114,12 @@ packages: engines: {node: '>=6'} dev: true + /minimatch@3.0.4: + resolution: {integrity: sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==} + dependencies: + brace-expansion: 1.1.11 + dev: true + /minimatch@3.0.5: resolution: {integrity: sha512-tUpxzX0VAzJHjLu0xUfFv1gwVp9ba3IOuRAVH2EGuRW8a5emA2FlACLqiT/lDVtS1W+TGNwqz3sWaNyLgDJWuw==} dependencies: @@ -2906,6 +3183,38 @@ packages: yargs-unparser: 2.0.0 dev: true + /mocha@8.4.0: + resolution: {integrity: sha512-hJaO0mwDXmZS4ghXsvPVriOhsxQ7ofcpQdm8dE+jISUOKopitvnXFQmpRR7jd2K6VBG6E26gU3IAbXXGIbu4sQ==} + engines: {node: '>= 10.12.0'} + hasBin: true + dependencies: + '@ungap/promise-all-settled': 1.1.2 + ansi-colors: 4.1.1 + browser-stdout: 1.3.1 + chokidar: 3.5.1 + debug: 4.3.1(supports-color@8.1.1) + diff: 5.0.0 + escape-string-regexp: 4.0.0 + find-up: 5.0.0 + glob: 7.1.6 + growl: 1.10.5 + he: 1.2.0 + js-yaml: 4.0.0 + log-symbols: 4.0.0 + minimatch: 3.0.4 + ms: 2.1.3 + nanoid: 3.1.20 + serialize-javascript: 5.0.1 + strip-json-comments: 3.1.1 + supports-color: 8.1.1 + which: 2.0.2 + wide-align: 1.1.3 + workerpool: 6.1.0 + yargs: 16.2.0 + yargs-parser: 20.2.4 + yargs-unparser: 2.0.0 + dev: true + /mongodb-connection-string-url@2.6.0: resolution: {integrity: sha512-WvTZlI9ab0QYtTYnuMLgobULWhokRjtC7db9LtcVfJ+Hsnyr5eo6ZtNAt3Ly24XZScGMelOcGtm7lSn0332tPQ==} dependencies: @@ -2967,6 +3276,11 @@ packages: - supports-color dev: false + /mri@1.2.0: + resolution: {integrity: sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==} + engines: {node: '>=4'} + dev: true + /ms@2.0.0: resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==} dev: false @@ -2983,6 +3297,12 @@ packages: hasBin: true dev: false + /nanoid@3.1.20: + resolution: {integrity: sha512-a1cQNyczgKbLX9jwbS/+d7W8fX/RfgYR7lVWwWOGIPNgK2m0MWvrGF6/m4kk6U3QcFMnZf3RIhL0v2Jgh/0Uxw==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + dev: true + /nanoid@3.3.3: resolution: {integrity: sha512-p1sjXuopFs0xg+fPASzQ28agW1oHD7xDsd9Xkf3T15H3c/cifrFHVwrh74PdoklAPi+i7MdRsE47vm2r6JoB+w==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} @@ -2994,6 +3314,10 @@ packages: engines: {node: '>= 0.6'} dev: false + /neo-async@2.6.2: + resolution: {integrity: sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==} + dev: true + /next-tick@1.1.0: resolution: {integrity: sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ==} dev: false @@ -3020,6 +3344,18 @@ packages: resolution: {integrity: sha512-eUXYNSY7DL53vqfTosggWkvyIW3bhAcqBDIlolgNYlZhianXTrCL50rlUJWD1eRqkIxMppXTfiFbp+9SjpPrgA==} dev: true + /node-fetch@2.7.0: + resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==} + engines: {node: 4.x || >=6.0.0} + peerDependencies: + encoding: ^0.1.0 + peerDependenciesMeta: + encoding: + optional: true + dependencies: + whatwg-url: 5.0.0 + dev: false + /node-gyp-build@4.6.0: resolution: {integrity: sha512-NTZVKn9IylLwUzaKjkas1e4u2DLNcV4rdYagA4PWdPwW87Bi7z+BznyKSRwS/761tV/lzCGXplWsiaMjLqP2zQ==} hasBin: true @@ -3048,6 +3384,10 @@ packages: - supports-color dev: false + /node-worker-threads-pool@1.5.1: + resolution: {integrity: sha512-7TXAhpMm+jO4MfESxYLtMGSnJWv+itdNHMdaFmeZuPXxwFGU90mtEB42BciUULXOUAxYBfXILAuvrSG3rQZ7mw==} + dev: false + /nodemon@2.0.22: resolution: {integrity: sha512-B8YqaKMmyuCO7BowF1Z1/mkPqLk6cs/l63Ojtd6otKjMx47Dq1utxfRxcavH1I7VSaL8n5BUaoutadnsX3AAVQ==} engines: {node: '>=8.10.0'} @@ -3386,6 +3726,11 @@ packages: engines: {node: '>=8.6'} dev: true + /picomatch@3.0.1: + resolution: {integrity: sha512-I3EurrIQMlRc9IaAZnqRR044Phh2DXY+55o7uJ0V+hYZAcQYSuFWsc9q5PvyDHUSCe1Qxn/iBz+78s86zWnGag==} + engines: {node: '>=10'} + dev: true + /pify@3.0.0: resolution: {integrity: sha512-C3FsVNH1udSEX48gGX1xfvwTWfsYWj5U+8/uK15BGzIGrKoUpghX8hWZwa/OFnakBiiVNmBvemTJR5mcy7iPcg==} engines: {node: '>=4'} @@ -3404,12 +3749,35 @@ packages: semver-compare: 1.0.0 dev: true + /prettier@2.8.8: + resolution: {integrity: sha512-tdN8qQGvNjw4CHbY+XXk0JgCXn9QiF21a55rBe5LJAU+kDyC4WQn4+awm2Xfk2lQMk5fKup9XgzTZtGkjBdP9Q==} + engines: {node: '>=10.13.0'} + hasBin: true + dev: true + /prettier@3.0.0: resolution: {integrity: sha512-zBf5eHpwHOGPC47h0zrPyNn+eAEIdEzfywMoYn2XPi0P44Zp0tSq64rq0xAREh4auw2cJZHo9QUob+NqCQky4g==} engines: {node: '>=14'} hasBin: true dev: true + /pretty-quick@3.3.1(prettier@2.8.8): + resolution: {integrity: sha512-3b36UXfYQ+IXXqex6mCca89jC8u0mYLqFAN5eTQKoXO6oCQYcIVYZEB/5AlBHI7JPYygReM2Vv6Vom/Gln7fBg==} + engines: {node: '>=10.13'} + hasBin: true + peerDependencies: + prettier: ^2.0.0 + dependencies: + execa: 4.1.0 + find-up: 4.1.0 + ignore: 5.3.1 + mri: 1.2.0 + picocolors: 1.0.0 + picomatch: 3.0.1 + prettier: 2.8.8 + tslib: 2.6.2 + dev: true + /process-nextick-args@2.0.1: resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==} dev: true @@ -3421,6 +3789,15 @@ packages: fromentries: 1.3.2 dev: true + /progress@2.0.3: + resolution: {integrity: sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==} + engines: {node: '>=0.4.0'} + dev: true + + /promise-tools@2.1.0: + resolution: {integrity: sha512-/k0nnTyAJjuMfIgAbIsoqkrfQf1U+dZapfsLkFKafc0WvmFfHWjnYdUBlWxmipqRetiUHCor+KPoAChHdXpjow==} + dev: false + /proxy-addr@2.0.7: resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} engines: {node: '>= 0.10'} @@ -3565,6 +3942,13 @@ packages: string_decoder: 1.3.0 util-deprecate: 1.0.2 + /readdirp@3.5.0: + resolution: {integrity: sha512-cMhu7c/8rdhkHXWsY+osBhfSy0JikwpHK/5+imo+LpeasTF8ouErHrlYkwT0++njiyuDvc7OFY5T3ukvZ8qmFQ==} + engines: {node: '>=8.10.0'} + dependencies: + picomatch: 2.3.1 + dev: true + /readdirp@3.6.0: resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==} engines: {node: '>=8.10.0'} @@ -3572,6 +3956,13 @@ packages: picomatch: 2.3.1 dev: true + /rechoir@0.6.2: + resolution: {integrity: sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==} + engines: {node: '>= 0.10'} + dependencies: + resolve: 1.22.2 + dev: true + /regenerator-runtime@0.13.11: resolution: {integrity: sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==} dev: true @@ -3681,6 +4072,7 @@ packages: /rimraf@3.0.2: resolution: {integrity: sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==} + deprecated: Rimraf versions prior to v4 are no longer supported hasBin: true dependencies: glob: 7.2.0 @@ -3785,6 +4177,12 @@ packages: - supports-color dev: false + /serialize-javascript@5.0.1: + resolution: {integrity: sha512-SaaNal9imEO737H2c05Og0/8LUXG7EnsZyMa8MzkmuHoELfT6txuj0cMqRj6zfPKnmQ1yasR4PCJc8x+M4JSPA==} + dependencies: + randombytes: 2.1.0 + dev: true + /serialize-javascript@6.0.0: resolution: {integrity: sha512-Qr3TosvguFt8ePWqsvRfrKyQXIiW+nGbYpy8XK24NQHE83caxWt+mIymTT19DGFbNWNLfEwsrkSmN64lVWB9ag==} dependencies: @@ -3835,6 +4233,16 @@ packages: engines: {node: '>=8'} dev: true + /shelljs@0.8.5: + resolution: {integrity: sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==} + engines: {node: '>=4'} + hasBin: true + dependencies: + glob: 7.2.0 + interpret: 1.4.0 + rechoir: 0.6.2 + dev: true + /side-channel@1.0.4: resolution: {integrity: sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==} dependencies: @@ -3892,6 +4300,13 @@ packages: smart-buffer: 4.2.0 dev: false + /source-map-support@0.5.21: + resolution: {integrity: sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==} + dependencies: + buffer-from: 1.1.2 + source-map: 0.6.1 + dev: true + /source-map@0.6.1: resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==} engines: {node: '>=0.10.0'} @@ -3899,6 +4314,7 @@ packages: /sparse-bitfield@3.0.3: resolution: {integrity: sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==} + requiresBuild: true dependencies: memory-pager: 1.5.0 dev: false @@ -3986,6 +4402,14 @@ packages: any-promise: 1.3.0 dev: true + /string-width@2.1.1: + resolution: {integrity: sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==} + engines: {node: '>=4'} + dependencies: + is-fullwidth-code-point: 2.0.0 + strip-ansi: 4.0.0 + dev: true + /string-width@3.1.0: resolution: {integrity: sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==} engines: {node: '>=6'} @@ -4019,6 +4443,13 @@ packages: dependencies: safe-buffer: 5.2.1 + /strip-ansi@4.0.0: + resolution: {integrity: sha512-4XaJ2zQdCzROZDivEVIDPkcQn8LMFSa8kj8Gxb/Lnwzv9A8VctNZ+lfivC/sV3ivW8ElJTERXZoPBRrZKkNKow==} + engines: {node: '>=4'} + dependencies: + ansi-regex: 3.0.1 + dev: true + /strip-ansi@5.2.0: resolution: {integrity: sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==} engines: {node: '>=6'} @@ -4048,6 +4479,11 @@ packages: engines: {node: '>=0.10.0'} dev: true + /strip-final-newline@2.0.0: + resolution: {integrity: sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==} + engines: {node: '>=6'} + dev: true + /strip-json-comments@3.1.1: resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} engines: {node: '>=8'} @@ -4258,6 +4694,10 @@ packages: punycode: 2.3.0 dev: false + /tr46@0.0.3: + resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} + dev: false + /tr46@3.0.0: resolution: {integrity: sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==} engines: {node: '>=12'} @@ -4305,6 +4745,22 @@ packages: yn: 3.1.1 dev: false + /ts-node@9.1.1(typescript@5.1.6): + resolution: {integrity: sha512-hPlt7ZACERQGf03M253ytLY3dHbGNGrAq9qIHWUY9XHYl1z7wYngSr3OQ5xmui8o2AaxsONxIzjafLUiWBo1Fg==} + engines: {node: '>=10.0.0'} + hasBin: true + peerDependencies: + typescript: '>=2.7' + dependencies: + arg: 4.1.3 + create-require: 1.1.1 + diff: 4.0.2 + make-error: 1.3.6 + source-map-support: 0.5.21 + typescript: 5.1.6 + yn: 3.1.1 + dev: true + /tsconfig-paths@4.2.0: resolution: {integrity: sha512-NoZ4roiN7LnbKn9QqE1amc9DJfzvZXxF4xDavcOWt1BPkdx+m+0gJuPM+S0vCe7zTJMYUP0R8pO2XMr+Y8oLIg==} engines: {node: '>=6'} @@ -4322,6 +4778,43 @@ packages: resolution: {integrity: sha512-7At1WUettjcSRHXCyYtTselblcHl9PJFFVKiCAy/bY97+BPZXSQ2wbq0P9s8tK2G7dFQfNnlJnPAiArVBVBsfA==} dev: true + /tslib@2.6.2: + resolution: {integrity: sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==} + dev: true + + /tslint@6.1.3(typescript@5.1.6): + resolution: {integrity: sha512-IbR4nkT96EQOvKE2PW/djGz8iGNeJ4rF2mBfiYaR/nvUWYKJhLwimoJKgjIFEIDibBtOevj7BqCRL4oHeWWUCg==} + engines: {node: '>=4.8.0'} + deprecated: TSLint has been deprecated in favor of ESLint. Please see https://github.com/palantir/tslint/issues/4534 for more information. + hasBin: true + peerDependencies: + typescript: '>=2.3.0-dev || >=2.4.0-dev || >=2.5.0-dev || >=2.6.0-dev || >=2.7.0-dev || >=2.8.0-dev || >=2.9.0-dev || >=3.0.0-dev || >= 3.1.0-dev || >= 3.2.0-dev || >= 4.0.0-dev' + dependencies: + '@babel/code-frame': 7.22.5 + builtin-modules: 1.1.1 + chalk: 2.4.2 + commander: 2.20.3 + diff: 4.0.2 + glob: 7.2.0 + js-yaml: 3.14.1 + minimatch: 3.1.2 + mkdirp: 0.5.6 + resolve: 1.22.2 + semver: 5.7.1 + tslib: 1.14.1 + tsutils: 2.29.0(typescript@5.1.6) + typescript: 5.1.6 + dev: true + + /tsutils@2.29.0(typescript@5.1.6): + resolution: {integrity: sha512-g5JVHCIJwzfISaXpXE1qvNalca5Jwob6FjI4AoPlqMusJ6ftFE7IkkFoMhVLRgK+4Kx3gkzb8UZK5t5yTTvEmA==} + peerDependencies: + typescript: '>=2.1.0 || >=2.1.0-dev || >=2.2.0-dev || >=2.3.0-dev || >=2.4.0-dev || >=2.5.0-dev || >=2.6.0-dev || >=2.7.0-dev || >=2.8.0-dev || >=2.9.0-dev || >= 3.0.0-dev || >= 3.1.0-dev' + dependencies: + tslib: 1.14.1 + typescript: 5.1.6 + dev: true + /tunnel-agent@0.6.0: resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} dependencies: @@ -4378,15 +4871,55 @@ packages: resolution: {integrity: sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==} dev: true + /typedoc-default-themes@0.10.2: + resolution: {integrity: sha512-zo09yRj+xwLFE3hyhJeVHWRSPuKEIAsFK5r2u47KL/HBKqpwdUSanoaz5L34IKiSATFrjG5ywmIu98hPVMfxZg==} + engines: {node: '>= 8'} + dependencies: + lunr: 2.3.9 + dev: true + + /typedoc@0.17.8(typescript@5.1.6): + resolution: {integrity: sha512-/OyrHCJ8jtzu+QZ+771YaxQ9s4g5Z3XsQE3Ma7q+BL392xxBn4UMvvCdVnqKC2T/dz03/VXSLVKOP3lHmDdc/w==} + engines: {node: '>= 8.0.0'} + hasBin: true + peerDependencies: + typescript: '>=3.8.3' + dependencies: + fs-extra: 8.1.0 + handlebars: 4.7.8 + highlight.js: 10.7.3 + lodash: 4.17.21 + lunr: 2.3.9 + marked: 1.0.0 + minimatch: 3.1.2 + progress: 2.0.3 + shelljs: 0.8.5 + typedoc-default-themes: 0.10.2 + typescript: 5.1.6 + dev: true + /typescript@5.1.6: resolution: {integrity: sha512-zaWCozRZ6DLEWAWFrVDz1H6FVXzUSfTy5FUMWsQlU8Ym5JP9eO4xkTIROFCQvhQf61z6O/G6ugw3SgAnvvm+HA==} engines: {node: '>=14.17'} hasBin: true + /uglify-js@3.17.4: + resolution: {integrity: sha512-T9q82TJI9e/C1TAxYvfb16xO120tMVFZrGA3f9/P4424DNu6ypK103y0GPFVa17yotwSyZW5iYXgjYHkGrJW/g==} + engines: {node: '>=0.8.0'} + hasBin: true + requiresBuild: true + dev: true + optional: true + /undefsafe@2.0.5: resolution: {integrity: sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==} dev: true + /universalify@0.1.2: + resolution: {integrity: sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==} + engines: {node: '>= 4.0.0'} + dev: true + /universalify@2.0.0: resolution: {integrity: sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==} engines: {node: '>= 10.0.0'} @@ -4462,6 +4995,10 @@ packages: extsprintf: 1.3.0 dev: false + /webidl-conversions@3.0.1: + resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} + dev: false + /webidl-conversions@7.0.0: resolution: {integrity: sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==} engines: {node: '>=12'} @@ -4475,6 +5012,13 @@ packages: webidl-conversions: 7.0.0 dev: false + /whatwg-url@5.0.0: + resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} + dependencies: + tr46: 0.0.3 + webidl-conversions: 3.0.1 + dev: false + /which-module@2.0.1: resolution: {integrity: sha512-iBdZ57RDvnOR9AGBhML2vFZf7h8vmBjhoaZqODJBFWHVtKkDmKuHai3cx5PgVMrX5YDNp27AofYbAwctSS+vhQ==} dev: true @@ -4494,6 +5038,12 @@ packages: isexe: 2.0.0 dev: true + /wide-align@1.1.3: + resolution: {integrity: sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==} + dependencies: + string-width: 2.1.1 + dev: true + /winston-transport@4.5.0: resolution: {integrity: sha512-YpZzcUzBedhlTAfJg6vJDlyEai/IFMIVcaEZZyl3UXIl4gmqRpU7AE89AHLkbzLUsv0NVmw7ts+iztqKxxPW1Q==} engines: {node: '>= 6.4.0'} @@ -4520,6 +5070,14 @@ packages: winston-transport: 4.5.0 dev: false + /wordwrap@1.0.0: + resolution: {integrity: sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==} + dev: true + + /workerpool@6.1.0: + resolution: {integrity: sha512-toV7q9rWNYha963Pl/qyeZ6wG+3nnsyvolaNUS8+R5Wtw6qJPTxIlOP1ZSvcGhEJw+l3HMMmtiNo9Gl61G4GVg==} + dev: true + /workerpool@6.2.1: resolution: {integrity: sha512-ILEIE97kDZvF9Wb9f6h5aXK4swSlKGUcOEGiIYb2OOu/IrDU9iwj0fD//SsA6E5ibwJxpEvhullJY4Sl4GcpAw==} dev: true @@ -4682,7 +5240,6 @@ packages: /yn@3.1.1: resolution: {integrity: sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==} engines: {node: '>=6'} - dev: false /yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==}