Consolidate Common and Dictionary packages (#220)

* Combine common package with dictionary

* Rename dictionary package to `@overture-stack/lectern-dictionary`
- Updates all imports of the dictionary package
- Update root README to reflect this change
- Runs `pnpm update` to make use latest version of all dependencies
  - dependency udpate was required to get all packages onto the same version of zod and work with the zod-to-json-schema utility in the generate script
- Relocate scripts to a single package in the `/script` dir.
  - Adds README to scripts package to describe available scripts
  - Designed to have a single scripts package with multiple utilities instead of separate package per script

* Update dependency versions in validation package

* Update lectern server package name in Dockerfile

* Adds processSchema usage example to Client README

* Updates Dictionary README to match packages format

* Rename processing Success types to not say Error

README.md

@@ -33,15 +33,15 @@ The repository is organized with the following directory structure:
 The modules in the monorepo are organized into two categories:
 
    * __apps/__ - Standalone processes meant to be run. These are published to [ghcr.io](https://ghcr.io) as container images.
-   * __packages/__ - Reusable packages shared between applications and other packages. Packages are published to [NPM](https://npmjs.com) .
+   * __packages/__ - Reusable packages shared between applications and other packages. Packages are published to [NPM](https://npmjs.com).
+   * __scripts__ - Utility scripts for use within this repo.
 
-| Component                                   | Type        | Package Name                       | Path                 | Published Location                                                       | Description                                                                                                                                                                                         |
-| ------------------------------------------- | ----------- | ---------------------------------- | -------------------- | ------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| [Lectern Server](apps/server/README.md)     | Application | server                             | apps/server/         | [![Lectern GHCR Packages](https://img.shields.io/badge/GHCR-lectern-brightgreen?style=for-the-badge&logo=github)](https://github.com/overture-stack/lectern/pkgs/container/lectern) | Lectern Server web application.                                                                                                                                                                     |
-| [Lectern Client](packages/client/README.md) | Package     | @overture-stack/lectern-client     | packages/client      | [![Lectern Client NPM Package](https://img.shields.io/npm/v/@overture-stack/lectern-client?color=%23cb3837&style=for-the-badge&logo=npm)](https://www.npmjs.com/package/@overture-stack/lectern-client)      | TypeScript Client to interact with Lectern Server and perform data validation.                                                                                                                      |
-| [common](packages/common/README.md)         | Package     | common                             | packages/common/     | N/A                                                                      | Non-specific but commonly reusable utilities. Includes shared Error classes.                                                                                                                        |
-| [dictionary](packages/dictionary/README.md) | Package     | dictionary                         | packages/dictionary/ | N/A                                                                      | Dictionary meta-schema definition, includes TS types, and Zod schemas. This also exports all utilities for getting the diff of two dictionaries, and for validating data records with a Dictionary. |
-| [validation](packages/validation/README.md) | Package     | @overture-stack/lectern-validation | packages/validation/ | [![Lectern Validation NPM Package](https://img.shields.io/npm/v/@overture-stack/lectern-client?color=%23cb3837&style=for-the-badge&logo=npm)](https://www.npmjs.com/package/@overture-stack/lectern-client)  | Validate data using Lectern Dictionaries.                                                                                                                                                           |
+| Component                                           | Package Name                       | Path                               | Published Location                                                                                                                                                                                          | Description                                                                                                                                                                                                                                                                                       |
+| --------------------------------------------------- | ---------------------------------- | ---------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [Lectern Server](apps/server/README.md)             | @overture-stack/lectern-server     | apps/server/                       | [![Lectern GHCR Packages](https://img.shields.io/badge/GHCR-lectern-brightgreen?style=for-the-badge&logo=github)](https://github.com/overture-stack/lectern/pkgs/container/lectern)                         | Lectern Server web application.                                                                                                                                                                                                                                                                   |
+| [Lectern Client](packages/client/README.md)         | @overture-stack/lectern-client     | packages/client                    | [![Lectern Client NPM Package](https://img.shields.io/npm/v/@overture-stack/lectern-client?color=%23cb3837&style=for-the-badge&logo=npm)](https://www.npmjs.com/package/@overture-stack/lectern-client)     | TypeScript Client to interact with Lectern Server and Lectern data dictionaries. This library provides a REST client to assist in fetching data from the Lectern server. It also exposes the functionality from the Lectern Validation library to use a Lectern data dictionary to validate data. |
+| [Lectern Dictionary](packages/dictionary/README.md) |                                    | @overture-stack/lectern-dictionary | packages/dictionary/                                                                                                                                                                                        | [![Lectern Client NPM Package](https://img.shields.io/npm/v/@overture-stack/lectern-dictionary?color=%23cb3837&style=for-the-badge&logo=npm)](https://www.npmjs.com/package/@overture-stack/lectern-dictionary)                                                                                   | Dictionary meta-schema definition, includes TS types, and Zod schemas. This also exports all utilities for getting the diff of two dictionaries. |
+| [Lectern Validation](packages/validation/README.md) | @overture-stack/lectern-validation | packages/validation/               | [![Lectern Validation NPM Package](https://img.shields.io/npm/v/@overture-stack/lectern-client?color=%23cb3837&style=for-the-badge&logo=npm)](https://www.npmjs.com/package/@overture-stack/lectern-client) | Validate data using Lectern Dictionaries.                                                                                                                                                                                                                                                         |
 
 ## Developer Instructions
 
@@ -85,19 +85,19 @@ This will build all modules.
 
 `pnpm test:all`
 
-This will test everything, building all dependencies needed to fully test.
+This will test all modules in the repo.
 
-## Additonal Content
+## Additional Content
 
 In addition to the code for Lectern, this repository contains some useful reference material.
 
 ### Meta-Schema
 
 Lectern provides a meta-schema definition that describes the structure of Lectern Dictionaries. The generated JSON Schema formatted copy of this schema can be found at [`./generated/DictionaryMetaSchema.json`](./generated/DictionaryMetaSchema.json).
 
-This can be used as a programing language agnostic schema for external applications that want to validate, generate, or interact with Lectern Dictionaries.
+This can be used as a programing language agnostic schema for external applications to validate Lectern Dictionaries.
 
-> **Note:**
+> [!NOTE]
 >
 > Don't manually update any files in the `./generated` path. This content is programatically generated from the source code.
 

apps/server/Dockerfile

@@ -25,7 +25,7 @@ RUN chown -R ${APP_USER}:${APP_USER} $(npm config get prefix)
 USER ${APP_USER}:${APP_USER}
 
 RUN pnpm install --ignore-scripts
-RUN pnpm nx build server
+RUN pnpm nx build @overture-stack/lectern-server
 
 EXPOSE 3000
 

apps/server/package.json

@@ -1,5 +1,6 @@
 {
-	"name": "server",
+	"name": "@overture-stack/lectern-server",
+	"private": true,
 	"version": "2.0.0-next.0",
 	"description": "Overture Data Dictionary Management",
 	"scripts": {
@@ -19,40 +20,40 @@
 	},
 	"homepage": "https://github.com/overture-stack/lectern#readme",
 	"devDependencies": {
-		"@types/body-parser": "^1.19.2",
+		"@types/body-parser": "^1.19.5",
 		"@types/errorhandler": "0.0.32",
-		"@types/express": "^4.17.17",
+		"@types/express": "^4.17.21",
 		"@types/jsonwebtoken": "^8.5.9",
-		"@types/lodash": "^4.14.195",
-		"@types/memoizee": "^0.4.8",
-		"@types/ms": "^0.7.31",
-		"@types/superagent": "^4.1.18",
+		"@types/lodash": "^4.17.7",
+		"@types/memoizee": "^0.4.11",
+		"@types/ms": "^0.7.34",
+		"@types/superagent": "^4.1.24",
 		"@types/swagger-ui-express": "^3.0.1",
 		"concurrently": "^5.3.0",
 		"husky": "^3.1.0",
 		"nodemon": "^2.0.22",
-		"prettier": "^3",
+		"prettier": "^3.3.3",
 		"testcontainers": "^1.3.1",
-		"zod-to-json-schema": "^3.21.3"
+		"typescript": "^5.5.4",
+		"zod-to-json-schema": "^3.23.2"
 	},
 	"dependencies": {
-		"ajv": "^8.12.0",
-		"axios": "^1.4.0",
+		"@overture-stack/lectern-dictionary": "workspace:^",
+		"ajv": "^8.17.1",
+		"axios": "^1.7.2",
 		"body-parser": "^1.20.2",
-		"common": "workspace:^",
-		"dictionary": "workspace:^",
-		"dotenv": "^16.3.1",
+		"dotenv": "^16.4.5",
 		"errorhandler": "^1.5.1",
-		"express": "^4.18.2",
-		"immer": "^10.0.2",
+		"express": "^4.19.2",
+		"immer": "^10.1.1",
 		"jsonwebtoken": "^8.5.1",
 		"lodash": "^4.17.21",
-		"memoizee": "^0.4.15",
-		"mongoose": "^7.3.2",
+		"memoizee": "^0.4.17",
+		"mongoose": "^7.8.0",
 		"ms": "^2.1.3",
 		"node-vault": "^0.9.22",
 		"swagger-ui-express": "^4.6.3",
-		"winston": "^3.9.0",
-		"zod": "^3.21.4"
+		"winston": "^3.13.1",
+		"zod": "^3.23.8"
 	}
 }

apps/server/src/controllers/dictionaryController.ts

@@ -17,8 +17,13 @@
  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-import { BadRequestError, NotFoundError } from 'common';
-import { Dictionary, Schema, replaceReferences } from 'dictionary';
+import {
+	BadRequestError,
+	Dictionary,
+	NotFoundError,
+	Schema,
+	replaceReferences,
+} from '@overture-stack/lectern-dictionary';
 import { Request, Response } from 'express';
 import * as dictionaryService from '../services/dictionaryService';
 

packages/dictionary/src/types/dbTypes.ts → apps/server/src/db/dbTypes.ts

@@ -17,8 +17,8 @@
  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+import { DictionaryBase } from '@overture-stack/lectern-dictionary';
 import { z as zod } from 'zod';
-import { DictionaryBase } from './dictionaryTypes';
 
 /**
  * A Dictionary stored in the DB is represented as a document and gets an `_id` property

apps/server/src/db/dictionary.ts

@@ -17,9 +17,10 @@
  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-import { Dictionary, DictionaryDocument, DictionaryDocumentSummary } from 'dictionary';
+import { Dictionary } from '@overture-stack/lectern-dictionary';
 import { omit } from 'lodash';
 import mongoose from 'mongoose';
+import type { DictionaryDocument, DictionaryDocumentSummary } from './dbTypes';
 
 export const DictionaryModel = mongoose.model(
 	'Dictionary',

apps/server/src/external/ego.ts

@@ -18,7 +18,7 @@
  */
 
 import axios from 'axios';
-import { ForbiddenError, UnauthorizedError } from 'common';
+import { ForbiddenError, UnauthorizedError } from '@overture-stack/lectern-dictionary';
 import { NextFunction, Request, RequestHandler, Response } from 'express';
 import jwt from 'jsonwebtoken';
 import memoize from 'memoizee';

apps/server/src/routers/diffRouter.ts

@@ -16,8 +16,7 @@
  * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
-import { BadRequestError } from 'common';
-import { DictionaryDiffArray, DiffUtils, replaceReferences } from 'dictionary';
+import { BadRequestError, DictionaryDiffArray, DiffUtils, replaceReferences } from '@overture-stack/lectern-dictionary';
 import { Router } from 'express';
 import * as dictionaryService from '../services/dictionaryService';
 import { wrapAsync } from './wrappers';

apps/server/src/services/dictionaryService.ts

@@ -17,13 +17,20 @@
  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-import { BadRequestError, ConflictError, NotFoundError } from 'common';
-import { Dictionary, DictionaryDocument, DictionaryDocumentSummary, Schema, VersionUtils } from 'dictionary';
+import {
+	BadRequestError,
+	ConflictError,
+	Dictionary,
+	NotFoundError,
+	Schema,
+	VersionUtils,
+} from '@overture-stack/lectern-dictionary';
 import * as immer from 'immer';
 import { omit } from 'lodash';
 import logger from '../config/logger';
 import * as DictionaryRepo from '../db/dictionary';
 import { normalizeSchema, validate } from '../services/schemaService';
+import type { DictionaryDocument, DictionaryDocumentSummary } from '../db/dbTypes';
 
 /**
  * Get latest version for all dictionaries with the provided name

apps/server/src/services/schemaService.ts

@@ -17,10 +17,9 @@
  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-import { References, Schema } from 'dictionary';
+import { References, replaceSchemaReferences, Schema } from '@overture-stack/lectern-dictionary';
 import * as immer from 'immer';
 import { ZodError } from 'zod';
-import { replaceSchemaReferences } from 'dictionary';
 
 export function validate(schema: Schema, references: References): { valid: boolean; errors?: ZodError } {
 	const schemaWithReplacements = replaceSchemaReferences(schema, references);

apps/server/test/fixtures/dictionaries/1_base.ts

@@ -17,7 +17,7 @@
  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-import { Dictionary } from 'dictionary';
+import { Dictionary } from '@overture-stack/lectern-dictionary';
 import primitivesSchema from '../schemas/primitives';
 
 const dictionary: Dictionary = {

apps/server/test/fixtures/dictionaries/2_base_schema_added.ts

@@ -17,7 +17,7 @@
  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-import { Dictionary } from 'dictionary';
+import { Dictionary } from '@overture-stack/lectern-dictionary';
 import primitivesSchema from '../schemas/primitives';
 import additionalSchema from '../schemas/schemaChangesBase';
 const dictionary: Dictionary = {

apps/server/test/fixtures/dictionaries/3_base_schema_updated.ts

@@ -17,7 +17,7 @@
  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-import { Dictionary } from 'dictionary';
+import { Dictionary } from '@overture-stack/lectern-dictionary';
 import primitivesSchema from '../schemas/primitives';
 import updatedSchema from '../schemas/schemaChangesUpdated';
 const dictionary: Dictionary = {

apps/server/test/fixtures/dictionaries/invalidReferences.ts

@@ -17,7 +17,7 @@
  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-import { Dictionary } from 'dictionary';
+import { Dictionary } from '@overture-stack/lectern-dictionary';
 import referencesSchema, { references } from '../schemas/references';
 const dictionary: Dictionary = {
 	name: 'Invalid References Dictionary',

apps/server/test/fixtures/dictionaries/linebreak/input1.ts

@@ -17,7 +17,7 @@
  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-import { Dictionary } from 'dictionary';
+import { Dictionary } from '@overture-stack/lectern-dictionary';
 
 const output: Dictionary = {
 	name: 'Test Dictionary',

apps/server/test/fixtures/dictionaries/linebreak/input2.ts

@@ -17,7 +17,7 @@
  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-import { Dictionary } from 'dictionary';
+import { Dictionary } from '@overture-stack/lectern-dictionary';
 
 const output: Dictionary = {
 	name: 'Test Dictionary',

apps/server/test/fixtures/dictionaries/linebreak/nLinebreaks.ts

@@ -17,7 +17,7 @@
  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-import { Dictionary } from 'dictionary';
+import { Dictionary } from '@overture-stack/lectern-dictionary';
 
 const output: Dictionary = {
 	name: 'Line Breaks',

apps/server/test/fixtures/dictionaries/linebreak/normalizedLinebreaks.ts

@@ -17,7 +17,7 @@
  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-import { Dictionary } from 'dictionary';
+import { Dictionary } from '@overture-stack/lectern-dictionary';
 
 const output: Dictionary = {
 	name: 'Line Breaks',

apps/server/test/fixtures/dictionaries/linebreak/output.ts

@@ -17,7 +17,7 @@
  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-import { Dictionary } from 'dictionary';
+import { Dictionary } from '@overture-stack/lectern-dictionary';
 
 const output: Dictionary = {
 	name: 'Test Dictionary',

apps/server/test/fixtures/dictionaries/linebreak/rnLinebreaks.ts

@@ -17,7 +17,7 @@
  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-import { Dictionary } from 'dictionary';
+import { Dictionary } from '@overture-stack/lectern-dictionary';
 
 const output: Dictionary = {
 	name: 'Line Breaks',

apps/server/test/fixtures/dictionaries/simple.ts

@@ -17,7 +17,7 @@
  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-import { Dictionary } from 'dictionary';
+import { Dictionary } from '@overture-stack/lectern-dictionary';
 import primitivesSchema from '../schemas/primitives';
 export default {
 	name: 'Simple Dictionary',

apps/server/test/fixtures/dictionaries/validReferences.ts

@@ -17,7 +17,7 @@
  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-import { Dictionary } from 'dictionary';
+import { Dictionary } from '@overture-stack/lectern-dictionary';
 import referencesSchema, { references } from '../schemas/references';
 const dictionary: Dictionary = {
 	name: 'Valid References Dictionary',

apps/server/test/fixtures/schemas/primitives.ts

@@ -17,7 +17,7 @@
  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-import { Schema } from 'dictionary';
+import { Schema } from '@overture-stack/lectern-dictionary';
 
 // Just a simple schema with no frills.
 

apps/server/test/fixtures/schemas/references.ts

@@ -17,7 +17,7 @@
  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-import { Schema } from 'dictionary';
+import { Schema } from '@overture-stack/lectern-dictionary';
 
 export const references = {
 	listA: ['a', 'b', 'c'],

apps/server/test/fixtures/schemas/schemaChangesBase.ts

@@ -17,7 +17,7 @@
  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-import { Schema } from 'dictionary';
+import { Schema } from '@overture-stack/lectern-dictionary';
 
 export default {
 	name: 'schema_tests',

apps/server/test/fixtures/schemas/schemaChangesUpdated.ts

@@ -17,7 +17,7 @@
  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-import { Schema } from 'dictionary';
+import { Schema } from '@overture-stack/lectern-dictionary';
 
 export default {
 	name: 'schema_tests',

apps/server/test/functional/services/dictionaryService.spec.ts

@@ -19,7 +19,7 @@
 
 import chai, { expect } from 'chai';
 import chaiAsPromised from 'chai-as-promised';
-import { DictionaryDocument, VersionUtils } from 'dictionary';
+import { VersionUtils } from '@overture-stack/lectern-dictionary';
 import * as immer from 'immer';
 import { Error as MongooseError } from 'mongoose';
 import sinon from 'sinon';
@@ -34,6 +34,7 @@ import SCHEMA_1_EXISTING from '../../fixtures/schemas/primitives';
 import SCHEMA_REFERENCES, { references } from '../../fixtures/schemas/references';
 import SCHEMA_2_ADD from '../../fixtures/schemas/schemaChangesBase';
 import SCHEMA_3_UPDATES from '../../fixtures/schemas/schemaChangesUpdated';
+import type { DictionaryDocument } from '../../../src/db/dbTypes';
 
 chai.use(chaiAsPromised);