refactor: improve the plugin api

.erb/configs/webpack.config.main.prod.ts

@@ -2,8 +2,9 @@
  * Webpack config for production electron main process
  */
 
-import path from 'path';
-import webpack from 'webpack';
+import { join } from 'path';
+import { readFileSync } from 'fs';
+import { Configuration, DefinePlugin, EnvironmentPlugin } from 'webpack';
 import { merge } from 'webpack-merge';
 import TerserPlugin from 'terser-webpack-plugin';
 import { BundleAnalyzerPlugin } from 'webpack-bundle-analyzer';
@@ -15,16 +16,22 @@ import deleteSourceMaps from '../scripts/delete-source-maps';
 checkNodeEnv('production');
 deleteSourceMaps();
 
-const configuration: webpack.Configuration = {
+const packageJson = JSON.parse(
+  readFileSync(webpackPaths.appPackagePath, {
+    encoding: 'utf8',
+  })
+);
+
+const configuration: Configuration = {
   devtool: 'source-map',
 
   mode: 'production',
 
   target: 'electron-main',
 
   entry: {
-    main: path.join(webpackPaths.srcMainPath, 'main.ts'),
-    preload: path.join(webpackPaths.srcMainPath, 'preload.ts'),
+    main: join(webpackPaths.srcMainPath, 'main.ts'),
+    preload: join(webpackPaths.srcMainPath, 'preload.ts'),
   },
 
   output: {
@@ -58,13 +65,15 @@ const configuration: webpack.Configuration = {
      * NODE_ENV should be production so that modules do not perform certain
      * development checks
      */
-    new webpack.EnvironmentPlugin({
+    new EnvironmentPlugin({
       NODE_ENV: 'production',
       DEBUG_PROD: false,
       START_MINIMIZED: false,
+      XTORY_PACKAGED: true,
+      XTORY_VERSION: packageJson.version,
     }),
 
-    new webpack.DefinePlugin({
+    new DefinePlugin({
       'process.type': '"browser"',
     }),
   ],

.eslintrc.js

@@ -32,6 +32,9 @@ module.exports = {
     'lines-between-class-members': 'off',
     'promise/no-callback-in-promise': 'off',
   },
+  globals: {
+    XTORY_VERSION: 'readonly',
+  },
   parserOptions: {
     ecmaVersion: 2020,
     sourceType: 'module',

README.md

@@ -21,7 +21,7 @@ In the 2018, I started this project as a fork of [Dialogger](https://github.com/
 
 ## Choice of Stack
 
-In the end, I've chosen the slowest option, which is a react app wrapped in an Electron layer. But to be honest, The problem with many of the greatest Open Source software is poor UI design which prevents them from appealing to a wider audience, [Blender](https://www.blender.org/) transition from 2.7x to 2.8x and [Musescore](https://musescore.org/en) version 3.xx to 4.xx shows this in practice. So With the help of react, Using a web approach to design and borrowing heavily from VSCode UI/UX, I hope it's easier to make a clean and usable UI with less development effort. It also lowers the entry barrier for plugin developers who whish to customize the software for their own needs.
+In the end, I've chosen the slowest option, which is a react app wrapped in an Electron layer. But to be honest, The problem with many of the greatest Open Source software is poor UI design which prevents them from appealing to a wider audience, [Blender](https://www.blender.org/) transition from 2.7x to 2.8x and [Musescore](https://musescore.org/en) version 3.xx to 4.xx shows this in practice. So I've decided to sacrifice performance in favor of faster development cycle. Using a web approach to design and borrowing heavily from VSCode UI/UX, I hope it's easier to make a clean and usable UI with less development effort. Coincidentally theming such UI would be an easier task as well. It also lowers the entry barrier for plugin developers who whish to customize the software for their own needs.
 
 # Features
 
@@ -42,10 +42,11 @@ Feel free to make issues and/or help with the development.
 
 Licensing of this project might not be clear from looking at the Github auto-detected license attribute. The Xtory software itself is licensed under GPL3 for maximizing the openness of the tooling, However all the operational parts such as C runtime, plugins and the plugin API itself are licensed under MIT so you can modify and embed them in your own proprietary engines and workflows without any licensing concerns.
 
-| Project                       | License |
-| ----------------------------- | ------- |
-| Xtory Software                | GPL3    |
-| @xtory/plugin-api             | MIT     |
-| @xtory/plugin-xflow           | MIT     |
-| @xtory/plugin-reference-nodes | MIT     |
-| xtory C runtime               | MIT     |
+| Project                       | Description                                                                                                         | License |
+| ----------------------------- | ------------------------------------------------------------------------------------------------------------------- | ------- |
+| Xtory Software                | The modular editor                                                                                                  | GPL3    |
+| @xtory/plugin-api             | Plugin API for developing xtory plugins                                                                             | MIT     |
+| @xtory/plugin-xflow           | Adds support for the main file type in xtory project, xflow is used for writing the plot graphs                     | MIT     |
+| @xtory/plugin-xconv           | Adds support for the conversation graphs, supporting linear and branching conversations                             | MIT     |
+| @xtory/plugin-reference-nodes | Adds a collection of reference nodes for adding developer/writer references to your graphs                          | MIT     |
+| xtory C runtime               | A single-header, cross-platform, C99 runtime and development kit for integrating xtory into your engine and tooling | MIT     |

assets/templates/Empty/xtory.json

@@ -4,6 +4,7 @@
   "logLevel": 2,
   "plugins": {
     "@xtory/plugin-xflow": "*",
+    "@xtory/plugin-xconv": "*",
     "@xtory/plugin-reference-nodes": "*"
   },
   "pluginsConfiguration": {}

docs/plugin-api.md

@@ -0,0 +1,149 @@
+# Plugin API
+
+## Architecture
+
+A plugin is a npm package with a standard `package.json` file, For a package to actually be registered as a plugin by the xtory, you'll need to have a `xtoryApiVersion` field.
+
+### Main Entry Point
+
+Each plugin should have a `main` field which will be loaded on the main xtory process. This main entry point runs in the nodejs context and has full access to the main process environment, can make system call using nodejs API and is freed to import and use any of the package dependencies.
+
+The main script is expected to have default function exported. This function is called with a single argument of type `PluginContext` which provides both the plugin's logger and the plugin API.
+
+```ts
+import type { PluginContext } from '@xtory/plugin-api';
+import MyService from './myService';
+
+export default function ({ logger, api }: PluginContext) {
+  // Add a service to run in the main process. Renderers can call its public methods using `serviceCall` IPC calls.
+  api.addService('my-service', () => new MyService(logger));
+
+  // Adds a new file view for a custom extension
+  api
+    .addFileView('flow') // a node graph
+    .setFileType('myFileExt')
+    .createMenuItem('New MyFile', 'templates/empty.myFileExt')
+    .setNodes([
+      // List of nodes configurations
+    ]);
+
+  // Extend an existing file type view, It is especially useful for adding extra nodes to a flow
+  // introduced by another plugin. For example `@xtory/plugin-reference-nodes` extends `xconv` flows
+  // only if the file type is already registered via the `@xtory/plugin-xconv` plugin.
+  api.addFileView('flow').setFileType('xconv').setOptional(true).setNodes([
+    // List of nodes that will be added to the `xconv` file type.
+  ]);
+}
+```
+
+### Renderer Entry Point
+
+Plugins can optionally have a renderer script configured by adding the `rendererMain` field to the `package.json`. The renderer script will run in a restricted browser process.
+The browser process the main or any other window. The renderer script is not allowed to import plugin's dependencies at runtime via relative paths as it evaluated out of the file-system context. Due to this restriction, for importing dependencies we require to use a bundler and create a singular javascript file as the entry point. However it is recommended to avoid using dependencies in the renderer and sticking with it as a tool for pure visual components.
+
+The renderer process does not provide nodejs integration or file-system access, The main means of communicating with the outside world is the IPC API exposed through `window.electron.ipcRenderer`.
+
+The IPC calls can be used with a large set of builtin operations, including the `serviceCall`. The `serviceCall` IPC channel can be used to call to any xtory service, including the builtin services, and services introduced by the plugins. You can read more about service calls and other IPC channels in the [IPC documentation(TODO: fix link when added)](#).
+
+```ts
+const myServiceMethodResult = await window.electron.ipcRenderer.invoke(
+  'serviceCall',
+  'my-service',
+  'myServiceMethodName',
+  ['argument1', 'argument2']
+);
+```
+
+Since the service is running in the main nodejs process, it is free to use any of the package's dependencies, and the nodejs API.
+
+### Render APIs
+
+By nature the of plugin's renderer script running in an restricted manner the compiled JavaScript entry point is not allowed to have any imports/requires which means you're only allowed to import types from the package's dependencies. The only things other than the browser API provided to the plugin's renderer are the global APIs exposed via the `@xtory/plugin-api/renderer`.
+
+This includes the global `React` instance, and the `window` extensions under `window.renderer` and `window.electron` properties. You can read more about the exposed electron calls in the [IPC documentation(TODO: fix link when added)](#).
+
+The `window.renderer` is an `XtoryRenderer` object shared across all plugins, providing them with the builtin UI components, exposed modules from the renderer, exposed custom react hooks, logger and other tools required for extending the graphical representation of the xtory.
+
+This object will replace all of the imports in your script, and can be used with object decustruction for a clean way of accessing the plugin dependencies.
+
+```ts
+const {
+  modules: {
+    ReactFlow: { Handle, Position, useReactFlow },
+  },
+  ui: { icons, TextField, NodeContainer, Button, PickVariable },
+  uuidv4,
+  registerNodeRenderer,
+} = window.renderer;
+```
+
+This object being shared with other plugins is an intentional design desicion so the plugins can extend the builtin functionalities for all plugins. It includes adding new functions, components or editing the existing toolset; for example replacing the default `TextArea` component with a rich text editor or adding support for color coding the nodes. While having a shared object can cause problematic plugin designs and coupling, it also provides many opportunities for extending the application in a way that wouldn't be possible otherwise. Therefore it is up to the plugin developers to use it with caution.
+
+### Node Renderers
+
+Plugins can add new types of flow nodes to the xtory, adding a new node usually requires a custom node renderer. While the nodes configurations are set up via the main function, The renderer should be registered in the renderer scripts. This can be done with the `window.renderer.registerNodeRenderer` function.
+
+A node renderer is just a react component which accepts `Renderer.NodeProps` as its properties. A node renderer is registered under a string ID, this ID can be used in the main function to set up a node for using this custom node renderer.
+
+```ts
+/// <reference types="@xtory/plugin-api/renderer" />
+
+const {
+  ui: { NodeContainer, Handle },
+  registerNodeRenderer,
+} = window.renderer;
+
+interface MyCustomNodeData {
+  myData?: string;
+}
+
+registerNodeRenderer(
+  'my-plugin-name/my-custom-node-renderer',
+  function ({ selected, data }: Renderer.NodeProps<MyCustomNodeData>) {
+    return (
+      <NodeContainer title="My Node Name" selected={selected}>
+        <Handle type="target" position={Position.Left} />
+        {data.myData}
+        <Handle type="source" position={Position.Right} id="default" />
+      </NodeContainer>
+    );
+  }
+);
+```
+
+With having a custom renderer registered as demonstrated in the example above, you can use it as your custom node renderer in the main function:
+
+```ts
+import type { PluginContext } from '@xtory/plugin-api';
+
+export default function ({ logger, api }: PluginContext) {
+  api
+    .addFileView('flow')
+    .setFileType('myFileExt')
+    .setNodes([
+      {
+        type: 'MyNode',
+        // ...rest of the options...
+        renderer: 'my-plugin-name/my-custom-node-renderer',
+      },
+    ]);
+}
+```
+
+Plugins can use renderers exposed for other plugins as well as the ones introduced by themselves. When depending on another plugin's renderers, is is highly recommended to have that plugin as a package dependency to ensure its presence.
+
+### plugin:// URIs
+
+While the renderer process is running independent of the file-system, the renderer scripts are still allowed to access the files in the plugins. However this can only happen indirectly and through the `plugin://` URIs. You can use `fetch` or even dynamic `import` calls on these URIs in order to load more files other than the renderer script in the rendering process.
+
+The current iteration of the `plugin` protocol has full file-system access and works with both absolute and relative paths, However it is advised not to rely on it as it will have significant changes in the short-term future.
+
+The current version of the protocol can be used similar to the `file://` URIs, they're treated as relative paths to the loaded project. The future iteration of the protocol will be much more limited to discourage anti-pattern design desicions in the plugins ecosystem.
+
+### The `xtoryApiVersion`
+
+Each release of of the xtory can have non-breaking or even breaking changes to its plugin APIs, The `xtoryApiVersion` is used for declaring the intended API version for a plugin. Xtory is then responsible for ensuring that its current API version is either the same or have backward compatibility with each plugin's expected API set.
+
+While in the pre `1.0` releases the backward compatibility is of a lower importance(meaning that there are breaking changes happening regularly), the API versioning is here to ensure a mature plugins ecosystem can thrive down the road; Allowing for compatibility layers, and decoupling the API which a plugin is developed against vs. the one it is going to run with.
+
+Note that the `xtoryApiVersion` is a single numeric version independent of the xtory's application version(which is a semantic version). Multiple versions of xtory can have the same `xtoryApiVersion`.

package.json

@@ -11,10 +11,11 @@
   "license": "GPL-3",
   "main": "./packages/main/src/main.ts",
   "scripts": {
-    "build": "concurrently \"npm run build:main\" \"npm run build:renderer\"",
+    "build": "npm run build:packages && concurrently \"npm run build:main\" \"npm run build:renderer\"",
     "build:main": "cross-env NODE_ENV=production TS_NODE_TRANSPILE_ONLY=true webpack --config ./.erb/configs/webpack.config.main.prod.ts",
     "build:renderer": "cross-env NODE_ENV=production TS_NODE_TRANSPILE_ONLY=true webpack --config ./.erb/configs/webpack.config.renderer.prod.ts",
-    "compile": "npm run build --workspaces --if-present",
+    "build:packages": "npm run build --workspaces --if-present",
+    "typecheck": "npm run typecheck --workspaces --if-present",
     "postinstall": "ts-node .erb/scripts/check-native-dep.js && cross-env CI=true electron-builder install-app-deps && cross-env NODE_ENV=development TS_NODE_TRANSPILE_ONLY=true webpack --config ./.erb/configs/webpack.config.renderer.dev.dll.ts",
     "lint": "cross-env NODE_ENV=development eslint . --ext .js,.jsx,.ts,.tsx",
     "lint:fix": "cross-env NODE_ENV=development eslint . --ext .js,.jsx,.ts,.tsx --fix",
@@ -112,7 +113,7 @@
     "@testing-library/react": "^13.4.0",
     "@types/fs-extra": "^11.0.1",
     "@types/jest": "^29.4.0",
-    "@types/node": "18.13.0",
+    "@types/node": "^18.13.0",
     "@types/react": "^18.2.0",
     "@types/react-beautiful-dnd": "^13.1.4",
     "@types/react-dom": "^18.2.0",

packages/main/package.json

@@ -5,8 +5,12 @@
   "description": "Xtory main process",
   "main": "src/main.ts",
   "license": "GPT-3",
-  "scripts": {},
+  "scripts": {
+    "typecheck": "tsc --noEmit"
+  },
   "dependencies": {
+    "@xtory/shared": "*",
+    "@xtory/plugin-api": "*",
     "uuid": "^9.0.0",
     "fs-extra": "^11.1.1",
     "fengari": "^0.1.4"