feature: support local model pull (#944)

Author: louis-jan

cortex-js/.eslintrc.cjs

@@ -0,0 +1,46 @@
+module.exports = {
+  parser: '@typescript-eslint/parser',
+  parserOptions: {
+    project: 'tsconfig.json',
+    tsconfigRootDir: __dirname,
+    sourceType: 'module',
+  },
+  plugins: ['@typescript-eslint/eslint-plugin'],
+  extends: [
+    'plugin:@typescript-eslint/recommended',
+    'plugin:prettier/recommended',
+  ],
+  root: true,
+  env: {
+    node: true,
+    jest: true,
+  },
+  ignorePatterns: ['.eslintrc.js'],
+  rules: {
+    '@typescript-eslint/interface-name-prefix': 'off',
+    '@typescript-eslint/explicit-function-return-type': 'off',
+    '@typescript-eslint/explicit-module-boundary-types': 'off',
+    '@typescript-eslint/no-explicit-any': 'off',
+    '@typescript-eslint/no-unused-vars': ['warn'],
+    '@typescript-eslint/no-floating-promises': 'warn',
+    '@typescript-eslint/no-var-requires': 'warn',
+    '@typescript-eslint/ban-types': 'warn',
+    'no-unused-vars': 'off',
+    'require-await': 'off',
+    'prefer-const': 'warn',
+    'no-restricted-syntax': [
+      'warn',
+      {
+        selector:
+          'CallExpression[callee.object.name=configService][callee.property.name=/^(get|getOrThrow)$/]:not(:has([arguments.1] Property[key.name=infer][value.value=true])), CallExpression[callee.object.property.name=configService][callee.property.name=/^(get|getOrThrow)$/]:not(:has([arguments.1] Property[key.name=infer][value.value=true]))',
+        message:
+          'Add "{ infer: true }" to configService.get() for correct typechecking. Example: configService.get("database.port", { infer: true })',
+      },
+      {
+        selector:
+          'CallExpression[callee.name=it][arguments.0.value!=/^should/]',
+        message: '"it" should start with "should"',
+      },
+    ],
+  },
+};

cortex-js/.eslintrc.js

@@ -24,7 +24,7 @@
     "start:dev": "nest start --watch",
     "start:debug": "nest start --debug --watch",
     "start:prod": "node dist/src/main --trace-deprecation",
-    "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
+    "lint": "eslint \"{src,apps,libs,test}/**/*.ts\"",
     "test": "jest",
     "test:watch": "jest --watch",
     "test:cov": "jest --coverage",
@@ -39,8 +39,6 @@
   },
   "dependencies": {
     "@cortexso/cortex.js": "^0.1.5",
-    "@huggingface/gguf": "^0.1.5",
-    "@huggingface/hub": "^0.15.1",
     "@nestjs/axios": "^3.0.2",
     "@nestjs/common": "^10.0.0",
     "@nestjs/config": "^3.2.2",
@@ -59,6 +57,7 @@
     "cortex-cpp": "0.4.34",
     "cpu-instructions": "^0.0.11",
     "decompress": "^4.2.1",
+    "hyllama": "^0.2.2",
     "js-yaml": "^4.1.0",
     "nest-commander": "^3.13.0",
     "ora": "5.4.1",
@@ -88,14 +87,16 @@
     "@types/supertest": "^6.0.2",
     "@types/update-notifier": "^6.0.8",
     "@types/uuid": "^9.0.8",
-    "@typescript-eslint/eslint-plugin": "^6.0.0",
-    "@typescript-eslint/parser": "^6.0.0",
+    "@typescript-eslint/eslint-plugin": "7.16.1",
+    "@typescript-eslint/parser": "7.16.1",
     "@vercel/ncc": "^0.38.0",
     "@yao-pkg/pkg": "^5.12.0",
     "cpx": "^1.5.0",
-    "eslint": "^8.42.0",
-    "eslint-config-prettier": "^9.0.0",
-    "eslint-plugin-prettier": "^5.0.0",
+    "env-cmd": "10.1.0",
+    "eslint": "8.57.0",
+    "eslint-config-prettier": "9.1.0",
+    "eslint-plugin-import": "2.29.1",
+    "eslint-plugin-prettier": "5.2.1",
     "hanbi": "^1.0.3",
     "is-primitive": "^3.0.1",
     "jest": "^29.5.0",
@@ -112,10 +113,6 @@
     "tsconfig-paths": "^4.2.0",
     "typescript": "^5.1.3"
   },
-  "resolutions": {
-    "ajv": "8.15.0",
-    "whatwg-url": "14.0.0"
-  },
   "files": [
     "dist"
   ],

cortex-js/package.json

@@ -93,7 +93,7 @@ export class ChatCommand extends BaseCommand {
     ) {
       console.log('Downloading engine...');
       await this.cortex.engines.init(engine);
-      await downloadProgress(this.cortex, undefined, DownloadType.Engine)
+      await downloadProgress(this.cortex, undefined, DownloadType.Engine);
     }
 
     if (!message) options.attach = true;
@@ -108,13 +108,13 @@ export class ChatCommand extends BaseCommand {
     );
 
     const preset = await this.fileService.getPreset(options.preset);
-      
+
     return this.chatClient.chat(
-        modelId,
-        options.threadId,
-        message, // Accept both message from inputs or arguments
-        preset ? preset : {},
-      )
+      modelId,
+      options.threadId,
+      message, // Accept both message from inputs or arguments
+      preset ? preset : {},
+    );
   }
 
   modelInquiry = async (models: Cortex.Model[]) => {

cortex-js/src/infrastructure/commanders/chat.command.ts

@@ -84,10 +84,7 @@ export class CortexCommand extends CommandRunner {
     return this.startServer(showLogs, dataFolderPath);
   }
 
-  private async startServer(
-    attach: boolean,
-    dataFolderPath?: string,
-  ) {
+  private async startServer(attach: boolean, dataFolderPath?: string) {
     const config = await this.fileManagerService.getConfig();
     try {
       const startEngineSpinner = ora('Starting Cortex engine...');
@@ -119,9 +116,13 @@ export class CortexCommand extends CommandRunner {
       } else {
         await this.cortexUseCases.startServerDetached(this.host, this.port);
       }
-      console.log(chalk.blue(`Started server at http://${this.host}:${this.port}`));
       console.log(
-        chalk.blue(`API Playground available at http://${this.host}:${this.port}/api`),
+        chalk.blue(`Started server at http://${this.host}:${this.port}`),
+      );
+      console.log(
+        chalk.blue(
+          `API Playground available at http://${this.host}:${this.port}/api`,
+        ),
       );
       await this.fileManagerService.writeConfigFile({
         ...config,

cortex-js/src/infrastructure/commanders/cortex-command.commander.ts

@@ -19,6 +19,7 @@ import { downloadProgress } from '@/utils/download-progress';
 import { CortexClient } from '../services/cortex.client';
 import { DownloadType } from '@/domain/models/download.interface';
 import ora from 'ora';
+import { isLocalFile } from '@/utils/urls';
 
 @SubCommand({
   name: 'pull',
@@ -61,9 +62,8 @@ export class ModelPullCommand extends BaseCommand {
       exit(1);
     });
 
-    ora().succeed('Model downloaded');
-
     await downloadProgress(this.cortex, modelId);
+    ora().succeed('Model downloaded');
 
     const existingModel = await this.cortex.models.retrieve(modelId);
     const engine = existingModel?.engine || Engines.llamaCPP;

cortex-js/src/infrastructure/commanders/models/model-pull.command.ts

@@ -13,6 +13,8 @@ import { ChatClient } from './services/chat-client';
 import { downloadProgress } from '@/utils/download-progress';
 import { CortexClient } from './services/cortex.client';
 import { DownloadType } from '@/domain/models/download.interface';
+import { isLocalFile } from '@/utils/urls';
+import { parse } from 'node:path';
 
 type RunOptions = {
   threadId?: string;
@@ -71,6 +73,12 @@ export class RunCommand extends BaseCommand {
       await downloadProgress(this.cortex, modelId);
       checkingSpinner.succeed('Model downloaded');
 
+      // Update to persisted modelId
+      // TODO: Should be retrieved from the request
+      if (isLocalFile(modelId)) {
+        modelId = parse(modelId).name;
+      }
+
       // Second check if model is available
       existingModel = await this.cortex.models.retrieve(modelId);
       if (!existingModel) {
@@ -93,6 +101,7 @@ export class RunCommand extends BaseCommand {
     }
 
     const startingSpinner = ora('Loading model...').start();
+
     return this.cortex.models
       .start(modelId, await this.fileService.getPreset(options.preset))
       .then(() => {

cortex-js/src/infrastructure/commanders/run.command.ts

@@ -55,7 +55,14 @@ export class ChatClient {
     });
 
     rl.on('line', (input) =>
-      this.sendCompletionMessage(input, messages, modelId, thread.id, rl, settings),
+      this.sendCompletionMessage(
+        input,
+        messages,
+        modelId,
+        thread.id,
+        rl,
+        settings,
+      ),
     );
   }