feat: proxy mode [sql changed] (#571)

closes #366

开启代理模式时如果找不到依赖会直接返回上游仓库的manifest信息并缓存于nfs，当请求的tgz文件不存在时从上游仓库获取并返回，同时创建对应版本的同步任务。每小时检查更新已缓存的manifest文件保证上游仓库发布新版本时不会因为缓存落后而404。

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

- **New Features**
- Introduced proxy cache management for package manifests and versions.
  - Added new HTTP methods for managing proxy caches.
- Implemented scheduled workers for updating and synchronizing proxy
cache.

- **Updates**
  - Expanded `SyncMode` enum to include a new value `proxy`.
- Updated constants with `PROXY_CACHE_DIR_NAME` and
`ABBREVIATED_META_TYPE`.

- **Tests**
- Added comprehensive test cases for `ProxyCacheService`,
`ProxyCacheRepository`, and related controllers.
- Verified functionality of scheduled workers for proxy cache updates
and synchronization.
- Enhanced testing coverage for handling package downloads in proxy
mode.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

---------

Co-authored-by: fengmk2 <[email protected]>

app/common/adapter/NPMRegistry.ts

@@ -111,7 +111,7 @@ export class NPMRegistry {
     };
   }
 
-  private genAuthorizationHeader(remoteAuthToken?:string) {
+  public genAuthorizationHeader(remoteAuthToken?:string) {
     return remoteAuthToken ? `Bearer ${remoteAuthToken}` : '';
   }
 }

app/common/constants.ts

@@ -1,10 +1,14 @@
 export const BUG_VERSIONS = 'bug-versions';
 export const LATEST_TAG = 'latest';
 export const GLOBAL_WORKER = 'GLOBAL_WORKER';
+export const PROXY_CACHE_DIR_NAME = 'proxy-cache-packages';
+export const ABBREVIATED_META_TYPE = 'application/vnd.npm.install-v1+json';
 export const NOT_IMPLEMENTED_PATH = [ '/-/npm/v1/security/audits/quick', '/-/npm/v1/security/advisories/bulk' ];
+
 export enum SyncMode {
   none = 'none',
   admin = 'admin',
+  proxy = 'proxy',
   exist = 'exist',
   all = 'all',
 }

app/common/enum/Task.ts

@@ -2,6 +2,7 @@ export enum TaskType {
   SyncPackage = 'sync_package',
   ChangesStream = 'changes_stream',
   SyncBinary = 'sync_binary',
+  UpdateProxyCache = 'update_proxy_cache',
   CreateHook = 'create_hook',
   TriggerHook = 'trigger_hook',
 }

app/core/entity/ProxyCache.ts

@@ -0,0 +1,42 @@
+import { Entity, EntityData } from './Entity';
+import { EasyData } from '../util/EntityUtil';
+import { DIST_NAMES } from './Package';
+import { isPkgManifest } from '../service/ProxyCacheService';
+import { PROXY_CACHE_DIR_NAME } from '../../common/constants';
+interface ProxyCacheData extends EntityData {
+  fullname: string;
+  fileType: DIST_NAMES;
+  version?: string;
+}
+
+export type CreateProxyCacheData = Omit<EasyData<ProxyCacheData, 'id'>, 'id'| 'filePath'>;
+
+export class ProxyCache extends Entity {
+  readonly fullname: string;
+  readonly fileType: DIST_NAMES;
+  readonly filePath: string;
+  readonly version?: string;
+
+  constructor(data: ProxyCacheData) {
+    super(data);
+    this.fullname = data.fullname;
+    this.fileType = data.fileType;
+    this.version = data.version;
+    if (isPkgManifest(data.fileType)) {
+      this.filePath = `/${PROXY_CACHE_DIR_NAME}/${data.fullname}/${data.fileType}`;
+    } else {
+      this.filePath = `/${PROXY_CACHE_DIR_NAME}/${data.fullname}/${data.version}/${data.fileType}`;
+    }
+  }
+
+  public static create(data: CreateProxyCacheData): ProxyCache {
+    const newData = { ...data, createdAt: new Date(), updatedAt: new Date() };
+    return new ProxyCache(newData);
+  }
+
+  public static update(data: ProxyCache): ProxyCache {
+    data.updatedAt = new Date();
+    return data;
+  }
+
+}

app/core/entity/Task.ts

@@ -3,8 +3,12 @@ import path from 'path';
 import { Entity, EntityData } from './Entity';
 import { EasyData, EntityUtil } from '../util/EntityUtil';
 import { TaskType, TaskState } from '../../common/enum/Task';
+import { PROXY_CACHE_DIR_NAME } from '../../common/constants';
 import dayjs from '../../common/dayjs';
 import { HookEvent } from './HookEvent';
+import { DIST_NAMES } from './Package';
+import { isPkgManifest } from '../service/ProxyCacheService';
+import { InternalServerError } from 'egg-errors';
 
 export const HOST_NAME = os.hostname();
 export const PID = process.pid;
@@ -40,6 +44,12 @@ export type SyncPackageTaskOptions = {
   specificVersions?: Array<string>;
 };
 
+export type UpdateProxyCacheTaskOptions = {
+  fullname: string,
+  version?: string,
+  fileType: DIST_NAMES,
+};
+
 export interface CreateHookTaskData extends TaskBaseData {
   hookEvent: HookEvent;
 }
@@ -58,6 +68,13 @@ export interface CreateSyncPackageTaskData extends TaskBaseData {
   specificVersions?: Array<string>;
 }
 
+export interface CreateUpdateProxyCacheTaskData extends TaskBaseData {
+  fullname: string,
+  version?: string,
+  fileType: DIST_NAMES,
+  filePath: string
+}
+
 export interface ChangesStreamTaskData extends TaskBaseData {
   since: string;
   last_package?: string,
@@ -75,6 +92,7 @@ export type CreateHookTask = Task<CreateHookTaskData>;
 export type TriggerHookTask = Task<TriggerHookTaskData>;
 export type CreateSyncPackageTask = Task<CreateSyncPackageTaskData>;
 export type ChangesStreamTask = Task<ChangesStreamTaskData>;
+export type CreateUpdateProxyCacheTask = Task<CreateUpdateProxyCacheTaskData>;
 
 export class Task<T extends TaskBaseData = TaskBaseData> extends Entity {
   taskId: string;
@@ -235,6 +253,30 @@ export class Task<T extends TaskBaseData = TaskBaseData> extends Entity {
     return [ TaskType.SyncBinary, TaskType.SyncPackage ].includes(type);
   }
 
+  public static createUpdateProxyCache(targetName: string, options: UpdateProxyCacheTaskOptions):CreateUpdateProxyCacheTask {
+    if (!isPkgManifest(options.fileType)) {
+      throw new InternalServerError('should not update package version manifest.');
+    }
+    const filePath = `/${PROXY_CACHE_DIR_NAME}/${options.fullname}/${options.fileType}`;
+    const data = {
+      type: TaskType.UpdateProxyCache,
+      state: TaskState.Waiting,
+      targetName,
+      authorId: `pid_${PID}`,
+      authorIp: HOST_NAME,
+      data: {
+        taskWorker: '',
+        fullname: options.fullname,
+        version: options?.version,
+        fileType: options.fileType,
+        filePath,
+      },
+    };
+    const task = this.create(data);
+    task.logPath = `/${PROXY_CACHE_DIR_NAME}/${options.fullname}/update-manifest-log/${options.fileType.split('.json')[0]}-${dayjs().format('YYYY/MM/DDHHmm')}-${task.taskId}.log`;
+    return task;
+  }
+
   start(): TaskUpdateCondition {
     const condition = {
       taskId: this.taskId,