diff --git a/app/common/adapter/NPMRegistry.ts b/app/common/adapter/NPMRegistry.ts index 9638c8d0..ac3789dc 100644 --- a/app/common/adapter/NPMRegistry.ts +++ b/app/common/adapter/NPMRegistry.ts @@ -111,7 +111,7 @@ export class NPMRegistry { }; } - private genAuthorizationHeader(remoteAuthToken?:string) { + public genAuthorizationHeader(remoteAuthToken?:string) { return remoteAuthToken ? `Bearer ${remoteAuthToken}` : ''; } } diff --git a/app/common/constants.ts b/app/common/constants.ts index 17dc818a..3127a5b7 100644 --- a/app/common/constants.ts +++ b/app/common/constants.ts @@ -1,10 +1,14 @@ export const BUG_VERSIONS = 'bug-versions'; export const LATEST_TAG = 'latest'; export const GLOBAL_WORKER = 'GLOBAL_WORKER'; +export const PROXY_CACHE_DIR_NAME = 'proxy-cache-packages'; +export const ABBREVIATED_META_TYPE = 'application/vnd.npm.install-v1+json'; export const NOT_IMPLEMENTED_PATH = [ '/-/npm/v1/security/audits/quick', '/-/npm/v1/security/advisories/bulk' ]; + export enum SyncMode { none = 'none', admin = 'admin', + proxy = 'proxy', exist = 'exist', all = 'all', } diff --git a/app/common/enum/Task.ts b/app/common/enum/Task.ts index 90fb5b3e..ee987083 100644 --- a/app/common/enum/Task.ts +++ b/app/common/enum/Task.ts @@ -2,6 +2,7 @@ export enum TaskType { SyncPackage = 'sync_package', ChangesStream = 'changes_stream', SyncBinary = 'sync_binary', + UpdateProxyCache = 'update_proxy_cache', CreateHook = 'create_hook', TriggerHook = 'trigger_hook', } diff --git a/app/core/entity/ProxyCache.ts b/app/core/entity/ProxyCache.ts new file mode 100644 index 00000000..7e6cea00 --- /dev/null +++ b/app/core/entity/ProxyCache.ts @@ -0,0 +1,42 @@ +import { Entity, EntityData } from './Entity'; +import { EasyData } from '../util/EntityUtil'; +import { DIST_NAMES } from './Package'; +import { isPkgManifest } from '../service/ProxyCacheService'; +import { PROXY_CACHE_DIR_NAME } from '../../common/constants'; +interface ProxyCacheData extends EntityData { + fullname: string; + fileType: DIST_NAMES; + version?: string; +} + +export type CreateProxyCacheData = Omit, 'id'| 'filePath'>; + +export class ProxyCache extends Entity { + readonly fullname: string; + readonly fileType: DIST_NAMES; + readonly filePath: string; + readonly version?: string; + + constructor(data: ProxyCacheData) { + super(data); + this.fullname = data.fullname; + this.fileType = data.fileType; + this.version = data.version; + if (isPkgManifest(data.fileType)) { + this.filePath = `/${PROXY_CACHE_DIR_NAME}/${data.fullname}/${data.fileType}`; + } else { + this.filePath = `/${PROXY_CACHE_DIR_NAME}/${data.fullname}/${data.version}/${data.fileType}`; + } + } + + public static create(data: CreateProxyCacheData): ProxyCache { + const newData = { ...data, createdAt: new Date(), updatedAt: new Date() }; + return new ProxyCache(newData); + } + + public static update(data: ProxyCache): ProxyCache { + data.updatedAt = new Date(); + return data; + } + +} diff --git a/app/core/entity/Task.ts b/app/core/entity/Task.ts index 43280513..d012ddae 100644 --- a/app/core/entity/Task.ts +++ b/app/core/entity/Task.ts @@ -3,8 +3,12 @@ import path from 'path'; import { Entity, EntityData } from './Entity'; import { EasyData, EntityUtil } from '../util/EntityUtil'; import { TaskType, TaskState } from '../../common/enum/Task'; +import { PROXY_CACHE_DIR_NAME } from '../../common/constants'; import dayjs from '../../common/dayjs'; import { HookEvent } from './HookEvent'; +import { DIST_NAMES } from './Package'; +import { isPkgManifest } from '../service/ProxyCacheService'; +import { InternalServerError } from 'egg-errors'; export const HOST_NAME = os.hostname(); export const PID = process.pid; @@ -40,6 +44,12 @@ export type SyncPackageTaskOptions = { specificVersions?: Array; }; +export type UpdateProxyCacheTaskOptions = { + fullname: string, + version?: string, + fileType: DIST_NAMES, +}; + export interface CreateHookTaskData extends TaskBaseData { hookEvent: HookEvent; } @@ -58,6 +68,13 @@ export interface CreateSyncPackageTaskData extends TaskBaseData { specificVersions?: Array; } +export interface CreateUpdateProxyCacheTaskData extends TaskBaseData { + fullname: string, + version?: string, + fileType: DIST_NAMES, + filePath: string +} + export interface ChangesStreamTaskData extends TaskBaseData { since: string; last_package?: string, @@ -75,6 +92,7 @@ export type CreateHookTask = Task; export type TriggerHookTask = Task; export type CreateSyncPackageTask = Task; export type ChangesStreamTask = Task; +export type CreateUpdateProxyCacheTask = Task; export class Task extends Entity { taskId: string; @@ -235,6 +253,30 @@ export class Task extends Entity { return [ TaskType.SyncBinary, TaskType.SyncPackage ].includes(type); } + public static createUpdateProxyCache(targetName: string, options: UpdateProxyCacheTaskOptions):CreateUpdateProxyCacheTask { + if (!isPkgManifest(options.fileType)) { + throw new InternalServerError('should not update package version manifest.'); + } + const filePath = `/${PROXY_CACHE_DIR_NAME}/${options.fullname}/${options.fileType}`; + const data = { + type: TaskType.UpdateProxyCache, + state: TaskState.Waiting, + targetName, + authorId: `pid_${PID}`, + authorIp: HOST_NAME, + data: { + taskWorker: '', + fullname: options.fullname, + version: options?.version, + fileType: options.fileType, + filePath, + }, + }; + const task = this.create(data); + task.logPath = `/${PROXY_CACHE_DIR_NAME}/${options.fullname}/update-manifest-log/${options.fileType.split('.json')[0]}-${dayjs().format('YYYY/MM/DDHHmm')}-${task.taskId}.log`; + return task; + } + start(): TaskUpdateCondition { const condition = { taskId: this.taskId, diff --git a/app/core/service/ProxyCacheService.ts b/app/core/service/ProxyCacheService.ts new file mode 100644 index 00000000..1cfe25ec --- /dev/null +++ b/app/core/service/ProxyCacheService.ts @@ -0,0 +1,253 @@ +import { EggHttpClient, HttpClientRequestOptions, HttpClientResponse } from 'egg'; +import { ForbiddenError } from 'egg-errors'; +import { SingletonProto, AccessLevel, Inject, EggContext } from '@eggjs/tegg'; +import { BackgroundTaskHelper } from '@eggjs/tegg-background-task'; +import { valid as semverValid } from 'semver'; +import { AbstractService } from '../../common/AbstractService'; +import { TaskService } from './TaskService'; +import { CacheService } from './CacheService'; +import { RegistryManagerService } from './RegistryManagerService'; +import { NPMRegistry } from '../../common/adapter/NPMRegistry'; +import { NFSAdapter } from '../../common/adapter/NFSAdapter'; +import { ProxyCache } from '../entity/ProxyCache'; +import { Task, UpdateProxyCacheTaskOptions, CreateUpdateProxyCacheTask } from '../entity/Task'; +import { ProxyCacheRepository } from '../../repository/ProxyCacheRepository'; +import { TaskType, TaskState } from '../../common/enum/Task'; +import { calculateIntegrity } from '../../common/PackageUtil'; +import { ABBREVIATED_META_TYPE, PROXY_CACHE_DIR_NAME } from '../../common/constants'; +import { DIST_NAMES } from '../entity/Package'; +import type { AbbreviatedPackageManifestType, AbbreviatedPackageJSONType, PackageManifestType, PackageJSONType } from '../../repository/PackageRepository'; + +function isoNow() { + return new Date().toISOString(); +} + +export function isPkgManifest(fileType: DIST_NAMES) { + return fileType === DIST_NAMES.FULL_MANIFESTS || fileType === DIST_NAMES.ABBREVIATED_MANIFESTS; +} + +type GetSourceManifestAndCacheReturnType = T extends DIST_NAMES.ABBREVIATED | DIST_NAMES.MANIFEST ? AbbreviatedPackageJSONType | PackageJSONType : + T extends DIST_NAMES.FULL_MANIFESTS | DIST_NAMES.ABBREVIATED_MANIFESTS ? AbbreviatedPackageManifestType|PackageManifestType : never; + + +@SingletonProto({ + accessLevel: AccessLevel.PUBLIC, +}) +export class ProxyCacheService extends AbstractService { + @Inject() + private readonly httpclient: EggHttpClient; + @Inject() + private readonly npmRegistry: NPMRegistry; + @Inject() + private readonly nfsAdapter: NFSAdapter; + @Inject() + private readonly proxyCacheRepository: ProxyCacheRepository; + @Inject() + private readonly registryManagerService: RegistryManagerService; + @Inject() + private readonly taskService: TaskService; + @Inject() + private readonly cacheService: CacheService; + @Inject() + private readonly backgroundTaskHelper:BackgroundTaskHelper; + + async getPackageVersionTarResponse(fullname: string, ctx: EggContext): Promise { + if (this.config.cnpmcore.syncPackageBlockList.includes(fullname)) { + throw new ForbiddenError(`stop proxy by block list: ${JSON.stringify(this.config.cnpmcore.syncPackageBlockList)}`); + } + return await this.getProxyResponse(ctx); + } + + async getPackageManifest(fullname: string, fileType: DIST_NAMES.FULL_MANIFESTS| DIST_NAMES.ABBREVIATED_MANIFESTS): Promise { + const cachedStoreKey = (await this.proxyCacheRepository.findProxyCache(fullname, fileType))?.filePath; + if (cachedStoreKey) { + const nfsBytes = await this.nfsAdapter.getBytes(cachedStoreKey); + const nfsString = Buffer.from(nfsBytes!).toString(); + const nfsPkgManifgest = JSON.parse(nfsString); + return nfsPkgManifgest; + } + + const manifest = await this.getRewrittenManifest(fullname, fileType); + this.backgroundTaskHelper.run(async () => { + await this.storeRewrittenManifest(manifest, fullname, fileType); + const cachedFiles = ProxyCache.create({ fullname, fileType }); + await this.proxyCacheRepository.saveProxyCache(cachedFiles); + }); + return manifest; + } + + // used by GET /:fullname/:versionOrTag + async getPackageVersionManifest(fullname: string, fileType: DIST_NAMES.ABBREVIATED | DIST_NAMES.MANIFEST, versionOrTag: string): Promise { + let version; + if (semverValid(versionOrTag)) { + version = versionOrTag; + } else { + const pkgManifest = await this.getPackageManifest(fullname, DIST_NAMES.ABBREVIATED_MANIFESTS); + const distTags = pkgManifest['dist-tags'] || {}; + version = distTags[versionOrTag] ? distTags[versionOrTag] : versionOrTag; + } + const cachedStoreKey = (await this.proxyCacheRepository.findProxyCache(fullname, fileType, version))?.filePath; + if (cachedStoreKey) { + const nfsBytes = await this.nfsAdapter.getBytes(cachedStoreKey); + const nfsString = Buffer.from(nfsBytes!).toString(); + return JSON.parse(nfsString) as PackageJSONType | AbbreviatedPackageJSONType; + } + const manifest = await this.getRewrittenManifest(fullname, fileType, versionOrTag); + this.backgroundTaskHelper.run(async () => { + await this.storeRewrittenManifest(manifest, fullname, fileType); + const cachedFiles = ProxyCache.create({ fullname, fileType, version }); + await this.proxyCacheRepository.saveProxyCache(cachedFiles); + }); + return manifest; + } + + async removeProxyCache(fullname: string, fileType: DIST_NAMES, version?: string) { + const storeKey = isPkgManifest(fileType) + ? `/${PROXY_CACHE_DIR_NAME}/${fullname}/${fileType}` + : `/${PROXY_CACHE_DIR_NAME}/${fullname}/${version}/${fileType}`; + await this.nfsAdapter.remove(storeKey); + await this.proxyCacheRepository.removeProxyCache(fullname, fileType, version); + } + + async createTask(targetName: string, options: UpdateProxyCacheTaskOptions): Promise { + return await this.taskService.createTask(Task.createUpdateProxyCache(targetName, options), false) as CreateUpdateProxyCacheTask; + } + + async findExecuteTask() { + return await this.taskService.findExecuteTask(TaskType.UpdateProxyCache); + } + + async executeTask(task: Task) { + const logs: string[] = []; + const fullname = (task as CreateUpdateProxyCacheTask).data.fullname; + const { fileType, version } = (task as CreateUpdateProxyCacheTask).data; + let cachedManifest; + logs.push(`[${isoNow()}] 🚧🚧🚧🚧🚧 Start update "${fullname}-${fileType}" 🚧🚧🚧🚧🚧`); + try { + const cachedFiles = await this.proxyCacheRepository.findProxyCache(fullname, fileType); + if (!cachedFiles) throw new Error('task params error, can not found record in repo.'); + cachedManifest = await this.getRewrittenManifest(fullname, fileType); + await this.storeRewrittenManifest(cachedManifest, fullname, fileType); + ProxyCache.update(cachedFiles); + await this.proxyCacheRepository.saveProxyCache(cachedFiles); + } catch (error) { + task.error = error; + logs.push(`[${isoNow()}] ❌ ${task.error}`); + logs.push(`[${isoNow()}] ❌❌❌❌❌ ${fullname}-${fileType} ${version ?? ''} ❌❌❌❌❌`); + await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n')); + this.logger.info('[ProxyCacheService.executeTask:fail] taskId: %s, targetName: %s, %s', + task.taskId, task.targetName, task.error); + return; + } + logs.push(`[${isoNow()}] 🟢 Update Success.`); + const isFullManifests = fileType === DIST_NAMES.FULL_MANIFESTS; + const cachedKey = await this.cacheService.getPackageEtag(fullname, isFullManifests); + if (cachedKey) { + const cacheBytes = Buffer.from(JSON.stringify(cachedManifest)); + const { shasum: etag } = await calculateIntegrity(cacheBytes); + await this.cacheService.savePackageEtagAndManifests(fullname, isFullManifests, etag, cacheBytes); + logs.push(`[${isoNow()}] 🟢 Update Cache Success.`); + } + await this.taskService.finishTask(task, TaskState.Success, logs.join('\n')); + } + + async getRewrittenManifest(fullname:string, fileType: T, versionOrTag?:string): Promise> { + let responseResult; + switch (fileType) { + case DIST_NAMES.FULL_MANIFESTS: + responseResult = await this.getUpstreamFullManifests(fullname); + break; + case DIST_NAMES.ABBREVIATED_MANIFESTS: + responseResult = await this.getUpstreamAbbreviatedManifests(fullname); + break; + case DIST_NAMES.MANIFEST: + responseResult = await this.getUpstreamPackageVersionManifest(fullname, versionOrTag!); + break; + case DIST_NAMES.ABBREVIATED: + responseResult = await this.getUpstreamAbbreviatedPackageVersionManifest(fullname, versionOrTag!); + break; + default: + break; + } + + // replace tarball url + const manifest = responseResult.data; + const { sourceRegistry, registry } = this.config.cnpmcore; + if (isPkgManifest(fileType)) { + // pkg manifest + const versionMap = manifest.versions || {}; + for (const key in versionMap) { + const versionItem = versionMap[key]; + if (versionItem?.dist?.tarball) { + versionItem.dist.tarball = versionItem.dist.tarball.replace(sourceRegistry, registry); + } + } + } else { + // pkg version manifest + const distItem = manifest.dist || {}; + if (distItem.tarball) { + distItem.tarball = distItem.tarball.replace(sourceRegistry, registry); + } + } + return manifest; + } + + private async storeRewrittenManifest(manifest, fullname: string, fileType: DIST_NAMES) { + let storeKey: string; + if (isPkgManifest(fileType)) { + storeKey = `/${PROXY_CACHE_DIR_NAME}/${fullname}/${fileType}`; + } else { + const version = manifest.version; + storeKey = `/${PROXY_CACHE_DIR_NAME}/${fullname}/${version}/${fileType}`; + } + const nfsBytes = Buffer.from(JSON.stringify(manifest)); + await this.nfsAdapter.uploadBytes(storeKey, nfsBytes); + } + + private async getProxyResponse(ctx: Partial, options?: HttpClientRequestOptions): Promise { + const registry = this.npmRegistry.registry; + const remoteAuthToken = await this.registryManagerService.getAuthTokenByRegistryHost(registry); + const authorization = this.npmRegistry.genAuthorizationHeader(remoteAuthToken); + + const url = `${this.npmRegistry.registry}${ctx.url}`; + + const res = await this.httpclient.request(url, { + timing: true, + followRedirect: true, + // once redirection is also count as a retry + retry: 7, + dataType: 'stream', + timeout: 10000, + compressed: true, + ...options, + headers: { + accept: ctx.header?.accept, + 'user-agent': ctx.header?.['user-agent'], + authorization, + 'x-forwarded-for': ctx?.ip, + via: `1.1, ${this.config.cnpmcore.registry}`, + }, + }) as HttpClientResponse; + this.logger.info('[ProxyCacheService:getProxyStreamResponse] %s, status: %s', url, res.status); + return res; + } + + private async getUpstreamFullManifests(fullname: string): Promise { + const url = `/${encodeURIComponent(fullname)}?t=${Date.now()}&cache=0`; + return await this.getProxyResponse({ url }, { dataType: 'json' }); + } + + private async getUpstreamAbbreviatedManifests(fullname: string): Promise { + const url = `/${encodeURIComponent(fullname)}?t=${Date.now()}&cache=0`; + return await this.getProxyResponse({ url, headers: { accept: ABBREVIATED_META_TYPE } }, { dataType: 'json' }); + } + private async getUpstreamPackageVersionManifest(fullname: string, versionOrTag: string): Promise { + const url = `/${encodeURIComponent(fullname)}/${encodeURIComponent(versionOrTag)}`; + return await this.getProxyResponse({ url }, { dataType: 'json' }); + } + private async getUpstreamAbbreviatedPackageVersionManifest(fullname: string, versionOrTag: string): Promise { + const url = `/${encodeURIComponent(fullname)}/${encodeURIComponent(versionOrTag)}`; + return await this.getProxyResponse({ url, headers: { accept: ABBREVIATED_META_TYPE } }, { dataType: 'json' }); + } + +} diff --git a/app/port/controller/ProxyCacheController.ts b/app/port/controller/ProxyCacheController.ts new file mode 100644 index 00000000..db4dd3c4 --- /dev/null +++ b/app/port/controller/ProxyCacheController.ts @@ -0,0 +1,156 @@ +import { + HTTPController, + HTTPMethod, + HTTPMethodEnum, + Inject, + HTTPQuery, + HTTPParam, + Context, + EggContext, +} from '@eggjs/tegg'; +import { ForbiddenError, NotFoundError, UnauthorizedError } from 'egg-errors'; +import { AbstractController } from './AbstractController'; +import { ProxyCacheRepository } from '../../repository/ProxyCacheRepository'; +import { Static } from 'egg-typebox-validate/typebox'; +import { QueryPageOptions } from '../typebox'; +import { FULLNAME_REG_STRING } from '../../common/PackageUtil'; +import { + ProxyCacheService, + isPkgManifest, +} from '../../core/service/ProxyCacheService'; +import { SyncMode, PROXY_CACHE_DIR_NAME } from '../../common/constants'; +import { NFSAdapter } from '../../common/adapter/NFSAdapter'; + +@HTTPController() +export class ProxyCacheController extends AbstractController { + @Inject() + private readonly proxyCacheRepository: ProxyCacheRepository; + @Inject() + private readonly proxyCacheService: ProxyCacheService; + @Inject() + private readonly nfsAdapter: NFSAdapter; + + @HTTPMethod({ + method: HTTPMethodEnum.GET, + path: '/-/proxy-cache', + }) + async listProxyCache( + @HTTPQuery() pageSize: Static['pageSize'], + @HTTPQuery() pageIndex: Static['pageIndex'], + ) { + if (this.config.cnpmcore.syncMode !== SyncMode.proxy) { + throw new ForbiddenError('proxy mode is not enabled'); + } + return await this.proxyCacheRepository.listCachedFiles({ + pageSize, + pageIndex, + }); + } + + @HTTPMethod({ + method: HTTPMethodEnum.GET, + path: `/-/proxy-cache/:fullname(${FULLNAME_REG_STRING})`, + }) + async showProxyCaches(@HTTPQuery() pageSize: Static['pageSize'], + @HTTPQuery() pageIndex: Static['pageIndex'], @HTTPParam() fullname: string) { + if (this.config.cnpmcore.syncMode !== SyncMode.proxy) { + throw new ForbiddenError('proxy mode is not enabled'); + } + return await this.proxyCacheRepository.listCachedFiles({ + pageSize, + pageIndex, + }, fullname); + } + + @HTTPMethod({ + method: HTTPMethodEnum.PATCH, + path: `/-/proxy-cache/:fullname(${FULLNAME_REG_STRING})`, + }) + async refreshProxyCaches(@HTTPParam() fullname: string) { + if (this.config.cnpmcore.syncMode !== SyncMode.proxy) { + throw new ForbiddenError('proxy mode is not enabled'); + } + + const refreshList = await this.proxyCacheRepository.findProxyCaches( + fullname, + ); + if (refreshList.length === 0) { + throw new NotFoundError(); + } + const taskList = refreshList + // 仅manifests需要更新，指定版本的package.json文件发布后不会改变 + .filter(i => isPkgManifest(i.fileType)) + .map(async item => { + const task = await this.proxyCacheService.createTask( + `${item.fullname}/${item.fileType}`, + { + fullname: item.fullname, + fileType: item.fileType, + }, + ); + return task; + }); + return { + ok: true, + tasks: await Promise.all(taskList), + }; + } + + @HTTPMethod({ + method: HTTPMethodEnum.DELETE, + path: `/-/proxy-cache/:fullname(${FULLNAME_REG_STRING})`, + }) + async removeProxyCaches(@Context() ctx: EggContext, @HTTPParam() fullname: string) { + const isAdmin = await this.userRoleManager.isAdmin(ctx); + if (!isAdmin) { + throw new UnauthorizedError('only admin can do this'); + } + + if (this.config.cnpmcore.syncMode !== SyncMode.proxy) { + throw new ForbiddenError('proxy mode is not enabled'); + } + + const proxyCachesList = await this.proxyCacheRepository.findProxyCaches( + fullname, + ); + if (proxyCachesList.length === 0) { + throw new NotFoundError(); + } + const removingList = proxyCachesList.map(item => { + return this.proxyCacheService.removeProxyCache(item.fullname, item.fileType, item.version); + }); + await Promise.all(removingList); + return { + ok: true, + result: proxyCachesList, + }; + } + + @HTTPMethod({ + method: HTTPMethodEnum.DELETE, + path: '/-/proxy-cache', + }) + async truncateProxyCaches(@Context() ctx: EggContext) { + const isAdmin = await this.userRoleManager.isAdmin(ctx); + if (!isAdmin) { + throw new UnauthorizedError('only admin can do this'); + } + + if (this.config.cnpmcore.syncMode !== SyncMode.proxy) { + throw new ForbiddenError('proxy mode is not enabled'); + } + + await this.proxyCacheRepository.truncateProxyCache(); + // 尝试删除proxy cache目录，若失败可手动管理 + ctx.runInBackground(async () => { + try { + await this.nfsAdapter.remove(`/${PROXY_CACHE_DIR_NAME}`); + } catch (err) { + this.logger.error('[ProxyCacheService.truncateProxyCaches] remove proxy cache dir error: %s', err); + } + }); + return { + ok: true, + }; + } +} diff --git a/app/port/controller/package/DownloadPackageVersionTar.ts b/app/port/controller/package/DownloadPackageVersionTar.ts index 123ad8ee..5395e2e4 100644 --- a/app/port/controller/package/DownloadPackageVersionTar.ts +++ b/app/port/controller/package/DownloadPackageVersionTar.ts @@ -1,3 +1,4 @@ +import { PassThrough } from 'node:stream'; import { NotFoundError, } from 'egg-errors'; @@ -12,15 +13,24 @@ import { } from '@eggjs/tegg'; import { AbstractController } from '../AbstractController'; import { FULLNAME_REG_STRING, getScopeAndName } from '../../../common/PackageUtil'; +import { SyncMode } from '../../../common/constants'; import { NFSAdapter } from '../../../common/adapter/NFSAdapter'; import { PackageManagerService } from '../../../core/service/PackageManagerService'; -import { SyncMode } from '../../../common/constants'; +import { ProxyCacheService } from '../../../core/service/ProxyCacheService'; +import { PackageSyncerService } from '../../../core/service/PackageSyncerService'; +import { RegistryManagerService } from '../../../core/service/RegistryManagerService'; @HTTPController() export class DownloadPackageVersionTarController extends AbstractController { @Inject() private packageManagerService: PackageManagerService; @Inject() + registryManagerService: RegistryManagerService; + @Inject() + private proxyCacheService: ProxyCacheService; + @Inject() + private packageSyncerService: PackageSyncerService; + @Inject() private nfsAdapter: NFSAdapter; // Support OPTIONS Request on tgz download @@ -54,8 +64,23 @@ export class DownloadPackageVersionTarController extends AbstractController { // check package version in database const allowSync = this.getAllowSync(ctx); - const pkg = await this.getPackageEntityByFullname(fullname, allowSync); - const packageVersion = await this.getPackageVersionEntity(pkg, version, allowSync); + let pkg; + let packageVersion; + try { + pkg = await this.getPackageEntityByFullname(fullname, allowSync); + packageVersion = await this.getPackageVersionEntity(pkg, version, allowSync); + } catch (error) { + if (this.config.cnpmcore.syncMode === SyncMode.proxy) { + // proxy mode package version not found. + const tgzStream = await this.getTgzProxyStream(ctx, fullname, version); + this.packageManagerService.plusPackageVersionCounter(fullname, version); + const passThroughRemoteStream = new PassThrough(); + tgzStream.pipe(passThroughRemoteStream); + ctx.attachment(`${filenameWithVersion}.tgz`); + return passThroughRemoteStream; + } + throw error; + } // read by nfs url if (downloadUrl) { @@ -90,4 +115,22 @@ export class DownloadPackageVersionTarController extends AbstractController { const filenameWithVersion = getScopeAndName(fullnameWithVersion)[1]; return await this.download(ctx, fullname, filenameWithVersion); } + + private async getTgzProxyStream(ctx: EggContext, fullname: string, version: string) { + const { headers, status, res } = await this.proxyCacheService.getPackageVersionTarResponse(fullname, ctx); + ctx.status = status; + ctx.set(headers as { [key: string]: string | string[] }); + ctx.runInBackground(async () => { + const task = await this.packageSyncerService.createTask(fullname, { + authorIp: ctx.ip, + authorId: `pid_${process.pid}`, + tips: `Sync specific version in proxy mode cause by "${ctx.href}"`, + skipDependencies: true, + specificVersions: [ version ], + }); + ctx.logger.info('[DownloadPackageVersionTarController.createSyncTask:success] taskId: %s, fullname: %s', + task.taskId, fullname); + }); + return res; + } } diff --git a/app/port/controller/package/ShowPackageController.ts b/app/port/controller/package/ShowPackageController.ts index 60fda994..75f4ba5e 100644 --- a/app/port/controller/package/ShowPackageController.ts +++ b/app/port/controller/package/ShowPackageController.ts @@ -12,6 +12,10 @@ import { getScopeAndName, FULLNAME_REG_STRING } from '../../../common/PackageUti import { isSyncWorkerRequest } from '../../../common/SyncUtil'; import { PackageManagerService } from '../../../core/service/PackageManagerService'; import { CacheService } from '../../../core/service/CacheService'; +import { ABBREVIATED_META_TYPE, SyncMode } from '../../../common/constants'; +import { ProxyCacheService } from '../../../core/service/ProxyCacheService'; +import { calculateIntegrity } from '../../../common/PackageUtil'; +import { DIST_NAMES } from '../../../core/entity/Package'; @HTTPController() export class ShowPackageController extends AbstractController { @@ -19,6 +23,8 @@ export class ShowPackageController extends AbstractController { private packageManagerService: PackageManagerService; @Inject() private cacheService: CacheService; + @Inject() + private proxyCacheService: ProxyCacheService; @HTTPMethod({ // GET /:fullname @@ -29,8 +35,7 @@ export class ShowPackageController extends AbstractController { async show(@Context() ctx: EggContext, @HTTPParam() fullname: string) { const [ scope, name ] = getScopeAndName(fullname); const isSync = isSyncWorkerRequest(ctx); - const abbreviatedMetaType = 'application/vnd.npm.install-v1+json'; - const isFullManifests = ctx.accepts([ 'json', abbreviatedMetaType ]) !== abbreviatedMetaType; + const isFullManifests = ctx.accepts([ 'json', ABBREVIATED_META_TYPE ]) !== ABBREVIATED_META_TYPE; // handle cache // fallback to db when cache error @@ -64,10 +69,21 @@ export class ShowPackageController extends AbstractController { // handle cache miss let result: { etag: string; data: any, blockReason: string }; - if (isFullManifests) { - result = await this.packageManagerService.listPackageFullManifests(scope, name, isSync); + if (this.config.cnpmcore.syncMode === SyncMode.proxy) { + // proxy mode + const fileType = isFullManifests ? DIST_NAMES.FULL_MANIFESTS : DIST_NAMES.ABBREVIATED_MANIFESTS; + const pkgManifest = await this.proxyCacheService.getPackageManifest(fullname, fileType); + + const nfsBytes = Buffer.from(JSON.stringify(pkgManifest)); + const { shasum: etag } = await calculateIntegrity(nfsBytes); + result = { data: pkgManifest, etag, blockReason: '' }; } else { - result = await this.packageManagerService.listPackageAbbreviatedManifests(scope, name, isSync); + // sync mode + if (isFullManifests) { + result = await this.packageManagerService.listPackageFullManifests(scope, name, isSync); + } else { + result = await this.packageManagerService.listPackageAbbreviatedManifests(scope, name, isSync); + } } const { etag, data, blockReason } = result; // 404, no data diff --git a/app/port/controller/package/ShowPackageVersionController.ts b/app/port/controller/package/ShowPackageVersionController.ts index b80b3102..7cf9db16 100644 --- a/app/port/controller/package/ShowPackageVersionController.ts +++ b/app/port/controller/package/ShowPackageVersionController.ts @@ -7,43 +7,78 @@ import { Context, EggContext, } from '@eggjs/tegg'; -import { NotFoundError } from 'egg-errors'; import { AbstractController } from '../AbstractController'; -import { getScopeAndName, FULLNAME_REG_STRING } from '../../../common/PackageUtil'; +import { + getScopeAndName, + FULLNAME_REG_STRING, +} from '../../../common/PackageUtil'; import { isSyncWorkerRequest } from '../../../common/SyncUtil'; import { PackageManagerService } from '../../../core/service/PackageManagerService'; +import { ProxyCacheService } from '../../../core/service/ProxyCacheService'; import { Spec } from '../../../port/typebox'; +import { ABBREVIATED_META_TYPE, SyncMode } from '../../../common/constants'; +import { DIST_NAMES } from '../../../core/entity/Package'; +import { NotFoundError } from 'egg-errors'; @HTTPController() export class ShowPackageVersionController extends AbstractController { @Inject() private packageManagerService: PackageManagerService; + @Inject() + private proxyCacheService: ProxyCacheService; @HTTPMethod({ // GET /:fullname/:versionSpec path: `/:fullname(${FULLNAME_REG_STRING})/:versionSpec`, method: HTTPMethodEnum.GET, }) - async show(@Context() ctx: EggContext, @HTTPParam() fullname: string, @HTTPParam() versionSpec: string) { + async show( + @Context() ctx: EggContext, + @HTTPParam() fullname: string, + @HTTPParam() versionSpec: string, + ) { // https://github.com/npm/registry/blob/master/docs/responses/package-metadata.md#full-metadata-format ctx.tValidate(Spec, `${fullname}@${versionSpec}`); const [ scope, name ] = getScopeAndName(fullname); const isSync = isSyncWorkerRequest(ctx); - const abbreviatedMetaType = 'application/vnd.npm.install-v1+json'; - const isFullManifests = ctx.accepts([ 'json', abbreviatedMetaType ]) !== abbreviatedMetaType; + const isFullManifests = + ctx.accepts([ 'json', ABBREVIATED_META_TYPE ]) !== ABBREVIATED_META_TYPE; + + const { blockReason, manifest, pkg } = + await this.packageManagerService.showPackageVersionManifest( + scope, + name, + versionSpec, + isSync, + isFullManifests, + ); + const allowSync = this.getAllowSync(ctx); - const { blockReason, manifest, pkg } = await this.packageManagerService.showPackageVersionManifest(scope, name, versionSpec, isSync, isFullManifests); - if (!pkg) { - const allowSync = this.getAllowSync(ctx); - throw this.createPackageNotFoundErrorWithRedirect(fullname, undefined, allowSync); - } if (blockReason) { this.setCDNHeaders(ctx); throw this.createPackageBlockError(blockReason, fullname, versionSpec); } - if (!manifest) { - throw new NotFoundError(`${fullname}@${versionSpec} not found`); + + if (!pkg || !manifest) { + if (this.config.cnpmcore.syncMode === SyncMode.proxy) { + const fileType = isFullManifests + ? DIST_NAMES.MANIFEST + : DIST_NAMES.ABBREVIATED; + return await this.proxyCacheService.getPackageVersionManifest( + fullname, + fileType, + versionSpec, + ); + } + + if (!pkg) { + throw this.createPackageNotFoundErrorWithRedirect(fullname, undefined, allowSync); + } + if (!manifest) { + throw new NotFoundError(`${fullname}@${versionSpec} not found`); + } } + this.setCDNHeaders(ctx); return manifest; } diff --git a/app/port/schedule/CheckProxyCacheUpdateWorker.ts b/app/port/schedule/CheckProxyCacheUpdateWorker.ts new file mode 100644 index 00000000..da7b0c72 --- /dev/null +++ b/app/port/schedule/CheckProxyCacheUpdateWorker.ts @@ -0,0 +1,53 @@ +import { EggAppConfig, EggLogger } from 'egg'; +import { CronParams, Schedule, ScheduleType } from '@eggjs/tegg/schedule'; +import { Inject } from '@eggjs/tegg'; +import { ProxyCacheRepository } from '../../repository/ProxyCacheRepository'; +import { SyncMode } from '../../common/constants'; +import { ProxyCacheService, isPkgManifest } from '../../core/service/ProxyCacheService'; + +@Schedule({ + type: ScheduleType.WORKER, + scheduleData: { + cron: '0 3 * * *', // run every day at 03:00 + }, +}) +export class CheckProxyCacheUpdateWorker { + + @Inject() + private readonly config: EggAppConfig; + + @Inject() + private readonly logger: EggLogger; + + @Inject() + private proxyCacheService: ProxyCacheService; + + @Inject() + private readonly proxyCacheRepository:ProxyCacheRepository; + + async subscribe() { + if (this.config.cnpmcore.syncMode !== SyncMode.proxy) return; + let pageIndex = 0; + let { data: list } = await this.proxyCacheRepository.listCachedFiles({ pageSize: 5, pageIndex }); + while (list.length !== 0) { + for (const item of list) { + try { + if (isPkgManifest(item.fileType)) { + // 仅manifests需要更新，指定版本的package.json文件发布后不会改变 + const task = await this.proxyCacheService.createTask(`${item.fullname}/${item.fileType}`, { + fullname: item.fullname, + fileType: item.fileType, + }); + this.logger.info('[CheckProxyCacheUpdateWorker.subscribe:createTask][%s] taskId: %s, targetName: %s', + pageIndex, task.taskId, task.targetName); + } + } catch (err) { + this.logger.error(err); + } + } + pageIndex++; + list = (await this.proxyCacheRepository.listCachedFiles({ pageSize: 5, pageIndex })).data; + } + + } +} diff --git a/app/port/schedule/CheckRecentlyUpdatedPackages.ts b/app/port/schedule/CheckRecentlyUpdatedPackages.ts index 721376b2..3699ceef 100644 --- a/app/port/schedule/CheckRecentlyUpdatedPackages.ts +++ b/app/port/schedule/CheckRecentlyUpdatedPackages.ts @@ -30,7 +30,7 @@ export class CheckRecentlyUpdatedPackages { private readonly httpclient: EggHttpClient; async subscribe() { - const notAllowUpdateModeList = [ SyncMode.none, SyncMode.admin ]; + const notAllowUpdateModeList = [ SyncMode.none, SyncMode.admin, SyncMode.proxy ]; if (notAllowUpdateModeList.includes(this.config.cnpmcore.syncMode) || !this.config.cnpmcore.enableCheckRecentlyUpdated) return; const pageSize = 36; const pageCount = this.config.env === 'unittest' ? 2 : 5; diff --git a/app/port/schedule/SyncProxyCacheWorker.ts b/app/port/schedule/SyncProxyCacheWorker.ts new file mode 100644 index 00000000..4185e5ea --- /dev/null +++ b/app/port/schedule/SyncProxyCacheWorker.ts @@ -0,0 +1,56 @@ +import { EggAppConfig, EggLogger } from 'egg'; +import { IntervalParams, Schedule, ScheduleType } from '@eggjs/tegg/schedule'; +import { Inject } from '@eggjs/tegg'; +import { ProxyCacheService } from '../../core/service/ProxyCacheService'; +import { SyncMode } from '../../common/constants'; + + +let executingCount = 0; + +@Schedule({ + type: ScheduleType.ALL, + scheduleData: { + interval: 1000, + }, +}) +export class SyncProxyCacheWorker { + @Inject() + private readonly proxyCacheService: ProxyCacheService; + + @Inject() + private readonly config: EggAppConfig; + + @Inject() + private readonly logger: EggLogger; + + async subscribe() { + if (this.config.cnpmcore.syncMode !== SyncMode.proxy) return; + if (executingCount >= this.config.cnpmcore.syncPackageWorkerMaxConcurrentTasks) return; + + executingCount++; + try { + let task = await this.proxyCacheService.findExecuteTask(); + while (task) { + const startTime = Date.now(); + this.logger.info('[SyncProxyCacheWorker:subscribe:executeTask:start][%s] taskId: %s, targetName: %s, attempts: %s, params: %j, updatedAt: %s, delay %sms', + executingCount, task.taskId, task.targetName, task.attempts, task.data, task.updatedAt, + startTime - task.updatedAt.getTime()); + await this.proxyCacheService.executeTask(task); + const use = Date.now() - startTime; + this.logger.info('[SyncProxyCacheWorker:subscribe:executeTask:success][%s] taskId: %s, targetName: %s, use %sms', + executingCount, task.taskId, task.targetName, use); + if (executingCount >= this.config.cnpmcore.syncPackageWorkerMaxConcurrentTasks) { + this.logger.info('[SyncProxyCacheWorker:subscribe:executeTask] current sync task count %s, exceed max concurrent tasks %s', + executingCount, this.config.cnpmcore.syncPackageWorkerMaxConcurrentTasks); + break; + } + // try next task + task = await this.proxyCacheService.findExecuteTask(); + } + } catch (err) { + this.logger.error('[SyncProxyCacheWorker:subscribe:executeTask:error][%s] %s', executingCount, err); + } finally { + executingCount--; + } + } +} diff --git a/app/repository/ProxyCacheRepository.ts b/app/repository/ProxyCacheRepository.ts new file mode 100644 index 00000000..d1af53c0 --- /dev/null +++ b/app/repository/ProxyCacheRepository.ts @@ -0,0 +1,62 @@ +import { AccessLevel, SingletonProto, Inject } from '@eggjs/tegg'; +import { ModelConvertor } from './util/ModelConvertor'; +import type { ProxyCache as ProxyModeCachedFilesModel } from './model/ProxyCache'; +import { ProxyCache as ProxyCacheEntity } from '../core/entity/ProxyCache'; +import { AbstractRepository } from './AbstractRepository'; +import { DIST_NAMES } from '../core/entity/Package'; +import { EntityUtil, PageOptions, PageResult } from '../core/util/EntityUtil'; +@SingletonProto({ + accessLevel: AccessLevel.PUBLIC, +}) +export class ProxyCacheRepository extends AbstractRepository { + @Inject() + private readonly ProxyCache: typeof ProxyModeCachedFilesModel; + + async saveProxyCache(proxyCacheEntity: ProxyCacheEntity) { + let model = proxyCacheEntity.version ? + await this.ProxyCache.findOne({ fullname: proxyCacheEntity.fullname, version: proxyCacheEntity.version, fileType: proxyCacheEntity.fileType }) : + await this.ProxyCache.findOne({ fullname: proxyCacheEntity.fullname, fileType: proxyCacheEntity.fileType }); + if (model) { + model.updatedAt = proxyCacheEntity.updatedAt; + await model.save(); + } else { + try { + model = await ModelConvertor.convertEntityToModel(proxyCacheEntity, this.ProxyCache); + } catch (e) { + e.message = '[ProxyCacheRepository] insert ProxyCache failed: ' + e.message; + throw e; + } + } + return model; + } + + async findProxyCache(fullname: string, fileType: DIST_NAMES, version?: string): Promise { + const model = version ? await this.ProxyCache.findOne({ fullname, version, fileType }) : await this.ProxyCache.findOne({ fullname, fileType }); + if (model) return ModelConvertor.convertModelToEntity(model, ProxyCacheEntity); + return null; + } + + // used by update & delete all cache + async findProxyCaches(fullname: string, version?: string) { + const models = version ? await this.ProxyCache.find({ fullname, version }) : await this.ProxyCache.find({ fullname }); + return models; + } + + async listCachedFiles(page: PageOptions, fullname?: string): Promise> { + const { offset, limit } = EntityUtil.convertPageOptionsToLimitOption(page); + const count = fullname ? await this.ProxyCache.find({ fullname }).count() : await this.ProxyCache.find().count(); + const models = fullname ? await this.ProxyCache.find({ fullname }).offset(offset).limit(limit) : await this.ProxyCache.find().offset(offset).limit(limit); + return { + count, + data: models.map(model => ModelConvertor.convertModelToEntity(model, ProxyCacheEntity)), + }; + } + + async removeProxyCache(fullname: string, fileType: string, version?: string) { + version ? await this.ProxyCache.remove({ fullname, version, fileType }) : await this.ProxyCache.remove({ fullname, fileType }); + } + + async truncateProxyCache() { + await this.ProxyCache.truncate(); + } +} diff --git a/app/repository/model/ProxyCache.ts b/app/repository/model/ProxyCache.ts new file mode 100644 index 00000000..8a99e0d1 --- /dev/null +++ b/app/repository/model/ProxyCache.ts @@ -0,0 +1,33 @@ +import { Attribute, Model } from '@eggjs/tegg/orm'; +import { DataTypes, Bone } from 'leoric'; +import { DIST_NAMES } from '../../core/entity/Package'; + +@Model() +export class ProxyCache extends Bone { + @Attribute(DataTypes.BIGINT, { + primary: true, + autoIncrement: true, + }) + id: bigint; + + @Attribute(DataTypes.DATE, { name: 'gmt_create' }) + createdAt: Date; + + @Attribute(DataTypes.DATE, { name: 'gmt_modified' }) + updatedAt: Date; + + @Attribute(DataTypes.STRING(214)) + fullname: string; + + @Attribute(DataTypes.STRING(30)) + fileType: DIST_NAMES; + + @Attribute(DataTypes.STRING(512), { + unique: true, + }) + filePath: string; + + @Attribute(DataTypes.STRING(214)) + version?: string; + +} diff --git a/package.json b/package.json index e05a4d12..84cd7dcd 100644 --- a/package.json +++ b/package.json @@ -70,6 +70,7 @@ "dependencies": { "@eggjs/tegg": "^3.12.0", "@eggjs/tegg-aop-plugin": "^3.12.0", + "@eggjs/tegg-background-task": "^3.29.0", "@eggjs/tegg-config": "^3.12.0", "@eggjs/tegg-controller-plugin": "^3.12.0", "@eggjs/tegg-eventbus-plugin": "^3.12.0", diff --git a/sql/3.67.0.sql b/sql/3.67.0.sql new file mode 100644 index 00000000..7fd5bf1a --- /dev/null +++ b/sql/3.67.0.sql @@ -0,0 +1,12 @@ +CREATE TABLE IF NOT EXISTS `proxy_caches` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key', + `gmt_create` datetime(3) NOT NULL COMMENT 'create time', + `gmt_modified` datetime(3) NOT NULL COMMENT 'modify time', + `fullname` varchar(214) NOT NULL DEFAULT '' COMMENT '@scope/package name', + `version` varchar(214) COMMENT 'package version', + `file_type` varchar(30) NOT NULL DEFAULT '' COMMENT 'file type', + `file_path` varchar(512) NOT NULL DEFAULT '' COMMENT 'nfs file path', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_package_version_path_name` (`file_path`), + UNIQUE KEY `ux_package_version_file_name` (`fullname`, `file_type`, `version`) +) ENGINE=InnoDB DEFAULT COLLATE utf8mb3_unicode_ci CHARSET=utf8mb3 COMMENT 'proxy mode cached files index'; diff --git a/test/core/service/ProxyCacheService.test.ts b/test/core/service/ProxyCacheService.test.ts new file mode 100644 index 00000000..721cd8d8 --- /dev/null +++ b/test/core/service/ProxyCacheService.test.ts @@ -0,0 +1,318 @@ +import assert from 'assert'; +import { app, mock } from 'egg-mock/bootstrap'; +import { TestUtil } from '../../TestUtil'; +import { ProxyCacheService } from '../../../app/core/service/ProxyCacheService'; +import { ProxyCacheRepository } from '../../../app/repository/ProxyCacheRepository'; +import { DIST_NAMES } from '../../../app/core/entity/Package'; +import { NFSAdapter } from '../../../app/common/adapter/NFSAdapter'; +import { ProxyCache } from '../../../app/core/entity/ProxyCache'; +import { TaskService } from '../../../app/core/service/TaskService'; + +describe('test/core/service/ProxyCacheService/index.test.ts', () => { + let proxyCacheService: ProxyCacheService; + let proxyCacheRepository: ProxyCacheRepository; + + beforeEach(async () => { + proxyCacheService = await app.getEggObject(ProxyCacheService); + proxyCacheRepository = await app.getEggObject(ProxyCacheRepository); + }); + + describe('getPackageVersionTarResponse()', () => { + it('should stop proxy when hit block list', async () => { + const name = 'cnpmcore-test-sync-blocklist'; + mock(app.config.cnpmcore, 'syncPackageBlockList', [ name ]); + try { + await proxyCacheService.getPackageVersionTarResponse(name, app.mockContext()); + } catch (error) { + assert(error.options.message.includes('block list')); + } + }); + }); + + describe('getPackageManifest()', () => { + it('should invoke getRewrittenManifest first.', async () => { + mock(proxyCacheService, 'getRewrittenManifest', async () => { + return { name: 'mock info' }; + }); + const manifest = await proxyCacheService.getPackageManifest( + 'foo', + DIST_NAMES.FULL_MANIFESTS, + ); + assert.equal(manifest.name, 'mock info'); + }); + + it('should read data from nfs when cached.', async () => { + const nfsAdapter = await app.getEggObject(NFSAdapter); + mock(proxyCacheService, 'getRewrittenManifest', async () => { + return { name: 'foo remote mock info' }; + }); + await proxyCacheRepository.saveProxyCache( + ProxyCache.create({ + fullname: 'foo', + fileType: DIST_NAMES.FULL_MANIFESTS, + }), + ); + mock(nfsAdapter, 'getBytes', async () => { + return Buffer.from('{"name": "nfs mock info"}'); + }); + const manifest = await proxyCacheService.getPackageManifest( + 'foo', + DIST_NAMES.FULL_MANIFESTS, + ); + assert.equal(manifest.name, 'nfs mock info'); + }); + }); + + describe('getPackageVersionManifest()', () => { + it('should invoke getRewrittenManifest first.', async () => { + mock(proxyCacheService, 'getRewrittenManifest', async () => { + return { name: 'mock package version info' }; + }); + const manifest = await proxyCacheService.getPackageVersionManifest( + 'foo', + DIST_NAMES.MANIFEST, + '1.0.0', + ); + assert.equal(manifest.name, 'mock package version info'); + }); + + it('should read data from nfs when cached.', async () => { + const nfsAdapter = await app.getEggObject(NFSAdapter); + mock(proxyCacheService, 'getRewrittenManifest', async () => { + return { name: 'foo remote mock info' }; + }); + await proxyCacheRepository.saveProxyCache( + ProxyCache.create({ + fullname: 'foo', + fileType: DIST_NAMES.MANIFEST, + version: '1.0.0', + }), + ); + mock(nfsAdapter, 'getBytes', async () => { + return Buffer.from('{"name": "package version nfs mock info"}'); + }); + const manifest = await proxyCacheService.getPackageVersionManifest( + 'foo', + DIST_NAMES.MANIFEST, + '1.0.0', + ); + assert.equal(manifest.name, 'package version nfs mock info'); + }); + + it('should get correct verison via tag and cache the pkg manifest', async () => { + app.mockHttpclient('https://registry.npmjs.org/foobar/latest', 'GET', { + data: await TestUtil.readFixturesFile( + 'registry.npmjs.org/foobar/1.0.0/abbreviated.json', + ), + persist: false, + }); + + mock(proxyCacheService, 'getUpstreamAbbreviatedManifests', async () => { + return { + status: 200, + data: await TestUtil.readJSONFile( + TestUtil.getFixtures('registry.npmjs.org/abbreviated_foobar.json'), + ), + }; + }); + // get manifest by http + const pkgVersionManifest = + await proxyCacheService.getPackageVersionManifest( + 'foobar', + DIST_NAMES.MANIFEST, + 'latest', + ); + assert(pkgVersionManifest); + assert.equal(pkgVersionManifest.version, '1.0.0'); + }); + }); + + describe('getRewrittenManifest()', () => { + it('should get full package manifest', async () => { + const data = await TestUtil.readJSONFile( + TestUtil.getFixtures('registry.npmjs.org/foobar.json'), + ); + mock(proxyCacheService, 'getUpstreamFullManifests', async () => { + return { + status: 200, + data, + }; + }); + const manifest = await proxyCacheService.getRewrittenManifest( + 'foobar', + DIST_NAMES.FULL_MANIFESTS, + ); + const versionArr = Object.values(manifest.versions); + for (const i of versionArr) { + assert(i.dist.tarball.includes('http://localhost:7001')); + } + }); + + it('should get abbreviated package manifest', async () => { + const data = await TestUtil.readJSONFile( + TestUtil.getFixtures('registry.npmjs.org/abbreviated_foobar.json'), + ); + mock(proxyCacheService, 'getUpstreamAbbreviatedManifests', async () => { + return { + status: 200, + data, + }; + }); + const manifest = await proxyCacheService.getRewrittenManifest( + 'foobar', + DIST_NAMES.ABBREVIATED_MANIFESTS, + ); + const versionArr = Object.values(manifest.versions); + for (const i of versionArr) { + assert(i.dist.tarball.includes('http://localhost:7001')); + } + }); + + it('should get full package version manifest', async () => { + const data = await TestUtil.readJSONFile( + TestUtil.getFixtures('registry.npmjs.org/foobar/1.0.0/package.json'), + ); + mock(proxyCacheService, 'getUpstreamPackageVersionManifest', async () => { + return { + status: 200, + data, + }; + }); + const manifest = await proxyCacheService.getRewrittenManifest( + 'foobar', + DIST_NAMES.MANIFEST, + '1.0.0', + ); + assert(manifest.dist); + assert(manifest.dist.tarball.includes('http://localhost:7001')); + }); + + it('should get abbreviated package version manifest', async () => { + const data = await TestUtil.readJSONFile( + TestUtil.getFixtures( + 'registry.npmjs.org/foobar/1.0.0/abbreviated.json', + ), + ); + mock( + proxyCacheService, + 'getUpstreamAbbreviatedPackageVersionManifest', + async () => { + return { + status: 200, + data, + }; + }, + ); + const manifest = await proxyCacheService.getRewrittenManifest( + 'foobar', + DIST_NAMES.ABBREVIATED, + '1.0.0', + ); + assert(manifest.dist); + assert(manifest.dist.tarball.includes('http://localhost:7001')); + }); + }); + + describe('removeProxyCache()', () => { + it('should remove cache', async () => { + await proxyCacheRepository.saveProxyCache( + ProxyCache.create({ + fullname: 'foo-bar', + fileType: DIST_NAMES.ABBREVIATED, + version: '1.0.0', + }), + ); + + await proxyCacheService.removeProxyCache( + 'foobar', + DIST_NAMES.ABBREVIATED, + '1.0.0', + ); + + const resultAfter = await proxyCacheRepository.findProxyCache( + 'foobar', + DIST_NAMES.ABBREVIATED, + '1.0.0', + ); + assert.equal(resultAfter, undefined); + }); + }); + + describe('createTask(), findExecuteTask()', () => { + it('should create task, and can be found.', async () => { + const task = await proxyCacheService.createTask( + `foobar/${DIST_NAMES.FULL_MANIFESTS}`, + { + fullname: 'foo', + fileType: DIST_NAMES.FULL_MANIFESTS, + }, + ); + assert(task); + assert.equal(task.targetName, `foobar/${DIST_NAMES.FULL_MANIFESTS}`); + const task2 = await proxyCacheService.findExecuteTask(); + assert.equal(task.id, task2?.id); + }); + + it('should be 500 when file type is package version manifest.', async () => { + try { + await proxyCacheService.createTask( + `foobar/${DIST_NAMES.FULL_MANIFESTS}`, + { + fullname: 'foo', + fileType: DIST_NAMES.MANIFEST, + }, + ); + } catch (error) { + assert.equal(error.status, 500); + } + }); + }); + + describe('executeTask()', () => { + it('should throw not found error', async () => { + const taskService = await app.getEggObject(TaskService); + const task = await proxyCacheService.createTask( + `foobar/${DIST_NAMES.FULL_MANIFESTS}`, + { + fullname: 'foo', + fileType: DIST_NAMES.FULL_MANIFESTS, + }, + ); + await proxyCacheService.executeTask(task); + const stream = await taskService.findTaskLog(task); + assert(stream); + const log = await TestUtil.readStreamToLog(stream); + assert(log.includes('can not found record in repo')); + }); + + it('should update success', async () => { + const taskService = await app.getEggObject(TaskService); + await proxyCacheRepository.saveProxyCache( + ProxyCache.create({ + fullname: 'foobar', + fileType: DIST_NAMES.FULL_MANIFESTS, + }), + ); + const task = await proxyCacheService.createTask( + `foobar/${DIST_NAMES.FULL_MANIFESTS}`, + { + fullname: 'foobar', + fileType: DIST_NAMES.FULL_MANIFESTS, + }, + ); + mock(proxyCacheService, 'getUpstreamFullManifests', async () => { + return { + status: 200, + data: await TestUtil.readJSONFile( + TestUtil.getFixtures('registry.npmjs.org/foobar.json'), + ), + }; + }); + await proxyCacheService.executeTask(task); + const stream = await taskService.findTaskLog(task); + assert(stream); + const log = await TestUtil.readStreamToLog(stream); + assert(log.includes('Update Success')); + }); + }); +}); diff --git a/test/fixtures/registry.npmjs.org/abbreviated_foobar.json b/test/fixtures/registry.npmjs.org/abbreviated_foobar.json new file mode 100644 index 00000000..db6c2348 --- /dev/null +++ b/test/fixtures/registry.npmjs.org/abbreviated_foobar.json @@ -0,0 +1,43 @@ +{ + "dist-tags": { + "latest": "1.1.0" + }, + "modified": "2022-01-26T20:31:13.648Z", + "name": "foobar", + "description": "cnpmcore mock json", + "versions": { + "1.0.0": { + "name": "foobar", + "version": "1.0.0", + "dependencies": {}, + "devDependencies": {}, + "directories": {}, + "dist": { + "shasum": "116ff82f61ce1e9545c5ba302ead651f5bb31247", + "tarball": "https://registry.npmjs.org/foobar/-/foobar-1.0.0.tgz", + "size": 10240, + "integrity": "sha512-ojyo6LOnMwMDfdEjVgzMpZ4UdV4iXauePAsYEZU746H95qF/PDQ8Q8lrACnuTp/5otZPabr2mFivvLacbXLyMg==" + }, + "engines": { + "node": "~0.4.12" + } + }, + "1.1.0": { + "name": "foobar", + "version": "1.1.0", + "dependencies": {}, + "devDependencies": {}, + "directories": {}, + "dist": { + "shasum": "0b76cc4e6c5b8d592db598fb870414c290511df2", + "tarball": "https://registry.npmjs.org/foobar/-/foobar-1.1.0.tgz", + "size": 10240, + "integrity": "sha512-Tgp9F/BvfxSOVik+cgMFLn5GUo0mjOrUeWsSwa1TgjZtuglfpIUpP5OaIYbyI8R2q8NVcnfYRGd5OuNxVC+o+g==" + }, + "engines": { + "node": "~0.4.12" + } + } + }, + "_source_registry_name": "default" +} \ No newline at end of file diff --git a/test/fixtures/registry.npmjs.org/foobar/1.0.0/abbreviated.json b/test/fixtures/registry.npmjs.org/foobar/1.0.0/abbreviated.json new file mode 100644 index 00000000..4cb6fd2c --- /dev/null +++ b/test/fixtures/registry.npmjs.org/foobar/1.0.0/abbreviated.json @@ -0,0 +1,16 @@ +{ + "name": "foobar", + "version": "1.0.0", + "dependencies": {}, + "devDependencies": {}, + "directories": {}, + "dist": { + "shasum": "116ff82f61ce1e9545c5ba302ead651f5bb31247", + "tarball": "https://registry.npmjs.org/foobar/-/foobar-1.0.0.tgz", + "size": 10240, + "integrity": "sha512-ojyo6LOnMwMDfdEjVgzMpZ4UdV4iXauePAsYEZU746H95qF/PDQ8Q8lrACnuTp/5otZPabr2mFivvLacbXLyMg==" + }, + "engines": { + "node": "~0.4.12" + } +} \ No newline at end of file diff --git a/test/fixtures/registry.npmjs.org/foobar/1.0.0/package.json b/test/fixtures/registry.npmjs.org/foobar/1.0.0/package.json new file mode 100644 index 00000000..4cb6fd2c --- /dev/null +++ b/test/fixtures/registry.npmjs.org/foobar/1.0.0/package.json @@ -0,0 +1,16 @@ +{ + "name": "foobar", + "version": "1.0.0", + "dependencies": {}, + "devDependencies": {}, + "directories": {}, + "dist": { + "shasum": "116ff82f61ce1e9545c5ba302ead651f5bb31247", + "tarball": "https://registry.npmjs.org/foobar/-/foobar-1.0.0.tgz", + "size": 10240, + "integrity": "sha512-ojyo6LOnMwMDfdEjVgzMpZ4UdV4iXauePAsYEZU746H95qF/PDQ8Q8lrACnuTp/5otZPabr2mFivvLacbXLyMg==" + }, + "engines": { + "node": "~0.4.12" + } +} \ No newline at end of file diff --git a/test/port/controller/ProxyCacheController/index.test.ts b/test/port/controller/ProxyCacheController/index.test.ts new file mode 100644 index 00000000..ab3b58ed --- /dev/null +++ b/test/port/controller/ProxyCacheController/index.test.ts @@ -0,0 +1,191 @@ +import { strict as assert } from 'node:assert'; +import { app, mock } from 'egg-mock/bootstrap'; +// import { TestUtil } from '../../../../test/TestUtil'; +import { DIST_NAMES } from '../../../../app/core/entity/Package'; +import { ProxyCache } from '../../../../app/core/entity/ProxyCache'; +import { ProxyCacheRepository } from '../../../../app/repository/ProxyCacheRepository'; +import { TaskRepository } from '../../../../app/repository/TaskRepository'; +import { SyncMode } from '../../../../app/common/constants'; +import { TestUtil } from '../../../TestUtil'; + +describe('test/port/controller/PackageVersionFileController/listFiles.test.ts', () => { + // let publisher; + // let adminUser; + let proxyCacheRepository: ProxyCacheRepository; + beforeEach(async () => { + proxyCacheRepository = await app.getEggObject(ProxyCacheRepository); + await proxyCacheRepository.saveProxyCache( + ProxyCache.create({ + fullname: 'foo-bar', + fileType: DIST_NAMES.ABBREVIATED, + version: '1.0.0', + }), + ); + await proxyCacheRepository.saveProxyCache( + ProxyCache.create({ + fullname: 'foo-bar', + fileType: DIST_NAMES.ABBREVIATED_MANIFESTS, + }), + ); + await proxyCacheRepository.saveProxyCache( + ProxyCache.create({ + fullname: 'foobar', + fileType: DIST_NAMES.ABBREVIATED, + version: '1.0.0', + }), + ); + await proxyCacheRepository.saveProxyCache( + ProxyCache.create({ + fullname: 'foobar', + fileType: DIST_NAMES.ABBREVIATED_MANIFESTS, + }), + ); + }); + + describe('[GET /-/proxy-cache] listProxyCache()', () => { + it('should 403 when syncMode !== proxy', async () => { + await app.httpRequest().get('/-/proxy-cache').expect(403); + }); + + it('should 200 when syncMode === proxy', async () => { + mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); + mock(app.config.cnpmcore, 'redirectNotFound', false); + const res = await app.httpRequest().get('/-/proxy-cache').expect(200); + assert(res.body.data.length === 4); + }); + + it('should pageSize work', async () => { + mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); + mock(app.config.cnpmcore, 'redirectNotFound', false); + const res0 = await app.httpRequest().get('/-/proxy-cache?pageSize=2&pageIndex=0').expect(200); + assert(res0.body.data.length === 2); + const res1 = await app.httpRequest().get('/-/proxy-cache?pageSize=2&pageIndex=1').expect(200); + assert(res1.body.data.length === 2); + const res2 = await app.httpRequest().get('/-/proxy-cache?pageSize=2&pageIndex=2').expect(200); + assert(res2.body.data.length === 0); + assert(res2.body.count === 4); + }); + }); + + describe('[GET /-/proxy-cache/:fullname] showProxyCaches()', () => { + it('should 403 when syncMode !== proxy', async () => { + await app.httpRequest().get('/-/proxy-cache/foo-bar').expect(403); + }); + + it('should 200 when search "foo-bar"', async () => { + mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); + mock(app.config.cnpmcore, 'redirectNotFound', false); + const res = await app.httpRequest().get('/-/proxy-cache/foo-bar').expect(200); + assert(res.body.count === 2); + }); + + it('should 404 when not found', async () => { + mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); + mock(app.config.cnpmcore, 'redirectNotFound', false); + const res = await app.httpRequest().get('/-/proxy-cache/foo-bar-xxx').expect(200); + assert(res.body.count === 0); + }); + }); + + describe('[PATCH /-/proxy-cache/:fullname] refreshProxyCaches()', () => { + it('should 403 when syncMode !== proxy', async () => { + await app.httpRequest().patch('/-/proxy-cache/foo-bar').expect(403); + }); + + it('should create two tasks.', async () => { + mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); + mock(app.config.cnpmcore, 'redirectNotFound', false); + const res = await app + .httpRequest() + .patch('/-/proxy-cache/foo-bar') + .expect(200); + // 仅需创建ABBREVIATED_MANIFESTS的更新任务 + assert(res.body.tasks.length === 1); + const taskRepository = await app.getEggObject(TaskRepository); + const waitingTask = await taskRepository.findTask( + res.body.tasks[0].taskId, + ); + assert(waitingTask); + }); + + it('should 404 when not found', async () => { + mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); + mock(app.config.cnpmcore, 'redirectNotFound', false); + await app.httpRequest().patch('/-/proxy-cache/foo-bar-xxx').expect(404); + }); + }); + + describe('[DELETE /-/proxy-cache/:fullname] removeProxyCaches()', () => { + it('should 403 when syncMode !== proxy', async () => { + const adminUser = await TestUtil.createAdmin(); + await app + .httpRequest() + .delete('/-/proxy-cache/foo-bar') + .set('authorization', adminUser.authorization) + .expect(403); + }); + + it('should 403 when not login', async () => { + mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); + mock(app.config.cnpmcore, 'redirectNotFound', false); + await app.httpRequest().delete('/-/proxy-cache/foo-bar').expect(401); + }); + + it('should delete all packages about "foo-bar".', async () => { + mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); + mock(app.config.cnpmcore, 'redirectNotFound', false); + const adminUser = await TestUtil.createAdmin(); + const res = await app + .httpRequest() + .delete('/-/proxy-cache/foo-bar') + .set('authorization', adminUser.authorization) + .expect(200); + assert(res.body.ok === true); + assert(res.body.result.length === 2); + const res1 = await app.httpRequest().get('/-/proxy-cache').expect(200); + assert(res1.body.data.length === 2); + }); + + it('should 404 when not found', async () => { + mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); + mock(app.config.cnpmcore, 'redirectNotFound', false); + const adminUser = await TestUtil.createAdmin(); + await app + .httpRequest() + .patch('/-/proxy-cache/foo-bar-xxx') + .set('authorization', adminUser.authorization) + .expect(404); + }); + }); + + describe('[DELETE /-/proxy-cache] truncateProxyCaches()', () => { + it('should 403 when syncMode !== proxy', async () => { + const adminUser = await TestUtil.createAdmin(); + await app + .httpRequest() + .delete('/-/proxy-cache') + .set('authorization', adminUser.authorization) + .expect(403); + }); + + it('should 403 when not login', async () => { + mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); + mock(app.config.cnpmcore, 'redirectNotFound', false); + await app.httpRequest().delete('/-/proxy-cache').expect(401); + }); + + it('should delete all packages about "foo-bar".', async () => { + mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); + mock(app.config.cnpmcore, 'redirectNotFound', false); + const adminUser = await TestUtil.createAdmin(); + const res = await app + .httpRequest() + .delete('/-/proxy-cache') + .set('authorization', adminUser.authorization) + .expect(200); + assert(res.body.ok === true); + const res1 = await app.httpRequest().get('/-/proxy-cache').expect(200); + assert(res1.body.data.length === 0); + }); + }); +}); diff --git a/test/port/controller/package/DownloadPackageVersionTarController.test.ts b/test/port/controller/package/DownloadPackageVersionTarController.test.ts index d3b10171..4b0e182d 100644 --- a/test/port/controller/package/DownloadPackageVersionTarController.test.ts +++ b/test/port/controller/package/DownloadPackageVersionTarController.test.ts @@ -1,7 +1,9 @@ -import { strict as assert } from 'node:assert'; +import { strict as assert } from 'assert'; +import { setTimeout } from 'node:timers/promises'; import { app, mock } from 'egg-mock/bootstrap'; import { TestUtil } from '../../../../test/TestUtil'; import { NFSClientAdapter } from '../../../../app/infra/NFSClientAdapter'; +import { SyncMode } from '../../../../app/common/constants'; describe('test/port/controller/package/DownloadPackageVersionTarController.test.ts', () => { let publisher: any; @@ -299,6 +301,23 @@ describe('test/port/controller/package/DownloadPackageVersionTarController.test. app.expectLog('[middleware:ErrorHandler][syncPackage] create sync package'); }); + it('should create sync specific version task when package version tgz not found in proxy mode ', async () => { + mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); + mock(app.config.cnpmcore, 'redirectNotFound', false); + app.mockHttpclient('https://registry.npmjs.org/foobar/-/foobar-1.0.0.tgz', 'GET', { + data: await TestUtil.readFixturesFile('registry.npmjs.org/foobar/-/foobar-1.0.0.tgz'), + persist: false, + }); + const res = await app.httpRequest() + .get('/foobar/-/foobar-1.0.0.tgz') + .set('user-agent', publisher.ua + ' node/16.0.0') + .set('Accept', 'application/vnd.npm.install-v1+json'); + assert(res.status === 200); + // run in background + await setTimeout(500); + app.expectLog('[DownloadPackageVersionTarController.createSyncTask:success]'); + }); + }); describe('[GET /:fullname/download/:fullname-:version.tgz] deprecatedDownload()', () => { diff --git a/test/port/controller/package/ShowPackageController.test.ts b/test/port/controller/package/ShowPackageController.test.ts index 0745462a..77dad8d7 100644 --- a/test/port/controller/package/ShowPackageController.test.ts +++ b/test/port/controller/package/ShowPackageController.test.ts @@ -7,6 +7,7 @@ import { PackageManagerService } from '../../../../app/core/service/PackageManag import { CacheService } from '../../../../app/core/service/CacheService'; import { DistRepository } from '../../../../app/repository/DistRepository'; import { BugVersionService } from '../../../../app/core/service/BugVersionService'; +import { SyncMode } from '../../../../app/common/constants'; describe('test/port/controller/package/ShowPackageController.test.ts', () => { let packageRepository: PackageRepository; @@ -863,5 +864,22 @@ describe('test/port/controller/package/ShowPackageController.test.ts', () => { assert(res.status === 302); assert(res.headers.location === 'https://registry.npmjs.org/egg'); }); + + it('should read manifest from source in proxy mode', async () => { + mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); + mock(app.config.cnpmcore, 'redirectNotFound', false); + const data = await TestUtil.readJSONFile(TestUtil.getFixtures('registry.npmjs.org/abbreviated_foobar.json')); + app.mockHttpclient('https://registry.npmjs.org/foobar', 'GET', { + data, + persist: false, + }); + const res = await app.httpRequest() + .get('/foobar') + .set('user-agent', publisher.ua + ' node/16.0.0') + .set('Accept', 'application/vnd.npm.install-v1+json'); + assert(res.status === 200); + assert(res.body.description === 'cnpmcore mock json'); + assert(res.body.versions['1.0.0'].dist.tarball.includes(app.config.cnpmcore.registry)); + }); }); }); diff --git a/test/port/controller/package/ShowPackageVersionController.test.ts b/test/port/controller/package/ShowPackageVersionController.test.ts index d99b663d..5857da7c 100644 --- a/test/port/controller/package/ShowPackageVersionController.test.ts +++ b/test/port/controller/package/ShowPackageVersionController.test.ts @@ -3,6 +3,7 @@ import { app, mock } from 'egg-mock/bootstrap'; import { TestUtil } from '../../../../test/TestUtil'; import { BugVersion } from '../../../../app/core/entity/BugVersion'; import { BugVersionService } from '../../../../app/core/service/BugVersionService'; +import { SyncMode } from '../../../../app/common/constants'; describe('test/port/controller/package/ShowPackageVersionController.test.ts', () => { let publisher; @@ -368,5 +369,21 @@ describe('test/port/controller/package/ShowPackageVersionController.test.ts', () .expect(200); assert(res.body._source_registry_name === 'self'); }); + + it('should read package version manifest from source in proxy mode', async () => { + mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); + mock(app.config.cnpmcore, 'redirectNotFound', false); + const data = await TestUtil.readJSONFile(TestUtil.getFixtures('registry.npmjs.org/foobar/1.0.0/abbreviated.json')); + app.mockHttpclient('https://registry.npmjs.org/foobar/1.0.0', 'GET', { + data, + persist: false, + }); + const res = await app.httpRequest() + .get('/foobar/1.0.0') + .set('user-agent', publisher.ua + ' node/16.0.0') + .set('Accept', 'application/vnd.npm.install-v1+json'); + assert(res.status === 200); + assert(res.body.dist.tarball.includes(app.config.cnpmcore.registry)); + }); }); }); diff --git a/test/repository/ProxyCachePepository.test.ts b/test/repository/ProxyCachePepository.test.ts new file mode 100644 index 00000000..7ec93dd0 --- /dev/null +++ b/test/repository/ProxyCachePepository.test.ts @@ -0,0 +1,70 @@ +import assert from 'assert'; +import { app } from 'egg-mock/bootstrap'; +import { ProxyCacheRepository } from '../../app/repository/ProxyCacheRepository'; +import { ProxyCache } from '../../app/core/entity/ProxyCache'; +import { DIST_NAMES } from '../../app/core/entity/Package'; + +describe('test/repository/ProxyCacheRepository.test.ts', () => { + let proxyCacheRepository: ProxyCacheRepository; + let proxyCacheModel: ProxyCache; + + beforeEach(async () => { + proxyCacheRepository = await app.getEggObject(ProxyCacheRepository); + proxyCacheModel = await proxyCacheRepository.saveProxyCache(ProxyCache.create({ + fullname: 'foo-bar', + fileType: DIST_NAMES.FULL_MANIFESTS, + })); + }); + + describe('ProxyCacheRepository', () => { + it('create work', async () => { + const newProxyCache = await proxyCacheRepository.saveProxyCache(ProxyCache.create({ + fullname: 'foo-bar-new', + fileType: DIST_NAMES.FULL_MANIFESTS, + })); + assert(newProxyCache); + assert(newProxyCache.fullname === 'foo-bar-new'); + }); + + it('update work', async () => { + const beforeUpdateTime = proxyCacheModel.updatedAt.getTime(); + const updatedProxyCache = await proxyCacheRepository.saveProxyCache(ProxyCache.update(proxyCacheModel)); + assert(updatedProxyCache); + assert(updatedProxyCache.fullname === 'foo-bar'); + const afterUpdateTime = updatedProxyCache.updatedAt.getTime(); + assert(afterUpdateTime !== beforeUpdateTime); + assert(afterUpdateTime - beforeUpdateTime < 1000); + }); + + it('list work', async () => { + const proxyCaches = await proxyCacheRepository.listCachedFiles({}); + assert(proxyCaches.count === 1); + }); + + it('query null', async () => { + const queryRes = await proxyCacheRepository.findProxyCache('not-exists', DIST_NAMES.FULL_MANIFESTS); + assert(queryRes === null); + }); + + it('query work', async () => { + const queryRes = await proxyCacheRepository.findProxyCache('foo-bar', DIST_NAMES.FULL_MANIFESTS); + assert(queryRes?.fullname === 'foo-bar'); + }); + + it('remove work', async () => { + await proxyCacheRepository.removeProxyCache('foo-bar', DIST_NAMES.FULL_MANIFESTS); + const { count } = await proxyCacheRepository.listCachedFiles({}); + assert.equal(count, 0); + }); + + it('truncate work', async () => { + await proxyCacheRepository.saveProxyCache(ProxyCache.create({ + fullname: 'foo-bar-new', + fileType: DIST_NAMES.FULL_MANIFESTS, + })); + await proxyCacheRepository.truncateProxyCache(); + const { count } = await proxyCacheRepository.listCachedFiles({}); + assert.equal(count, 0); + }); + }); +}); diff --git a/test/schedule/CheckProxyCacheUpdateWorker.test.ts b/test/schedule/CheckProxyCacheUpdateWorker.test.ts new file mode 100644 index 00000000..20b5e4de --- /dev/null +++ b/test/schedule/CheckProxyCacheUpdateWorker.test.ts @@ -0,0 +1,28 @@ +import assert from 'assert'; +import { app, mock } from 'egg-mock/bootstrap'; +import { SyncMode } from '../../app/common/constants'; +import { ProxyCacheRepository } from '../../app/repository/ProxyCacheRepository'; +import { ProxyCache } from '../../app/core/entity/ProxyCache'; +import { DIST_NAMES } from '../../app/core/entity/Package'; +import { TaskService } from '../../app/core/service/TaskService'; +import { TaskType } from '../../app/common/enum/Task'; + +const CheckProxyCacheUpdateWorkerPath = require.resolve('../../app/port/schedule/CheckProxyCacheUpdateWorker'); + +describe('test/schedule/CheckProxyCacheUpdateWorker.test.ts', () => { + it('should create update task by repo', async () => { + mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); + mock(app.config.cnpmcore, 'redirectNotFound', false); + const proxyCacheRepository = await app.getEggObject(ProxyCacheRepository); + const taskService = await app.getEggObject(TaskService); + await proxyCacheRepository.saveProxyCache(ProxyCache.create({ + fullname: 'foo-bar', + fileType: DIST_NAMES.FULL_MANIFESTS, + })); + await app.runSchedule(CheckProxyCacheUpdateWorkerPath); + const task = await taskService.findExecuteTask(TaskType.UpdateProxyCache); + assert(task); + assert.equal(task.targetName, `foo-bar/${DIST_NAMES.FULL_MANIFESTS}`); + }); + +}); diff --git a/test/schedule/SyncProxyCacheWorker.test.ts b/test/schedule/SyncProxyCacheWorker.test.ts new file mode 100644 index 00000000..42098a06 --- /dev/null +++ b/test/schedule/SyncProxyCacheWorker.test.ts @@ -0,0 +1,37 @@ +import { app, mock } from 'egg-mock/bootstrap'; +import { SyncMode } from '../../app/common/constants'; +import { ProxyCacheRepository } from '../../app/repository/ProxyCacheRepository'; +import { ProxyCache } from '../../app/core/entity/ProxyCache'; +import { DIST_NAMES } from '../../app/core/entity/Package'; +import { ProxyCacheService } from '../../app/core/service/ProxyCacheService'; + +const SyncProxyCacheWorkerPath = require.resolve('../../app/port/schedule/SyncProxyCacheWorker'); + +describe('test/schedule/SyncProxyCacheWorker.test.ts', () => { + + beforeEach(async () => { + mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); + mock(app.config.cnpmcore, 'redirectNotFound', false); + }); + + it('should execute task success', async () => { + + const proxyCacheRepository = await app.getEggObject(ProxyCacheRepository); + const proxyCacheService = await app.getEggObject(ProxyCacheService); + await proxyCacheRepository.saveProxyCache(ProxyCache.create({ + fullname: 'foobar', + fileType: DIST_NAMES.FULL_MANIFESTS, + })); + + + await proxyCacheService.createTask(`foobar/${DIST_NAMES.ABBREVIATED_MANIFESTS}`, { + fullname: 'foobar', + fileType: DIST_NAMES.ABBREVIATED_MANIFESTS, + }); + + await app.runSchedule(SyncProxyCacheWorkerPath); + app.expectLog('[SyncProxyCacheWorker:subscribe:executeTask:start]'); + app.expectLog('[SyncProxyCacheWorker:subscribe:executeTask:success]'); + }); + +});