diff --git a/.env.test b/.env.test index c015226b9..6fcda9cb4 100644 --- a/.env.test +++ b/.env.test @@ -3,7 +3,7 @@ NODE_ENV=test PORT=5421 IPFS_NODE_URL=http://host.docker.internal:5003 -PUBLIC_IPFS_RESOLVER=https://pub.desci.com +PUBLIC_IPFS_RESOLVER=http://host.docker.internal:5003 GUEST_IPFS_NODE_URL=http://host.docker.internal:5003 # Share the same kubo node ### Database - Postgres diff --git a/.github/workflows/build-openalex-importer.yaml b/.github/workflows/build-openalex-importer.yaml new file mode 100644 index 000000000..eca208990 --- /dev/null +++ b/.github/workflows/build-openalex-importer.yaml @@ -0,0 +1,95 @@ +on: + push: + paths: + - .github/workflows/build-openalex-importer.yaml + - openalex-importer/** + branches: + - main + - develop + +name: Build openalex-importer + +env: + AWS_DEFAULT_REGION: us-east-2 + AWS_DEFAULT_OUTPUT: json + AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + CONTAINER_IMAGE: openalex-importer + DOCKER_BUILDKIT: 1 + +jobs: + build-and-push: + name: Build and push image + runs-on: blacksmith-4vcpu-ubuntu-2204 + outputs: + image_tag: ${{ steps.image-tag.outputs.tag }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Generate image tag + id: image-tag + run: echo "tag=$(echo ${{ github.sha }} | cut -c1-7)-$(date +%s)" >> $GITHUB_OUTPUT + + - name: Build and tag the image + run: | + docker build \ + -t $CONTAINER_IMAGE:latest \ + -t $CONTAINER_IMAGE:${{ steps.image-tag.outputs.tag }} \ + -t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE \ + ./openalex-importer + + - name: Push to ECR + run: | + aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com + docker tag $CONTAINER_IMAGE:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE:${{ github.sha }} + docker tag $CONTAINER_IMAGE:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE:latest + docker tag $CONTAINER_IMAGE:${{ steps.image-tag.outputs.tag }} $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE:${{ steps.image-tag.outputs.tag }} + docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE:${{ github.sha }} + docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE:latest + docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE:${{ steps.image-tag.outputs.tag }} + + verify-deployment: + name: Verify deployment + needs: build-and-push + if: always() && needs.build-and-push.result == 'success' + runs-on: blacksmith-4vcpu-ubuntu-2204 + timeout-minutes: 10 + steps: + - name: Verify image exists in ECR + run: | + IMAGE_META=$(aws ecr describe-images \ + --repository-name $CONTAINER_IMAGE \ + --image-ids imageTag=${{ github.sha }} \ + --region $AWS_DEFAULT_REGION 2>&1) + if [ $? -eq 0 ]; then + echo "Image $CONTAINER_IMAGE:${{ github.sha }} verified in ECR" + echo "$IMAGE_META" | jq '.imageDetails[0].imageTags' + else + echo "::error::Image not found in ECR: $IMAGE_META" + exit 1 + fi + + - name: Tag image for Flux + run: | + MANIFEST=$(aws ecr batch-get-image \ + --repository-name $CONTAINER_IMAGE \ + --image-ids imageTag=${{ github.sha }} \ + --region $AWS_DEFAULT_REGION \ + --query 'images[0].imageManifest' --output text) + for TAG in latest ${{ needs.build-and-push.outputs.image_tag }}; do + OUTPUT=$(aws ecr put-image \ + --repository-name $CONTAINER_IMAGE \ + --image-tag "$TAG" \ + --image-manifest "$MANIFEST" \ + --region $AWS_DEFAULT_REGION 2>&1) || { + if echo "$OUTPUT" | grep -q "ImageAlreadyExistsException"; then + echo "Tag :$TAG already points to this digest — ok" + else + echo "::error::Failed to tag :$TAG — $OUTPUT" + exit 1 + fi + } + echo "Tagged as :$TAG" + done diff --git a/desci-server/kubernetes/deployment_dev.yaml b/desci-server/kubernetes/deployment_dev.yaml index 3fd574527..b5cbc21a3 100644 --- a/desci-server/kubernetes/deployment_dev.yaml +++ b/desci-server/kubernetes/deployment_dev.yaml @@ -187,43 +187,3 @@ spec: periodSeconds: 1 serviceAccountName: 'vault-auth' ---- -apiVersion: v1 -kind: Service -metadata: - name: desci-server-dev-service - labels: - App: DesciServerDev -spec: - type: LoadBalancer - selector: - App: DesciServerDev - ports: - - protocol: TCP - name: api - port: 80 - targetPort: 5420 ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: desci-server-dev-ingress-new - annotations: - kubernetes.io/ingress.class: 'alb' - alb.ingress.kubernetes.io/scheme: 'internet-facing' - alb.ingress.kubernetes.io/target-type: 'ip' - alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS": 443}]' - alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-2:523044037273:certificate/b6ddf370-5957-4996-bc09-4f48894c0c1f - alb.ingress.kubernetes.io/ssl-redirect: '443' -spec: - rules: - - host: 'nodes-api-dev.desci.com' - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: desci-server-dev-service - port: - number: 80 diff --git a/desci-server/kubernetes/deployment_prod.yaml b/desci-server/kubernetes/deployment_prod.yaml index b45f19701..5155920b3 100755 --- a/desci-server/kubernetes/deployment_prod.yaml +++ b/desci-server/kubernetes/deployment_prod.yaml @@ -186,44 +186,3 @@ spec: failureThreshold: 200 periodSeconds: 1 serviceAccountName: 'vault-auth' ---- -apiVersion: v1 -kind: Service -metadata: - name: desci-server-prod-service - labels: - App: DesciServer -spec: - type: LoadBalancer - selector: - App: DesciServer - ports: - - protocol: TCP - name: api - port: 80 - targetPort: 5420 ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: desci-server-prod-ingress-new - annotations: - kubernetes.io/ingress.class: 'alb' - alb.ingress.kubernetes.io/scheme: 'internet-facing' - alb.ingress.kubernetes.io/target-type: 'ip' - alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS": 443}]' - # alb.ingress.kubernetes.io/ssl-redirect: '443' - # If you have a certificate for SSL termination at ALB - alb.ingress.kubernetes.io/certificate-arn: 'arn:aws:acm:us-east-2:523044037273:certificate/9b192c61-a321-4f65-9309-b2f326b99d05' -spec: - rules: - - host: 'nodes-api.desci.com' - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: desci-server-prod-service - port: - number: 80 diff --git a/desci-server/package.json b/desci-server/package.json index a07193bfc..a9bf57fba 100755 --- a/desci-server/package.json +++ b/desci-server/package.json @@ -77,11 +77,12 @@ "@opentelemetry/api": "^1.8.0", "@opentelemetry/auto-instrumentations-node": "^0.37", "@penseapp/discord-notification": "^2.0.9", - "@prisma/client": "4.10.1", + "@prisma/client": "6.15.0", + "@prisma/instrumentation": "6.15.0", "@react-email/components": "0.0.15", "@sendgrid/mail": "^7.7.0", - "@sentry/node": "8.29.0", - "@sentry/profiling-node": "8.32.0", + "@sentry/node": "8.55.0", + "@sentry/profiling-node": "8.55.0", "@sentry/tracing": "^7.12.0", "@socket.io/redis-adapter": "^8.3.0", "arweave": "^1.10.18", @@ -128,7 +129,7 @@ "pino-pretty": "^10.0.0", "pino-std-serializers": "^7.0.0", "posthog-node": "^5.24.5", - "prisma": "4.10.1", + "prisma": "6.15.0", "qs": "^6.12.0", "react-email": "2.1.0", "redis": "^4.6.7", diff --git a/desci-server/prisma/migrations/20260505113000_add_revenuecat_purchase_fulfillment/migration.sql b/desci-server/prisma/migrations/20260505113000_add_revenuecat_purchase_fulfillment/migration.sql new file mode 100644 index 000000000..6fcc855f5 --- /dev/null +++ b/desci-server/prisma/migrations/20260505113000_add_revenuecat_purchase_fulfillment/migration.sql @@ -0,0 +1,42 @@ +CREATE TYPE "RevenueCatPurchaseFulfillmentType" AS ENUM ('BUNDLE_CHATS', 'LIFETIME_UNLOCK'); + +CREATE TABLE "RevenueCatPurchaseFulfillment" ( + "id" SERIAL NOT NULL, + "userId" INTEGER NOT NULL, + "revenueCatEventId" TEXT, + "revenueCatTransactionId" TEXT NOT NULL, + "revenueCatOriginalTransactionId" TEXT, + "revenueCatProductId" TEXT NOT NULL, + "revenueCatStore" TEXT, + "amountPaid" DOUBLE PRECISION, + "currency" TEXT, + "fulfillmentType" "RevenueCatPurchaseFulfillmentType" NOT NULL, + "purchasedUnits" INTEGER NOT NULL, + "grantedUnits" INTEGER NOT NULL, + "skipReason" TEXT, + "details" JSONB, + "reversedAt" TIMESTAMP(3), + "reversalReason" TEXT, + "reversalDetails" JSONB, + "fulfilledAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "RevenueCatPurchaseFulfillment_pkey" PRIMARY KEY ("id") +); + +CREATE UNIQUE INDEX "RevenueCatPurchaseFulfillment_revenueCatTransactionId_key" +ON "RevenueCatPurchaseFulfillment"("revenueCatTransactionId"); + +CREATE INDEX "RevenueCatPurchaseFulfillment_userId_idx" +ON "RevenueCatPurchaseFulfillment"("userId"); + +CREATE INDEX "RevenueCatPurchaseFulfillment_revenueCatProductId_idx" +ON "RevenueCatPurchaseFulfillment"("revenueCatProductId"); + +CREATE INDEX "RevenueCatPurchaseFulfillment_fulfilledAt_idx" +ON "RevenueCatPurchaseFulfillment"("fulfilledAt"); + +ALTER TABLE "RevenueCatPurchaseFulfillment" +ADD CONSTRAINT "RevenueCatPurchaseFulfillment_userId_fkey" +FOREIGN KEY ("userId") REFERENCES "User"("id") ON DELETE RESTRICT ON UPDATE CASCADE; diff --git a/desci-server/prisma/migrations/20260518120000_add_bundle_auto_replenishment/migration.sql b/desci-server/prisma/migrations/20260518120000_add_bundle_auto_replenishment/migration.sql new file mode 100644 index 000000000..13643061e --- /dev/null +++ b/desci-server/prisma/migrations/20260518120000_add_bundle_auto_replenishment/migration.sql @@ -0,0 +1,22 @@ +CREATE TABLE "BundleAutoReplenishment" ( + "id" SERIAL NOT NULL, + "userId" INTEGER NOT NULL, + "isEnabled" BOOLEAN NOT NULL DEFAULT false, + "threshold" INTEGER NOT NULL DEFAULT 5, + "replenishmentInProgress" BOOLEAN NOT NULL DEFAULT false, + "lastAttemptedAt" TIMESTAMP(3), + "lastSucceededAt" TIMESTAMP(3), + "lastFailedAt" TIMESTAMP(3), + "lastFailureReason" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "BundleAutoReplenishment_pkey" PRIMARY KEY ("id") +); + +CREATE UNIQUE INDEX "BundleAutoReplenishment_userId_key" ON "BundleAutoReplenishment"("userId"); +CREATE INDEX "BundleAutoReplenishment_isEnabled_replenishmentInProgress_idx" ON "BundleAutoReplenishment"("isEnabled", "replenishmentInProgress"); + +ALTER TABLE "BundleAutoReplenishment" +ADD CONSTRAINT "BundleAutoReplenishment_userId_fkey" +FOREIGN KEY ("userId") REFERENCES "User"("id") ON DELETE RESTRICT ON UPDATE CASCADE; diff --git a/desci-server/prisma/schema.prisma b/desci-server/prisma/schema.prisma index e29fed886..ec00b4048 100755 --- a/desci-server/prisma/schema.prisma +++ b/desci-server/prisma/schema.prisma @@ -1,6 +1,5 @@ generator client { - provider = "prisma-client-js" - previewFeatures = ["tracing"] + provider = "prisma-client-js" } datasource db { @@ -284,8 +283,10 @@ model User { Invoice Invoice[] ImportTaskQueue ImportTaskQueue[] AbandonedCheckout AbandonedCheckout[] + BundleAutoReplenishment BundleAutoReplenishment? accountDeletionRequest AccountDeletionRequest? StripeCheckoutFulfillment StripeCheckoutFulfillment[] + RevenueCatPurchaseFulfillment RevenueCatPurchaseFulfillment[] @@index([orcid]) @@index([walletAddress]) @@ -1517,6 +1518,24 @@ model PaymentMethod { @@index([stripePaymentMethodId]) } +model BundleAutoReplenishment { + id Int @id @default(autoincrement()) + userId Int @unique + isEnabled Boolean @default(false) + threshold Int @default(5) + replenishmentInProgress Boolean @default(false) + lastAttemptedAt DateTime? + lastSucceededAt DateTime? + lastFailedAt DateTime? + lastFailureReason String? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + user User @relation(fields: [userId], references: [id]) + + @@index([isEnabled, replenishmentInProgress]) +} + model Invoice { id Int @id @default(autoincrement()) userId Int @@ -1635,6 +1654,35 @@ model StripeCheckoutFulfillment { @@index([fulfilledAt]) } +model RevenueCatPurchaseFulfillment { + id Int @id @default(autoincrement()) + userId Int + revenueCatEventId String? + revenueCatTransactionId String @unique + revenueCatOriginalTransactionId String? + revenueCatProductId String + revenueCatStore String? + amountPaid Float? + currency String? + fulfillmentType RevenueCatPurchaseFulfillmentType + purchasedUnits Int + grantedUnits Int + skipReason String? + details Json? + reversedAt DateTime? + reversalReason String? + reversalDetails Json? + fulfilledAt DateTime @default(now()) + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + user User @relation(fields: [userId], references: [id]) + + @@index([userId]) + @@index([revenueCatProductId]) + @@index([fulfilledAt]) +} + enum ImportTaskQueueStatus { PENDING IN_PROGRESS @@ -1646,6 +1694,11 @@ enum StripeCheckoutFulfillmentType { BUNDLE_CHATS } +enum RevenueCatPurchaseFulfillmentType { + BUNDLE_CHATS + LIFETIME_UNLOCK +} + enum Feature { REFEREE_FINDER RESEARCH_ASSISTANT diff --git a/desci-server/src/config/mobileBundles.ts b/desci-server/src/config/mobileBundles.ts new file mode 100644 index 000000000..b40b77f72 --- /dev/null +++ b/desci-server/src/config/mobileBundles.ts @@ -0,0 +1,24 @@ +export const MOBILE_CHAT_BUNDLE_PRODUCT_IDS = { + CHAT_BUNDLE_10: 'com.desci.research.chatbundle10', + CHAT_BUNDLE_30: 'com.desci.research.chatbundle30', + CHAT_BUNDLE_200: 'com.desci.research.chatbundle200', + LIFETIME: 'com.desci.research.lifetime', +} as const; + +const CHAT_BUNDLE_GRANTS: Record = { + [MOBILE_CHAT_BUNDLE_PRODUCT_IDS.CHAT_BUNDLE_10]: 10, + [MOBILE_CHAT_BUNDLE_PRODUCT_IDS.CHAT_BUNDLE_30]: 30, + [MOBILE_CHAT_BUNDLE_PRODUCT_IDS.CHAT_BUNDLE_200]: 200, +}; + +export function getChatBundleUnitsForProduct(productId: string): number | null { + return CHAT_BUNDLE_GRANTS[productId] ?? null; +} + +export function isLifetimeMobileProduct(productId: string): boolean { + return productId === MOBILE_CHAT_BUNDLE_PRODUCT_IDS.LIFETIME; +} + +export function isRevenueCatBundleOrLifetimeProduct(productId: string): boolean { + return getChatBundleUnitsForProduct(productId) !== null || isLifetimeMobileProduct(productId); +} diff --git a/desci-server/src/controllers/data/retrieve.ts b/desci-server/src/controllers/data/retrieve.ts index 3dcfa5122..aeecd5172 100644 --- a/desci-server/src/controllers/data/retrieve.ts +++ b/desci-server/src/controllers/data/retrieve.ts @@ -3,6 +3,7 @@ import { Request, Response } from 'express'; import { prisma } from '../../client.js'; import { logger as parentLogger } from '../../logger.js'; +import { getFromCache, ONE_DAY_TTL, setToCache } from '../../redisClient.js'; import { getLatestDriveTime } from '../../services/draftTrees.js'; import { FileTreeService } from '../../services/FileTreeService.js'; import { NodeUuid } from '../../services/manifestRepo.js'; @@ -133,6 +134,18 @@ export const pubTree = async (req: Request, res: Response(cacheKey, ONE_DAY_TTL); + if (cached) { + logger.info({ cacheKey }, '[pubTree] cache hit'); + return res.status(200).json(cached); + } + const result = await FileTreeService.getPublishedTree({ manifestCid, rootCid, @@ -163,5 +176,7 @@ export const pubTree = async (req: Request, res: Response) { + try { + const cleanupResult = await prisma.$transaction([ + prisma.draftNodeTree.deleteMany({ where: { nodeId } }), + prisma.dataReference.deleteMany({ where: { nodeId } }), + prisma.guestDataReference.deleteMany({ where: { nodeId } }), + prisma.cidPruneList.deleteMany({ where: { nodeId } }), + prisma.nodeVersion.deleteMany({ where: { nodeId } }), + prisma.node.delete({ where: { id: nodeId } }), + ]); + + logger.info({ nodeId, cleanupResult }, 'Cleaned up draft after Automerge document creation failure'); + } catch (err) { + logger.error({ err, nodeId }, 'Failed to clean up draft after Automerge document creation failure'); + } +} diff --git a/desci-server/src/controllers/nodes/feed.ts b/desci-server/src/controllers/nodes/feed.ts index 6de57f090..b80e07a07 100644 --- a/desci-server/src/controllers/nodes/feed.ts +++ b/desci-server/src/controllers/nodes/feed.ts @@ -1,5 +1,4 @@ -import { NodeFeedItem } from '@prisma/client'; -import { Sql } from '@prisma/client/runtime/data-proxy.js'; +import { NodeFeedItem, Prisma } from '@prisma/client'; import { Request, Response } from 'express'; import { prisma } from '../../client.js'; @@ -7,23 +6,21 @@ import { prisma } from '../../client.js'; export const feed = async (req: Request, res: Response) => { // get the latest feed items - const page = parseInt(req.query.page as string) || 1; - const size = parseInt(req.query.size as string) || 10; - const feedItems = await prisma.$queryRaw( - new Sql( - [ - ` - SELECT "NodeFeedItem".*, "NodeFeedItemEndorsement"."desciCommunityId" as "endorsementOrganizationId" - from "NodeFeedItem" - LEFT JOIN "NodeFeedItemEndorsement" ON "NodeFeedItem"."id" = "NodeFeedItemEndorsement"."nodeFeedItemId" - where "NodeFeedItemEndorsement".id is not null - ORDER BY "NodeFeedItem"."createdAt" DESC - LIMIT ${size} OFFSET ${Math.max(0, page - 1) * size} - + const MAX_PAGE_SIZE = 100; + const rawPage = Number(req.query.page); + const rawSize = Number(req.query.size); + const page = Number.isInteger(rawPage) && rawPage > 0 ? rawPage : 1; + const size = Number.isInteger(rawSize) && rawSize > 0 ? Math.min(rawSize, MAX_PAGE_SIZE) : 10; + const offset = (page - 1) * size; + const feedItems = await prisma.$queryRaw( + Prisma.sql` + SELECT "NodeFeedItem".*, "NodeFeedItemEndorsement"."desciCommunityId" as "endorsementOrganizationId" + FROM "NodeFeedItem" + LEFT JOIN "NodeFeedItemEndorsement" ON "NodeFeedItem"."id" = "NodeFeedItemEndorsement"."nodeFeedItemId" + WHERE "NodeFeedItemEndorsement".id is not null + ORDER BY "NodeFeedItem"."createdAt" DESC + LIMIT ${size} OFFSET ${offset} `, - ], - [], - ), ); res.send({ feedItem: feedItems }); diff --git a/desci-server/src/controllers/nodes/publish.ts b/desci-server/src/controllers/nodes/publish.ts index 3c00d0fb0..34abca8f6 100644 --- a/desci-server/src/controllers/nodes/publish.ts +++ b/desci-server/src/controllers/nodes/publish.ts @@ -137,6 +137,11 @@ export const publish = async (req: PublishRequest, res: Response // Make sure we don't serve stale manifest state when a publish is happening delFromCache(`node-draft-${ensureUuidEndsWithDot(node.uuid)}`); + // Invalidate versions cache so new version shows up immediately + delFromCache(`indexed-versions-${ensureUuidEndsWithDot(node.uuid)}`); + // Invalidate "latest" manifest resolution. Per-index and per-CID keys are + // immutable (each version is content-addressed) so they don't need it. + delFromCache(`resolve-manifest:${ensureUuidEndsWithDot(node.uuid)}:latest`); // Invalidate dpid metadata cache so link previews reflect the new version if (dpidAlias) { diff --git a/desci-server/src/controllers/raw/resolve.ts b/desci-server/src/controllers/raw/resolve.ts index fdd0a6b11..7461c58df 100644 --- a/desci-server/src/controllers/raw/resolve.ts +++ b/desci-server/src/controllers/raw/resolve.ts @@ -6,12 +6,30 @@ import { } from '@desci-labs/desci-models'; import axios from 'axios'; import { NextFunction, Request, Response } from 'express'; +import { pipeline } from 'node:stream/promises'; import { logger as parentLogger } from '../../logger.js'; +import { getFromCache, ONE_DAY_TTL, setToCache } from '../../redisClient.js'; import { getIndexedResearchObjects } from '../../theGraph.js'; -import { decodeBase64UrlSafeToHex, hexToCid } from '../../utils.js'; +import { decodeBase64UrlSafeToHex, ensureUuidEndsWithDot, hexToCid } from '../../utils.js'; const IPFS_RESOLVER_OVERRIDE = process.env.IPFS_RESOLVER_OVERRIDE || ''; +/** + * Cache key for manifest resolution. Used by both the read (cache lookup at + * controller entry) and write (after successful IPFS fetch) paths, plus + * invalidation in publish.ts on `latest`. + * + * firstParam="" → resolve-manifest:.:latest (invalidate on publish) + * firstParam="0".."N" → resolve-manifest:.: (immutable per index) + * firstParam= → resolve-manifest:.: (immutable per CID) + * + * The uuid is normalized via ensureUuidEndsWithDot so the key shape matches + * regardless of whether the URL included a trailing dot. publish.ts uses the + * same normalization for invalidation. + */ +const buildResolveManifestCacheKey = (uuid: string, firstParam: string | undefined) => + `resolve-manifest:${ensureUuidEndsWithDot(uuid)}:${firstParam && firstParam.trim().length ? firstParam : 'latest'}`; + export const resolve = async (req: Request, res: Response, next: NextFunction) => { /** * DCITE resolution scheme @@ -44,6 +62,22 @@ export const resolve = async (req: Request, res: Response, next: NextFunction) = }); logger.debug(`[resolve::resolve] firstParam=${firstParam} secondParam=${secondParam}`); + // Cache the manifest-by-uuid+version path. Heavy work below: + // getIndexedResearchObjects (theGraph, 5-8s) + IPFS gateway fetch (1-2s). + // Only cache when no secondParam — i.e. the response is the bare manifest, + // fully deterministic by uuid+firstParam. Component/code/PDF responses + // and 4xx/5xx outcomes are not cached. + const isCacheableManifestRequest = !secondParam; + const manifestCacheKey = isCacheableManifestRequest ? buildResolveManifestCacheKey(uuid, firstParam) : null; + + if (manifestCacheKey) { + const cached = await getFromCache(manifestCacheKey); + if (cached) { + logger.info({ manifestCacheKey }, '[resolve] cache hit'); + return res.send(cached); + } + } + let result; try { const res = await getIndexedResearchObjects([uuid]); @@ -73,6 +107,7 @@ export const resolve = async (req: Request, res: Response, next: NextFunction) = try { logger.info(`Calling IPFS Resolver ${ipfsResolver} for CID ${cidString}`); const { data } = await axios.get(`${ipfsResolver}/${cidString}`); + if (manifestCacheKey) await setToCache(manifestCacheKey, data, ONE_DAY_TTL); return res.send(data); } catch (err) { return res.status(500).send({ ok: false, msg: 'ipfs uplink failed, try setting ?g= querystring to resolver' }); @@ -116,13 +151,24 @@ export const resolve = async (req: Request, res: Response, next: NextFunction) = cidString = hexToCid(version.cid); }; - const { data } = await axios.get( - `${ipfsResolver}/${cidString}`, - { headers: { 'Bypass-Tunnel-Reminder': true } } - ); + let data; + try { + const response = await axios.get( + `${ipfsResolver}/${cidString}`, + { headers: { 'Bypass-Tunnel-Reminder': true } } + ); + data = response.data; + } catch (err) { + logger.warn({ err, ipfsResolver, cidString }, 'ipfs uplink failed'); + return res.status(502).send({ + ok: false, + msg: 'ipfs uplink failed, try setting ?g= querystring to resolver', + }); + } if (!secondParam) { logger.info("Returning manifest as there is no additional path"); + if (manifestCacheKey) await setToCache(manifestCacheKey, data, ONE_DAY_TTL); return res.send(data); }; @@ -160,13 +206,27 @@ export const resolve = async (req: Request, res: Response, next: NextFunction) = if (thirdParam == '!') { logger.debug('recognize zip'); //send the zip - return axios.get( - `${ipfsResolver}/${codeComponent.payload.url}`, - { responseType: 'stream' } - ).then((response) => { - // The response will give you the zip file - response.data.pipe(res); - }); + try { + const response = await axios.get( + `${ipfsResolver}/${codeComponent.payload.url}`, + { responseType: 'stream' } + ); + // Use stream.pipeline so errors from either side (source aborts, + // client disconnects mid-stream) are forwarded and both streams + // are torn down properly. A bare .pipe() leaks on error. + await pipeline(response.data, res); + return res; + } catch (err) { + logger.warn({ err, ipfsResolver, url: codeComponent.payload.url }, 'ipfs zip stream failed'); + // If headers are already sent (mid-stream failure) we can't send + // a JSON error — just destroy the response so the client sees an + // aborted connection instead of hanging. + if (res.headersSent) { + res.destroy(err instanceof Error ? err : new Error(String(err))); + return res; + } + return res.status(502).send({ ok: false, msg: 'ipfs uplink failed' }); + } }; // send the individual file diff --git a/desci-server/src/controllers/raw/versions.ts b/desci-server/src/controllers/raw/versions.ts index cd63bc00b..fd0e5e912 100644 --- a/desci-server/src/controllers/raw/versions.ts +++ b/desci-server/src/controllers/raw/versions.ts @@ -1,7 +1,8 @@ import { Request, Response, NextFunction } from 'express'; import { logger as parentLogger } from '../../logger.js'; -import { getIndexedResearchObjects, IndexedResearchObject } from '../../theGraph.js'; +import { getIndexedResearchObjects } from '../../theGraph.js'; +import { getOrCache, ONE_DAY_TTL } from '../../redisClient.js'; import { ensureUuidEndsWithDot } from '../../utils.js'; const logger = parentLogger.child({ @@ -9,23 +10,33 @@ const logger = parentLogger.child({ }); /** - * Get all versions of research object from index (publicView) + * Get all versions of research object from index (publicView). + * Cached in Redis for 1 day — the dpid.org /api/v2/query/history + * endpoint takes 5-8 seconds uncached. */ export const versions = async (req: Request, res: Response, next: NextFunction) => { const uuid = ensureUuidEndsWithDot(req.params.uuid); - let result: IndexedResearchObject; + const cacheKey = `indexed-versions-${uuid}`; try { - const { researchObjects } = await getIndexedResearchObjects([uuid]); - result = researchObjects[0]; + const result = await getOrCache( + cacheKey, + async () => { + const { researchObjects } = await getIndexedResearchObjects([uuid]); + return researchObjects[0] ?? null; + }, + ONE_DAY_TTL, + ); + + if (!result) { + logger.warn({ uuid }, 'could not find indexed versions'); + res.status(404).send({ ok: false, msg: `could not locate uuid ${uuid}` }); + return; + } + + res.send(result); } catch (err) { - logger.error({ result, err }, `[ERROR] graph lookup fail ${err.message}`); + logger.error({ uuid, err }, `[ERROR] versions lookup fail ${err.message}`); + res.status(500).send({ ok: false, msg: 'Failed to fetch versions' }); } - if (!result) { - logger.warn({ uuid, result }, 'could not find indexed versions'); - res.status(404).send({ ok: false, msg: `could not locate uuid ${uuid}` }); - return; - } - - res.send(result); }; diff --git a/desci-server/src/controllers/stripe/subscription.ts b/desci-server/src/controllers/stripe/subscription.ts index 29ad3cd48..e4f39a84d 100644 --- a/desci-server/src/controllers/stripe/subscription.ts +++ b/desci-server/src/controllers/stripe/subscription.ts @@ -35,6 +35,11 @@ const getUserStripePurchasesSchema = z.object({ }), }); +const bundleAutoReplenishmentSchema = z.object({ + enabled: z.boolean(), + threshold: z.coerce.number().int().min(1).max(50).optional(), +}); + const formatZodIssues = (error: ZodError) => error.issues.map((issue) => ({ path: issue.path.join('.'), @@ -133,6 +138,12 @@ export const createSubscriptionCheckout = async (req: RequestWithUser, res: Resp }, }; + if (isBundleCheckout) { + sessionConfig.payment_intent_data = { + setup_future_usage: 'off_session', + }; + } + // Add promotion codes or specific coupon support if (allowPromotionCodes) { sessionConfig.allow_promotion_codes = true; @@ -262,20 +273,19 @@ export const createCustomerPortal = async (req: RequestWithUser, res: Response): logger.info({ userId }, 'Creating customer portal session'); - // Get user's Stripe customer ID from any subscription (not just active ones) - const subscription = await SubscriptionService.getUserSubscriptionWithDetails(userId); - if (!subscription?.stripeCustomerId) { + const stripeCustomerId = await SubscriptionService.getStripeCustomerId(userId); + if (!stripeCustomerId) { logger.warn({ userId }, 'No customer found for user'); return res.status(404).json({ error: 'No customer found' }); } - logger.info({ customerId: subscription.stripeCustomerId }, 'Found customer for portal'); + logger.info({ customerId: stripeCustomerId }, 'Found customer for portal'); // Create portal session try { const stripe = getStripe(); const portalSession = await stripe.billingPortal.sessions.create({ - customer: subscription.stripeCustomerId, + customer: stripeCustomerId, return_url: returnUrl || `${req.headers.origin}/settings/subscription`, }); @@ -287,7 +297,7 @@ export const createCustomerPortal = async (req: RequestWithUser, res: Response): err: stripeError, type: stripeError.type, code: stripeError.code, - customerId: subscription.stripeCustomerId, + customerId: stripeCustomerId, }, 'Stripe portal creation failed', ); @@ -392,6 +402,81 @@ export const getUserStripePurchases = async (req: RequestWithUser, res: Response } }; +export const getBundleAutoReplenishment = async (req: RequestWithUser, res: Response): Promise => { + try { + const userId = req.user?.id; + if (!userId) { + return res.status(401).json({ error: 'Unauthorized' }); + } + + const status = await SubscriptionService.getBundleAutoReplenishmentStatus(userId); + + return res.status(200).json({ + enabled: status.enabled, + threshold: status.threshold, + replenishmentInProgress: status.replenishmentInProgress, + lastAttemptedAt: status.lastAttemptedAt, + lastSucceededAt: status.lastSucceededAt, + lastFailedAt: status.lastFailedAt, + lastFailureReason: status.lastFailureReason, + latestBundlePurchase: status.latestBundlePurchase + ? { + purchasedUnits: status.latestBundlePurchase.purchasedUnits, + fulfilledAt: status.latestBundlePurchase.fulfilledAt, + } + : null, + hasSavedPaymentMethod: status.hasSavedPaymentMethod, + paymentMethodSummary: status.paymentMethodSummary, + }); + } catch (error: any) { + logger.error({ err: error, userId: req.user?.id }, 'Failed to get bundle auto-replenishment status'); + return res.status(500).json({ error: 'Failed to get bundle auto-replenishment status' }); + } +}; + +export const updateBundleAutoReplenishment = async (req: RequestWithUser, res: Response): Promise => { + try { + const userId = req.user?.id; + if (!userId) { + return res.status(401).json({ error: 'Unauthorized' }); + } + + const { enabled, threshold } = bundleAutoReplenishmentSchema.parse(req.body); + const status = await SubscriptionService.updateBundleAutoReplenishment(userId, { enabled, threshold }); + + return res.status(200).json({ + enabled: status.enabled, + threshold: status.threshold, + replenishmentInProgress: status.replenishmentInProgress, + lastAttemptedAt: status.lastAttemptedAt, + lastSucceededAt: status.lastSucceededAt, + lastFailedAt: status.lastFailedAt, + lastFailureReason: status.lastFailureReason, + latestBundlePurchase: status.latestBundlePurchase + ? { + purchasedUnits: status.latestBundlePurchase.purchasedUnits, + fulfilledAt: status.latestBundlePurchase.fulfilledAt, + } + : null, + hasSavedPaymentMethod: status.hasSavedPaymentMethod, + paymentMethodSummary: status.paymentMethodSummary, + }); + } catch (error: any) { + if (error instanceof ZodError) { + return res.status(400).json({ + error: 'Invalid request body', + details: formatZodIssues(error), + }); + } + + logger.error({ err: error, userId: req.user?.id }, 'Failed to update bundle auto-replenishment'); + + return res.status(400).json({ + error: error instanceof Error ? error.message : 'Failed to update bundle auto-replenishment', + }); + } +}; + export const updateSubscription = async (req: RequestWithUser, res: Response): Promise => { try { const userId = req.user?.id; diff --git a/desci-server/src/controllers/stripe/webhook.ts b/desci-server/src/controllers/stripe/webhook.ts index 8cf54975b..f30c0a47a 100644 --- a/desci-server/src/controllers/stripe/webhook.ts +++ b/desci-server/src/controllers/stripe/webhook.ts @@ -98,6 +98,9 @@ export const handleStripeWebhook = async (req: Request, res: Response): Promise< case 'payment_intent.succeeded': await handlePaymentIntentSucceeded(event.data.object as Stripe.PaymentIntent); break; + case 'payment_intent.payment_failed': + await handlePaymentIntentPaymentFailed(event.data.object as Stripe.PaymentIntent); + break; // Charge events case 'charge.succeeded': @@ -136,10 +139,13 @@ async function handleCustomerCreated(customer: Stripe.Customer) { try { await SubscriptionService.handleCustomerCreated(customer); } catch (err: any) { - logger.error({ - customerId: customer.id, - err, - }, 'Failed to handle customer created'); + logger.error( + { + customerId: customer.id, + err, + }, + 'Failed to handle customer created', + ); throw err; } } @@ -150,10 +156,13 @@ async function handleSubscriptionCreated(subscription: Stripe.Subscription) { try { await SubscriptionService.handleSubscriptionCreated(subscription); } catch (err: any) { - logger.error({ - subscriptionId: subscription.id, - err, - }, 'Failed to handle subscription created'); + logger.error( + { + subscriptionId: subscription.id, + err, + }, + 'Failed to handle subscription created', + ); throw err; } } @@ -164,10 +173,13 @@ async function handleSubscriptionUpdated(subscription: Stripe.Subscription) { try { await SubscriptionService.handleSubscriptionUpdated(subscription); } catch (err: any) { - logger.error({ - subscriptionId: subscription.id, - err, - }, 'Failed to handle subscription updated'); + logger.error( + { + subscriptionId: subscription.id, + err, + }, + 'Failed to handle subscription updated', + ); throw err; } } @@ -178,10 +190,13 @@ async function handleSubscriptionDeleted(subscription: Stripe.Subscription) { try { await SubscriptionService.handleSubscriptionDeleted(subscription); } catch (err: any) { - logger.error({ - subscriptionId: subscription.id, - err, - }, 'Failed to handle subscription deleted'); + logger.error( + { + subscriptionId: subscription.id, + err, + }, + 'Failed to handle subscription deleted', + ); throw err; } } @@ -192,10 +207,13 @@ async function handleInvoiceCreated(invoice: Stripe.Invoice) { try { await SubscriptionService.handleInvoiceCreated(invoice); } catch (err: any) { - logger.error({ - invoiceId: invoice.id, - err, - }, 'Failed to handle invoice created'); + logger.error( + { + invoiceId: invoice.id, + err, + }, + 'Failed to handle invoice created', + ); throw err; } } @@ -206,10 +224,13 @@ async function handleInvoicePaymentSucceeded(invoice: Stripe.Invoice) { try { await SubscriptionService.handleInvoicePaymentSucceeded(invoice); } catch (err: any) { - logger.error({ - invoiceId: invoice.id, - err, - }, 'Failed to handle invoice payment succeeded'); + logger.error( + { + invoiceId: invoice.id, + err, + }, + 'Failed to handle invoice payment succeeded', + ); throw err; } } @@ -220,10 +241,13 @@ async function handleInvoicePaymentFailed(invoice: Stripe.Invoice) { try { await SubscriptionService.handleInvoicePaymentFailed(invoice); } catch (err: any) { - logger.error({ - invoiceId: invoice.id, - err, - }, 'Failed to handle invoice payment failed'); + logger.error( + { + invoiceId: invoice.id, + err, + }, + 'Failed to handle invoice payment failed', + ); throw err; } } @@ -234,10 +258,13 @@ async function handlePaymentMethodAttached(paymentMethod: Stripe.PaymentMethod) try { await SubscriptionService.handlePaymentMethodAttached(paymentMethod); } catch (err: any) { - logger.error({ - paymentMethodId: paymentMethod.id, - err, - }, 'Failed to handle payment method attached'); + logger.error( + { + paymentMethodId: paymentMethod.id, + err, + }, + 'Failed to handle payment method attached', + ); throw err; } } @@ -248,10 +275,13 @@ async function handlePaymentMethodDetached(paymentMethod: Stripe.PaymentMethod) try { await SubscriptionService.handlePaymentMethodDetached(paymentMethod); } catch (err: any) { - logger.error({ - paymentMethodId: paymentMethod.id, - err, - }, 'Failed to handle payment method detached'); + logger.error( + { + paymentMethodId: paymentMethod.id, + err, + }, + 'Failed to handle payment method detached', + ); throw err; } } @@ -262,10 +292,13 @@ async function handleTrialWillEnd(subscription: Stripe.Subscription) { try { await SubscriptionService.handleTrialWillEnd(subscription); } catch (err: any) { - logger.error({ - subscriptionId: subscription.id, - err, - }, 'Failed to handle trial will end'); + logger.error( + { + subscriptionId: subscription.id, + err, + }, + 'Failed to handle trial will end', + ); throw err; } } @@ -276,8 +309,13 @@ async function handlePaymentIntentCreated(paymentIntent: Stripe.PaymentIntent) { } async function handlePaymentIntentSucceeded(paymentIntent: Stripe.PaymentIntent) { - logger.info({ paymentIntentId: paymentIntent.customer }, 'stripe::handlePaymentIntentSucceeded'); - // Payment success is handled via invoice.payment_succeeded, no action needed + logger.info({ paymentIntentId: paymentIntent.id }, 'stripe::handlePaymentIntentSucceeded'); + const handledAutoReplenishment = await SubscriptionService.handleBundleAutoReplenishmentSucceeded(paymentIntent); + if (handledAutoReplenishment) { + return; + } + + // Payment success is handled via invoice.payment_succeeded for recurring subscriptions. const stripe = getStripe(); const subscriptions = await stripe.subscriptions.list({ customer: paymentIntent.customer as string, @@ -293,6 +331,11 @@ async function handlePaymentIntentSucceeded(paymentIntent: Stripe.PaymentIntent) await SubscriptionService.handleSubscriptionCreated(activeSubscription); } +async function handlePaymentIntentPaymentFailed(paymentIntent: Stripe.PaymentIntent) { + logger.info({ paymentIntentId: paymentIntent.id }, 'stripe::handlePaymentIntentPaymentFailed'); + await SubscriptionService.handleBundleAutoReplenishmentFailed(paymentIntent); +} + async function handleChargeSucceeded(charge: Stripe.Charge) { logger.info({ chargeId: charge.id }, 'Processing charge succeeded'); // Charges are handled via invoice events, no action needed diff --git a/desci-server/src/index.ts b/desci-server/src/index.ts index b38f174cf..ded34e58f 100755 --- a/desci-server/src/index.ts +++ b/desci-server/src/index.ts @@ -13,9 +13,16 @@ const logger = parentLogger.child({ // Create the server instance for production const server = createServer(); -server.ready().then((_) => { - console.log('server is ready'); -}); +server.ready() + .then((_) => { + console.log('server is ready'); + }) + .catch((err) => { + // Now that unhandledRejection is non-fatal, a startup failure here would + // otherwise be silently logged and the process would limp along + // half-initialized. Treat startup failure as fatal explicitly. + void handleFatalError(err, 'server.ready'); + }); export const app = server.app; /** @@ -84,8 +91,8 @@ process.on('SIGUSR2', () => gracefulShutdown('SIGUSR2')); */ const FATAL_TIMEOUT_MS = 500; -/** Handler to use for uncaught exceptions and promise rejections, from which we - * want to extract as much log info as possible to the log aggregator. +/** Handler for uncaughtException — process state is unknown after one of + * these, so we log and then exit. This is the standard Node.js recommendation. */ const handleFatalError = async (error: unknown, type: string) => { // Mostly error should be an Error, but in theory anything can be thrown @@ -108,8 +115,26 @@ const handleFatalError = async (error: unknown, type: string) => { } }; +/** Handler for unhandledRejection — log it, but DO NOT crash the process. + * + * Historically this called handleFatalError, which exited. That turned every + * un-try/catched upstream HTTP failure (e.g. a transient 503 from + * ipfs.desci.com) into a full pod crash, taking down the whole desci-server + * fleet in lockstep with any upstream hiccup. A long-running web server should + * survive a single bad request — the offending handler should be fixed to + * catch its own errors, but the process must keep serving traffic in the + * meantime. Sentry still picks these up via its global integration. + */ +const handleUnhandledRejection = (reason: unknown) => { + const normalizedError = reason instanceof Error ? reason : new Error(String(reason)); + logger.error( + { err: errWithCause(normalizedError), type: 'unhandledRejection' }, + 'Unhandled promise rejection (process continuing)', + ); +}; + process.on('uncaughtException', (err) => handleFatalError(err, 'uncaughtException')); -process.on('unhandledRejection', (reason) => handleFatalError(reason, 'unhandledRejection')); +process.on('unhandledRejection', handleUnhandledRejection); process.on('exit', () => { if (!isShuttingDown) { diff --git a/desci-server/src/routes/v1/stripe.ts b/desci-server/src/routes/v1/stripe.ts index ea8483abd..bba84058d 100644 --- a/desci-server/src/routes/v1/stripe.ts +++ b/desci-server/src/routes/v1/stripe.ts @@ -11,6 +11,8 @@ import { getPricingOptions, createPaymentIntent, resetStripeTestStateForCurrentUser, + getBundleAutoReplenishment, + updateBundleAutoReplenishment, } from '../../controllers/stripe/subscription.js'; import { handleStripeWebhook } from '../../controllers/stripe/webhook.js'; import { ensureAdmin } from '../../middleware/ensureAdmin.js'; @@ -29,6 +31,8 @@ router.post('/subscription/checkout', [requireStripe, ensureUser], asyncHandler( router.post('/subscription/portal', [requireStripe, ensureUser], asyncHandler(createCustomerPortal)); router.get('/subscription', [requireStripe, ensureUser], asyncHandler(getUserSubscription)); router.get('/purchases', [requireStripe, ensureUser], asyncHandler(getUserStripePurchases)); +router.get('/bundle-auto-replenishment', [requireStripe, ensureUser], asyncHandler(getBundleAutoReplenishment)); +router.post('/bundle-auto-replenishment', [requireStripe, ensureUser], asyncHandler(updateBundleAutoReplenishment)); router.put('/subscription', [requireStripe, ensureUser], asyncHandler(updateSubscription)); router.delete('/subscription', [requireStripe, ensureUser], asyncHandler(cancelSubscription)); router.post('/test/reset-current-user', [requireStripe, ensureUser, ensureAdmin], asyncHandler(resetStripeTestStateForCurrentUser)); diff --git a/desci-server/src/scripts/warm-versions-cache.ts b/desci-server/src/scripts/warm-versions-cache.ts new file mode 100644 index 000000000..ca8b1df60 --- /dev/null +++ b/desci-server/src/scripts/warm-versions-cache.ts @@ -0,0 +1,72 @@ +/** + * Pre-warm the Redis cache for /v1/pub/versions responses. + * + * Queries all published nodes from the DB, then calls getIndexedResearchObjects + * for each one, which populates the `indexed-versions-{uuid}` cache key. + * + * Usage: + * npx ts-node src/scripts/warm-versions-cache.ts + * # or with concurrency limit: + * CONCURRENCY=5 npx ts-node src/scripts/warm-versions-cache.ts + */ +import { prisma } from '../client.js'; +import { getIndexedResearchObjects } from '../theGraph.js'; +import { getOrCache, ONE_DAY_TTL } from '../redisClient.js'; +import { ensureUuidEndsWithDot } from '../utils.js'; + +const CONCURRENCY = parseInt(process.env.CONCURRENCY || '3', 10); + +async function main() { + console.log('Fetching all published nodes with dpidAlias...'); + + const nodes = await prisma.node.findMany({ + select: { uuid: true, dpidAlias: true }, + where: { + dpidAlias: { not: null }, + isDeleted: false, + }, + orderBy: { dpidAlias: 'desc' }, + }); + + console.log(`Found ${nodes.length} published nodes. Warming cache with concurrency=${CONCURRENCY}...`); + + let warmed = 0; + let failed = 0; + + for (let i = 0; i < nodes.length; i += CONCURRENCY) { + const batch = nodes.slice(i, i + CONCURRENCY); + + await Promise.allSettled( + batch.map(async (node) => { + const uuid = ensureUuidEndsWithDot(node.uuid); + const cacheKey = `indexed-versions-${uuid}`; + + try { + await getOrCache( + cacheKey, + async () => { + const { researchObjects } = await getIndexedResearchObjects([uuid]); + return researchObjects[0] ?? null; + }, + ONE_DAY_TTL, + ); + warmed++; + if (warmed % 10 === 0) { + console.log(` Progress: ${warmed}/${nodes.length} warmed, ${failed} failed`); + } + } catch (e) { + failed++; + console.error(` Failed dpid=${node.dpidAlias} uuid=${uuid}: ${(e as Error).message}`); + } + }), + ); + } + + console.log(`\nDone. Warmed: ${warmed}, Failed: ${failed}, Total: ${nodes.length}`); + process.exit(0); +} + +main().catch((e) => { + console.error('Fatal error:', e); + process.exit(1); +}); diff --git a/desci-server/src/server.ts b/desci-server/src/server.ts index cc3c6ba16..05608301e 100644 --- a/desci-server/src/server.ts +++ b/desci-server/src/server.ts @@ -3,9 +3,10 @@ import 'dotenv/config'; import 'reflect-metadata'; import * as child from 'child_process'; import type { Server as HttpServer } from 'http'; +import { createRequire } from 'module'; +import { PrismaInstrumentation } from '@prisma/instrumentation'; import * as Sentry from '@sentry/node'; -import { nodeProfilingIntegration } from '@sentry/profiling-node'; import bodyParser from 'body-parser'; import cookieParser from 'cookie-parser'; import express from 'express'; @@ -37,6 +38,7 @@ import { SubmissionQueueJob } from './workers/doiSubmissionQueue.js'; const logger = parentLogger.child({ module: 'server.ts', }); +const require = createRequire(import.meta.url); const ENABLE_TELEMETRY = process.env.NODE_ENV === 'production'; const IS_DEV = !ENABLE_TELEMETRY; @@ -264,7 +266,10 @@ export class AppServer { status: res.statusCode, }; } else { - return res; + return { + statusCode: res.statusCode, + responseTime: res.responseTime, + }; } }, req: (req) => { @@ -276,7 +281,13 @@ export class AppServer { url: req.url, }; } else { - return req; + const urlWithoutQuery = (req.originalUrl || req.url || '').split('?')[0]; + return { + id: req.id, + method: req.method, + url: urlWithoutQuery, + route: req.route?.path, + }; } }, }, @@ -287,11 +298,25 @@ export class AppServer { #initTelemetry() { if (ENABLE_TELEMETRY) { logger.info('[DeSci Nodes] Telemetry enabled'); + const sentryIntegrations = [ + Sentry.prismaIntegration({ + prismaInstrumentation: new PrismaInstrumentation(), + }), + ]; + + // Profiling uses a native binding that is not always present in test/local environments. + try { + const { nodeProfilingIntegration } = require('@sentry/profiling-node'); + sentryIntegrations.push(nodeProfilingIntegration()); + } catch (error) { + logger.warn({ error }, 'Sentry profiling unavailable; continuing without profiling integration'); + } + Sentry.init({ dsn: 'https://d508a5c408f34b919ccd94aac093e076@o1330109.ingest.sentry.io/6619754', environment: SERVER_ENV, release: 'desci-nodes-server@' + process.env.npm_package_version, - integrations: [Sentry.prismaIntegration(), nodeProfilingIntegration()], + integrations: sentryIntegrations, tracesSampleRate: 0.1, profilesSampleRate: 0.1, }); diff --git a/desci-server/src/services/Attestation.ts b/desci-server/src/services/Attestation.ts index bb508fc7c..eed3454ea 100644 --- a/desci-server/src/services/Attestation.ts +++ b/desci-server/src/services/Attestation.ts @@ -1072,9 +1072,8 @@ export class AttestationService { return queryResult; } - async getRecommendedAttestations(filter?: Prisma.CommunityEntryAttestationFindManyArgs) { + async getRecommendedAttestations(filter?: { where?: Prisma.CommunityEntryAttestationWhereInput }) { const attestations = await prisma.communityEntryAttestation.findMany({ - ...filter, include: { attestation: { select: { community: true } }, attestationVersion: { diff --git a/desci-server/src/services/Communities.ts b/desci-server/src/services/Communities.ts index 893751693..2718e0715 100644 --- a/desci-server/src/services/Communities.ts +++ b/desci-server/src/services/Communities.ts @@ -1092,7 +1092,9 @@ export class CommunityService { manifestUrl: true, manifestDocumentId: true, versions: { - where: { OR: { transactionId: { not: null }, commitId: { not: null } } }, + where: { + OR: [{ transactionId: { not: null } }, { commitId: { not: null } }], + }, orderBy: { createdAt: 'desc' }, select: { transactionId: true, diff --git a/desci-server/src/services/FeatureLimits/FeatureLimitsService.ts b/desci-server/src/services/FeatureLimits/FeatureLimitsService.ts index 1bd3a4cac..685654c35 100644 --- a/desci-server/src/services/FeatureLimits/FeatureLimitsService.ts +++ b/desci-server/src/services/FeatureLimits/FeatureLimitsService.ts @@ -247,8 +247,15 @@ async function getOrCreateUserFeatureLimit(userId: number, feature: Feature): Pr * Maintains the original billing cycle (e.g., if billing started on 5th, keep it on 5th) */ async function checkAndResetPeriodIfNeeded(limit: UserFeatureLimit): Promise { - // if feature is research assistant, there'll be no period reset, so return the limit as is - if (limit.feature === Feature.RESEARCH_ASSISTANT) { + // Legacy weekly research assistant limits intentionally behave like a + // long-lived allowance with daily drip and do not roll over. Newer monthly + // / yearly limited access (for bundles or migrated rows) should still use + // active windows so old usage does not accumulate forever. Unlimited access + // also ignores rollover because the period is irrelevant. + if ( + limit.feature === Feature.RESEARCH_ASSISTANT && + (limit.useLimit === null || limit.period === Period.WEEK) + ) { return limit; } diff --git a/desci-server/src/services/FeatureLimits/FeatureUsageService.ts b/desci-server/src/services/FeatureLimits/FeatureUsageService.ts index 4d903dc3b..0d505c125 100644 --- a/desci-server/src/services/FeatureLimits/FeatureUsageService.ts +++ b/desci-server/src/services/FeatureLimits/FeatureUsageService.ts @@ -6,9 +6,9 @@ import { logger as parentLogger } from '../../logger.js'; import { sendEmail } from '../email/email.js'; import { SciweaveEmailTypes } from '../email/sciweaveEmailTypes.js'; // import { isUserStudentSciweave } from '../interactionLog.js'; // unused after coupon campaign disable +import { getUserNameByUser } from '../user.js'; import { FeatureLimitsService } from './FeatureLimitsService.js'; -import { getUserNameByUser } from '../user.js'; const logger = parentLogger.child({ module: 'FeatureUsageService' }); @@ -68,6 +68,8 @@ async function consumeUsage(request: ConsumeUsageRequest): Promise + SubscriptionService.triggerBundleAutoReplenishmentIfNeeded({ + userId, + remainingUses: result.remainingUsesAfterConsume, + }), + ) + .catch((triggerError) => { + logger.error({ triggerError, userId }, 'Failed to trigger bundle auto-replenishment'); + }); + // Send out-of-chats email if user just hit their limit (do this after successful transaction) if (result.shouldSendLimitEmail) { try { diff --git a/desci-server/src/services/PublishServices.ts b/desci-server/src/services/PublishServices.ts index cc8ccbec7..1dd666106 100644 --- a/desci-server/src/services/PublishServices.ts +++ b/desci-server/src/services/PublishServices.ts @@ -490,7 +490,11 @@ async function updatePublishStatusEntry({ data: Prisma.PublishStatusUncheckedUpdateInput; }) { try { - const identifier = publishStatusId ? { id: publishStatusId } : nodeUuid && version ? { nodeUuid, version } : null; + const identifier = publishStatusId + ? { id: publishStatusId } + : nodeUuid && version + ? { nodeUuid_version: { nodeUuid: ensureUuidEndsWithDot(nodeUuid), version } } + : null; if (!identifier) { throw 'No identifier provided'; } diff --git a/desci-server/src/services/RevenueCatService.ts b/desci-server/src/services/RevenueCatService.ts index 8a82da35e..cd6cbb019 100644 --- a/desci-server/src/services/RevenueCatService.ts +++ b/desci-server/src/services/RevenueCatService.ts @@ -3,6 +3,7 @@ */ import crypto from 'node:crypto'; +import { getChatBundleUnitsForProduct, isLifetimeMobileProduct } from '../config/mobileBundles.js'; import { REVENUECAT_API_KEY, REVENUECAT_ENTITLEMENT_ID, REVENUECAT_WEBHOOK_SECRET } from '../config.js'; import { logger as parentLogger } from '../logger.js'; import { delFromCache, getFromCache, setToCache } from '../redisClient.js'; @@ -74,8 +75,14 @@ export interface RevenueCatWebhookPayload { api_version: string; event: { app_user_id: string; + cancel_reason?: string | null; + currency?: string | null; event_timestamp_ms: number; expiration_at_ms: number; + id?: string; + original_transaction_id?: string | null; + presented_offering_id?: string | null; + price?: number | null; product_id: string; store: string; subscriber_attributes: { @@ -86,6 +93,19 @@ export interface RevenueCatWebhookPayload { }; } +function getWebhookUserId(payload: RevenueCatWebhookPayload): number | null { + const attributeUserId = payload.event.subscriber_attributes['userId']?.value; + if (attributeUserId) { + const parsed = Number(attributeUserId); + if (!Number.isNaN(parsed)) { + return parsed; + } + } + + const appUserId = Number(payload.event.app_user_id); + return Number.isNaN(appUserId) ? null : appUserId; +} + function getAuthHeaders(): Record { if (!REVENUECAT_API_KEY?.trim()) { throw new Error('REVENUECAT_API_KEY is not set'); @@ -264,14 +284,156 @@ async function cacheMobileSubscriptionFromCustomerInfo( export async function handleWebhookEvent( payload: RevenueCatWebhookPayload, ): Promise<{ ok: true } | { ok: false; status: number; error: string }> { - const userIdValue = payload.event.subscriber_attributes['userId']?.value; - const userId = userIdValue ? Number(userIdValue) : null; - const user = userId != null && !Number.isNaN(userId) ? await getUserById(userId) : null; + const userId = getWebhookUserId(payload); + const user = userId != null ? await getUserById(userId) : null; if (!user) { - logger.warn({ userId: userIdValue }, 'RevenueCat webhook: user not found'); + logger.warn({ appUserId: payload.event.app_user_id }, 'RevenueCat webhook: user not found'); return { ok: false, status: 404, error: 'User not found' }; } const appUserId = payload.event.app_user_id; + const productId = payload.event.product_id; + const bundleUnits = getChatBundleUnitsForProduct(productId); + const isLifetimeProduct = isLifetimeMobileProduct(productId); + + if (bundleUnits !== null || isLifetimeProduct) { + switch (payload.event.type) { + case 'NON_RENEWING_PURCHASE': + case 'INITIAL_PURCHASE': + if (!payload.event.transaction_id) { + logger.warn( + { userId, productId, eventType: payload.event.type }, + 'RevenueCat purchase missing transaction id', + ); + break; + } + + if (bundleUnits !== null) { + const fulfillment = await SubscriptionService.handleRevenueCatBundlePurchase({ + userId, + productId, + transactionId: payload.event.transaction_id, + originalTransactionId: payload.event.original_transaction_id, + eventId: payload.event.id, + store: payload.event.store, + amountPaid: payload.event.price, + currency: payload.event.currency, + presentedOfferingId: payload.event.presented_offering_id, + }); + + logger.info( + { + userId, + productId, + transactionId: payload.event.transaction_id, + grantedChats: fulfillment.grantedChats, + totalChats: fulfillment.totalChats, + skipReason: fulfillment.skipReason, + alreadyFulfilled: fulfillment.alreadyFulfilled, + }, + 'RevenueCat bundle purchase fulfilled', + ); + } else { + const fulfillment = await SubscriptionService.handleRevenueCatLifetimePurchase({ + userId, + productId, + transactionId: payload.event.transaction_id, + originalTransactionId: payload.event.original_transaction_id, + eventId: payload.event.id, + store: payload.event.store, + amountPaid: payload.event.price, + currency: payload.event.currency, + presentedOfferingId: payload.event.presented_offering_id, + }); + + logger.info( + { + userId, + productId, + transactionId: payload.event.transaction_id, + createdNewLifetime: fulfillment.createdNewLifetime, + alreadyFulfilled: fulfillment.alreadyFulfilled, + }, + 'RevenueCat lifetime purchase fulfilled', + ); + } + break; + case 'CANCELLATION': + if (!payload.event.transaction_id) { + logger.warn( + { userId, productId, eventType: payload.event.type }, + 'RevenueCat cancellation missing transaction id', + ); + break; + } + + if (bundleUnits !== null) { + const reversal = await SubscriptionService.reverseRevenueCatBundlePurchase({ + userId, + productId, + transactionId: payload.event.transaction_id, + eventId: payload.event.id, + reason: payload.event.cancel_reason, + }); + + logger.info( + { + userId, + productId, + transactionId: payload.event.transaction_id, + reversed: reversal.reversed, + alreadyReversed: reversal.alreadyReversed, + reversedChats: reversal.reversedChats, + }, + 'RevenueCat bundle cancellation handled', + ); + } else { + const cancellation = await SubscriptionService.handleRevenueCatLifetimeCancellation({ + userId, + transactionId: payload.event.transaction_id, + eventId: payload.event.id, + reason: payload.event.cancel_reason, + }); + + logger.info( + { + userId, + productId, + transactionId: payload.event.transaction_id, + canceled: cancellation.canceled, + alreadyReversed: cancellation.alreadyReversed, + }, + 'RevenueCat lifetime cancellation handled', + ); + } + break; + case 'EXPIRATION': + if (isLifetimeProduct && payload.event.transaction_id) { + const cancellation = await SubscriptionService.handleRevenueCatLifetimeCancellation({ + userId, + transactionId: payload.event.transaction_id, + eventId: payload.event.id, + reason: 'expiration', + }); + + logger.info( + { + userId, + productId, + transactionId: payload.event.transaction_id, + canceled: cancellation.canceled, + alreadyReversed: cancellation.alreadyReversed, + }, + 'RevenueCat lifetime expiration handled', + ); + } + break; + default: + logger.info({ eventType: payload.event.type, userId, productId }, 'RevenueCat bundle/lifetime webhook ignored'); + break; + } + + return { ok: true }; + } const customerInfo = await getCustomerInfo(appUserId); if (customerInfo) { diff --git a/desci-server/src/services/SubscriptionService.ts b/desci-server/src/services/SubscriptionService.ts index 1194a34cb..0ac5f6bae 100644 --- a/desci-server/src/services/SubscriptionService.ts +++ b/desci-server/src/services/SubscriptionService.ts @@ -8,10 +8,13 @@ import { Feature, PlanCodename, Period, + RevenueCatPurchaseFulfillmentType, + StripeCheckoutFulfillmentType, SentEmailType, } from '@prisma/client'; import Stripe from 'stripe'; +import { getChatBundleUnitsForProduct } from '../config/mobileBundles.js'; import { getPlanTypeFromPriceId, getBillingIntervalFromPriceId } from '../config/stripe.js'; import { capturePostHogEvent } from '../lib/PostHog.js'; import { logger as parentLogger } from '../logger.js'; @@ -28,6 +31,29 @@ const logger = parentLogger.child({ const prisma = new PrismaClient(); const BUNDLE_ENTITLEMENT_TYPE = 'bundle_chats'; +const DEFAULT_BUNDLE_AUTO_REPLENISHMENT_THRESHOLD = 5; + +export interface BundleAutoReplenishmentStatus { + enabled: boolean; + threshold: number; + replenishmentInProgress: boolean; + lastAttemptedAt: Date | null; + lastSucceededAt: Date | null; + lastFailedAt: Date | null; + lastFailureReason: string | null; + latestBundlePurchase: { + stripePriceId: string; + purchasedUnits: number; + fulfilledAt: Date; + } | null; + hasSavedPaymentMethod: boolean; + paymentMethodSummary: { + brand: string | null; + last4: string | null; + expiryMonth: number | null; + expiryYear: number | null; + } | null; +} export class SubscriptionService { static async getOrCreateStripeCustomer(userId: number) { @@ -558,6 +584,378 @@ export class SubscriptionService { return await this.updateFeatureLimitsAfterCancellation(userId); } + static async handleRevenueCatBundlePurchase({ + userId, + productId, + transactionId, + originalTransactionId, + eventId, + store, + amountPaid, + currency, + presentedOfferingId, + }: { + userId: number; + productId: string; + transactionId: string; + originalTransactionId?: string | null; + eventId?: string | null; + store?: string | null; + amountPaid?: number | null; + currency?: string | null; + presentedOfferingId?: string | null; + }): Promise<{ + alreadyFulfilled: boolean; + grantedChats: number; + totalChats: number; + skipReason: string | null; + }> { + const bundleChatsPerUnit = getChatBundleUnitsForProduct(productId); + + if (!bundleChatsPerUnit) { + throw new Error(`Unsupported RevenueCat chat bundle product: ${productId}`); + } + + const { FeatureLimitsService } = await import('./FeatureLimits/FeatureLimitsService.js'); + const limitResult = await FeatureLimitsService.getOrCreateUserFeatureLimit(userId, Feature.RESEARCH_ASSISTANT); + if (limitResult.isErr()) { + throw limitResult.error; + } + + return await prisma.$transaction(async (tx) => { + const existingFulfillment = await tx.revenueCatPurchaseFulfillment.findUnique({ + where: { revenueCatTransactionId: transactionId }, + }); + + if (existingFulfillment) { + return { + alreadyFulfilled: true, + grantedChats: existingFulfillment.grantedUnits, + totalChats: existingFulfillment.purchasedUnits, + skipReason: existingFulfillment.skipReason, + }; + } + + const activeLimit = await tx.userFeatureLimit.findFirst({ + where: { + userId, + feature: Feature.RESEARCH_ASSISTANT, + isActive: true, + }, + orderBy: { createdAt: 'desc' }, + }); + + if (!activeLimit) { + throw new Error(`Active RESEARCH_ASSISTANT feature limit not found for user ${userId}`); + } + + const totalChats = bundleChatsPerUnit; + let grantedChats = 0; + let skipReason: string | null = null; + + if (activeLimit.useLimit === null) { + skipReason = 'user_has_unlimited_chat_access'; + } else { + grantedChats = totalChats; + await tx.userFeatureLimit.update({ + where: { id: activeLimit.id }, + data: { useLimit: activeLimit.useLimit + totalChats }, + }); + } + + await tx.revenueCatPurchaseFulfillment.create({ + data: { + userId, + revenueCatEventId: eventId ?? null, + revenueCatTransactionId: transactionId, + revenueCatOriginalTransactionId: originalTransactionId ?? null, + revenueCatProductId: productId, + revenueCatStore: store ?? null, + amountPaid: amountPaid ?? null, + currency: currency ?? null, + fulfillmentType: RevenueCatPurchaseFulfillmentType.BUNDLE_CHATS, + purchasedUnits: totalChats, + grantedUnits: grantedChats, + skipReason, + details: { + presentedOfferingId: presentedOfferingId ?? null, + bundleChatsPerUnit, + }, + }, + }); + + return { + alreadyFulfilled: false, + grantedChats, + totalChats, + skipReason, + }; + }); + } + + static async reverseRevenueCatBundlePurchase({ + userId, + productId, + transactionId, + eventId, + reason, + }: { + userId: number; + productId: string; + transactionId: string; + eventId?: string | null; + reason?: string | null; + }): Promise<{ + reversed: boolean; + alreadyReversed: boolean; + reversedChats: number; + }> { + const bundleChatsPerUnit = getChatBundleUnitsForProduct(productId); + if (!bundleChatsPerUnit) { + throw new Error(`Unsupported RevenueCat chat bundle product: ${productId}`); + } + + return await prisma.$transaction(async (tx) => { + const fulfillment = await tx.revenueCatPurchaseFulfillment.findUnique({ + where: { revenueCatTransactionId: transactionId }, + }); + + if (!fulfillment || fulfillment.fulfillmentType !== RevenueCatPurchaseFulfillmentType.BUNDLE_CHATS) { + return { reversed: false, alreadyReversed: false, reversedChats: 0 }; + } + + if (fulfillment.reversedAt) { + return { reversed: false, alreadyReversed: true, reversedChats: 0 }; + } + + const activeLimit = await tx.userFeatureLimit.findFirst({ + where: { + userId, + feature: Feature.RESEARCH_ASSISTANT, + isActive: true, + }, + orderBy: { createdAt: 'desc' }, + }); + + let reversedChats = 0; + if (activeLimit && activeLimit.useLimit !== null && fulfillment.grantedUnits > 0) { + const nextUseLimit = Math.max(0, activeLimit.useLimit - fulfillment.grantedUnits); + await tx.userFeatureLimit.update({ + where: { id: activeLimit.id }, + data: { useLimit: nextUseLimit }, + }); + reversedChats = fulfillment.grantedUnits; + } + + await tx.revenueCatPurchaseFulfillment.update({ + where: { id: fulfillment.id }, + data: { + reversedAt: new Date(), + reversalReason: reason ?? 'revenuecat_cancellation', + reversalDetails: { + eventId: eventId ?? null, + }, + }, + }); + + return { reversed: true, alreadyReversed: false, reversedChats }; + }); + } + + static async handleRevenueCatLifetimePurchase({ + userId, + productId, + transactionId, + originalTransactionId, + eventId, + store, + amountPaid, + currency, + presentedOfferingId, + }: { + userId: number; + productId: string; + transactionId: string; + originalTransactionId?: string | null; + eventId?: string | null; + store?: string | null; + amountPaid?: number | null; + currency?: string | null; + presentedOfferingId?: string | null; + }): Promise<{ + alreadyFulfilled: boolean; + createdNewLifetime: boolean; + }> { + const now = new Date(); + const syntheticSubscriptionId = `revenuecat_lifetime_${transactionId}`; + + const result = await prisma.$transaction(async (tx) => { + const existingFulfillment = await tx.revenueCatPurchaseFulfillment.findUnique({ + where: { revenueCatTransactionId: transactionId }, + }); + + if (existingFulfillment) { + return { + alreadyFulfilled: true, + createdNewLifetime: false, + }; + } + + const existingSubscription = await tx.subscription.findFirst({ + where: { + stripeSubscriptionId: syntheticSubscriptionId, + }, + }); + + await tx.subscription.upsert({ + where: { stripeSubscriptionId: syntheticSubscriptionId }, + create: { + userId, + stripeCustomerId: null, + stripeSubscriptionId: syntheticSubscriptionId, + stripePriceId: productId, + status: SubscriptionStatus.ACTIVE, + planType: PlanType.SCIWEAVE_LIFETIME, + billingInterval: BillingInterval.LIFETIME, + currentPeriodStart: now, + currentPeriodEnd: null, + cancelAtPeriodEnd: false, + canceledAt: null, + trialStart: null, + trialEnd: null, + }, + update: { + userId, + stripePriceId: productId, + status: SubscriptionStatus.ACTIVE, + planType: PlanType.SCIWEAVE_LIFETIME, + billingInterval: BillingInterval.LIFETIME, + currentPeriodStart: existingSubscription?.currentPeriodStart ?? now, + currentPeriodEnd: null, + cancelAtPeriodEnd: false, + canceledAt: null, + trialStart: null, + trialEnd: null, + }, + }); + + await tx.revenueCatPurchaseFulfillment.create({ + data: { + userId, + revenueCatEventId: eventId ?? null, + revenueCatTransactionId: transactionId, + revenueCatOriginalTransactionId: originalTransactionId ?? null, + revenueCatProductId: productId, + revenueCatStore: store ?? null, + amountPaid: amountPaid ?? null, + currency: currency ?? null, + fulfillmentType: RevenueCatPurchaseFulfillmentType.LIFETIME_UNLOCK, + purchasedUnits: 1, + grantedUnits: 1, + details: { + presentedOfferingId: presentedOfferingId ?? null, + syntheticSubscriptionId, + }, + }, + }); + + return { + alreadyFulfilled: false, + createdNewLifetime: !existingSubscription, + }; + }); + + await this.updateUserFeatureLimits(userId, PlanType.SCIWEAVE_LIFETIME); + + if (result.createdNewLifetime) { + try { + const user = await this.getUserForEmail(userId); + if (user) { + await sendEmail({ + type: SciweaveEmailTypes.SCIWEAVE_UPGRADE_EMAIL, + payload: { + email: user.email, + firstName: user.firstName, + lastName: user.lastName, + }, + }); + logger.info({ userId, transactionId, productId }, 'Upgrade email sent for RevenueCat lifetime purchase'); + } + } catch (emailError) { + logger.error( + { error: emailError, userId, transactionId, productId }, + 'Failed to send upgrade email for RevenueCat lifetime purchase', + ); + } + } + + return result; + } + + static async handleRevenueCatLifetimeCancellation({ + userId, + transactionId, + eventId, + reason, + }: { + userId: number; + transactionId: string; + eventId?: string | null; + reason?: string | null; + }): Promise<{ + canceled: boolean; + alreadyReversed: boolean; + }> { + const syntheticSubscriptionId = `revenuecat_lifetime_${transactionId}`; + + const result = await prisma.$transaction(async (tx) => { + const fulfillment = await tx.revenueCatPurchaseFulfillment.findUnique({ + where: { revenueCatTransactionId: transactionId }, + }); + + if (!fulfillment || fulfillment.fulfillmentType !== RevenueCatPurchaseFulfillmentType.LIFETIME_UNLOCK) { + return { canceled: false, alreadyReversed: false }; + } + + if (fulfillment.reversedAt) { + return { canceled: false, alreadyReversed: true }; + } + + await tx.revenueCatPurchaseFulfillment.update({ + where: { id: fulfillment.id }, + data: { + reversedAt: new Date(), + reversalReason: reason ?? 'revenuecat_cancellation', + reversalDetails: { + eventId: eventId ?? null, + }, + }, + }); + + await tx.subscription.updateMany({ + where: { + userId, + stripeSubscriptionId: syntheticSubscriptionId, + status: SubscriptionStatus.ACTIVE, + }, + data: { + status: SubscriptionStatus.CANCELED, + cancelAtPeriodEnd: false, + canceledAt: new Date(), + currentPeriodEnd: new Date(), + }, + }); + + return { canceled: true, alreadyReversed: false }; + }); + + if (result.canceled) { + await this.updateFeatureLimitsAfterCancellation(userId); + } + + return result; + } + /** * Tracks subscription renewal event for recurring payments * Only fires for renewals (payments after the first invoice) @@ -733,18 +1131,11 @@ export class SubscriptionService { return; } - await prisma.paymentMethod.create({ - data: { - userId, - stripeCustomerId: paymentMethod.customer as string, - stripePaymentMethodId: paymentMethod.id, - type: this.mapStripePaymentMethodType(paymentMethod.type), - last4: paymentMethod.card?.last4, - brand: paymentMethod.card?.brand, - expiryMonth: paymentMethod.card?.exp_month, - expiryYear: paymentMethod.card?.exp_year, - isDefault: false, // Will be updated if set as default - }, + await this.upsertPaymentMethodRecord({ + userId, + stripeCustomerId: paymentMethod.customer as string, + paymentMethod, + isDefault: false, }); logger.info('Payment method attached processed', { paymentMethodId: paymentMethod.id }); @@ -855,6 +1246,14 @@ export class SubscriptionService { }, 'Processed bundle checkout completion', ); + + if (paymentIntentId && customerId) { + await this.saveBundlePaymentMethodForReuse({ + userId, + customerId, + paymentIntentId, + }); + } return; } @@ -1149,6 +1548,312 @@ export class SubscriptionService { }; } + static async getBundleAutoReplenishmentStatus(userId: number): Promise { + const [settings, latestBundlePurchase, paymentMethod] = await Promise.all([ + prisma.bundleAutoReplenishment.findUnique({ + where: { userId }, + }), + this.getLatestBundlePurchase(userId), + prisma.paymentMethod.findFirst({ + where: { userId }, + orderBy: [{ isDefault: 'desc' }, { createdAt: 'desc' }], + }), + ]); + + return { + enabled: settings?.isEnabled ?? false, + threshold: settings?.threshold ?? DEFAULT_BUNDLE_AUTO_REPLENISHMENT_THRESHOLD, + replenishmentInProgress: settings?.replenishmentInProgress ?? false, + lastAttemptedAt: settings?.lastAttemptedAt ?? null, + lastSucceededAt: settings?.lastSucceededAt ?? null, + lastFailedAt: settings?.lastFailedAt ?? null, + lastFailureReason: settings?.lastFailureReason ?? null, + latestBundlePurchase: latestBundlePurchase + ? { + stripePriceId: latestBundlePurchase.stripePriceId, + purchasedUnits: latestBundlePurchase.purchasedUnits, + fulfilledAt: latestBundlePurchase.fulfilledAt, + } + : null, + hasSavedPaymentMethod: !!paymentMethod, + paymentMethodSummary: paymentMethod + ? { + brand: paymentMethod.brand ?? null, + last4: paymentMethod.last4 ?? null, + expiryMonth: paymentMethod.expiryMonth ?? null, + expiryYear: paymentMethod.expiryYear ?? null, + } + : null, + }; + } + + static async updateBundleAutoReplenishment( + userId: number, + { + enabled, + threshold = DEFAULT_BUNDLE_AUTO_REPLENISHMENT_THRESHOLD, + }: { + enabled: boolean; + threshold?: number; + }, + ): Promise { + const latestBundlePurchase = await this.getLatestBundlePurchase(userId); + + if (enabled) { + if (!latestBundlePurchase) { + throw new Error('You need a previous chat bundle purchase before enabling auto-replenishment.'); + } + + const activePaidSubscription = await prisma.subscription.findFirst({ + where: { + userId, + status: SubscriptionStatus.ACTIVE, + planType: { not: PlanType.FREE }, + }, + select: { id: true }, + }); + + if (activePaidSubscription) { + throw new Error('Auto-replenishment is only available for chat bundle users without an active subscription.'); + } + + const customerId = await this.getStripeCustomerId(userId); + if (!customerId) { + throw new Error('No Stripe customer found for this user.'); + } + + const paymentMethodId = await this.getReusablePaymentMethodId(userId, customerId); + if (!paymentMethodId) { + throw new Error('No saved payment method found. Buy a bundle or add a card first.'); + } + } + + await prisma.bundleAutoReplenishment.upsert({ + where: { userId }, + update: { + isEnabled: enabled, + threshold, + replenishmentInProgress: false, + ...(enabled ? { lastFailureReason: null } : {}), + }, + create: { + userId, + isEnabled: enabled, + threshold, + replenishmentInProgress: false, + }, + }); + + return await this.getBundleAutoReplenishmentStatus(userId); + } + + static async triggerBundleAutoReplenishmentIfNeeded({ + userId, + remainingUses, + }: { + userId: number; + remainingUses: number | null; + }): Promise { + if (remainingUses === null) { + return false; + } + + const settings = await prisma.bundleAutoReplenishment.findUnique({ + where: { userId }, + }); + + if (!settings?.isEnabled || settings.replenishmentInProgress || remainingUses > settings.threshold) { + return false; + } + + const latestBundlePurchase = await this.getLatestBundlePurchase(userId); + if (!latestBundlePurchase) { + await prisma.bundleAutoReplenishment.update({ + where: { userId }, + data: { + isEnabled: false, + replenishmentInProgress: false, + lastFailedAt: new Date(), + lastFailureReason: 'No prior bundle purchase is available for auto-replenishment.', + }, + }); + return false; + } + + const claim = await prisma.bundleAutoReplenishment.updateMany({ + where: { + userId, + isEnabled: true, + replenishmentInProgress: false, + }, + data: { + replenishmentInProgress: true, + lastAttemptedAt: new Date(), + lastFailureReason: null, + }, + }); + + if (claim.count === 0) { + return false; + } + + try { + const customerId = await this.getStripeCustomerId(userId); + if (!customerId) { + throw new Error('No Stripe customer found.'); + } + + const paymentMethodId = await this.getReusablePaymentMethodId(userId, customerId); + if (!paymentMethodId) { + throw new Error('No reusable payment method found.'); + } + + const stripe = getStripe(); + const price = await stripe.prices.retrieve(latestBundlePurchase.stripePriceId); + if (!price.unit_amount || !price.currency) { + throw new Error('Last bundle price is missing amount or currency.'); + } + + const bundleChatsPerUnit = Number.parseInt(price.metadata?.bundle_chats || '', 10); + if (!Number.isFinite(bundleChatsPerUnit) || bundleChatsPerUnit <= 0) { + throw new Error('Last bundle price is missing valid bundle metadata.'); + } + + await stripe.paymentIntents.create({ + amount: price.unit_amount, + currency: price.currency, + customer: customerId, + payment_method: paymentMethodId, + off_session: true, + confirm: true, + description: `SciWeave bundle auto-replenishment (${bundleChatsPerUnit} chats)`, + metadata: { + type: 'bundle_auto_replenishment', + userId: userId.toString(), + priceId: latestBundlePurchase.stripePriceId, + bundleChats: bundleChatsPerUnit.toString(), + quantity: '1', + }, + }); + + return true; + } catch (error) { + logger.error({ error, userId }, 'Failed to trigger bundle auto-replenishment'); + + await prisma.bundleAutoReplenishment.update({ + where: { userId }, + data: { + isEnabled: false, + replenishmentInProgress: false, + lastFailedAt: new Date(), + lastFailureReason: error instanceof Error ? error.message : 'Failed to trigger auto-replenishment.', + }, + }); + + return false; + } + } + + static async handleBundleAutoReplenishmentSucceeded(paymentIntent: Stripe.PaymentIntent) { + if (paymentIntent.metadata?.type !== 'bundle_auto_replenishment') { + return false; + } + + const metadataUserId = paymentIntent.metadata?.userId ? Number.parseInt(paymentIntent.metadata.userId, 10) : null; + const userId = + metadataUserId && Number.isFinite(metadataUserId) + ? metadataUserId + : paymentIntent.customer + ? await this.getUserIdFromCustomer(paymentIntent.customer as string) + : null; + + if (!userId) { + throw new Error(`Unable to resolve user for auto-replenishment payment intent ${paymentIntent.id}`); + } + + const priceId = paymentIntent.metadata?.priceId; + if (!priceId) { + throw new Error(`Missing priceId metadata for auto-replenishment payment intent ${paymentIntent.id}`); + } + + const bundleChatsPerUnit = Number.parseInt(paymentIntent.metadata?.bundleChats || '', 10); + if (!Number.isFinite(bundleChatsPerUnit) || bundleChatsPerUnit <= 0) { + throw new Error(`Invalid bundleChats metadata for auto-replenishment payment intent ${paymentIntent.id}`); + } + + await this.handleBundleCheckoutFulfillment({ + sessionId: `auto_replenishment:${paymentIntent.id}`, + userId, + customerId: + typeof paymentIntent.customer === 'string' ? paymentIntent.customer : (paymentIntent.customer?.id ?? null), + priceId, + paymentIntentId: paymentIntent.id, + amountPaid: paymentIntent.amount_received || paymentIntent.amount || null, + currency: paymentIntent.currency ?? null, + quantity: Math.max(1, Number.parseInt(paymentIntent.metadata?.quantity || '1', 10) || 1), + bundleChatsPerUnit, + }); + + await prisma.bundleAutoReplenishment.upsert({ + where: { userId }, + update: { + replenishmentInProgress: false, + lastSucceededAt: new Date(), + lastFailureReason: null, + }, + create: { + userId, + isEnabled: false, + threshold: DEFAULT_BUNDLE_AUTO_REPLENISHMENT_THRESHOLD, + replenishmentInProgress: false, + lastSucceededAt: new Date(), + }, + }); + + return true; + } + + static async handleBundleAutoReplenishmentFailed(paymentIntent: Stripe.PaymentIntent) { + if (paymentIntent.metadata?.type !== 'bundle_auto_replenishment') { + return false; + } + + const metadataUserId = paymentIntent.metadata?.userId ? Number.parseInt(paymentIntent.metadata.userId, 10) : null; + const userId = + metadataUserId && Number.isFinite(metadataUserId) + ? metadataUserId + : paymentIntent.customer + ? await this.getUserIdFromCustomer(paymentIntent.customer as string) + : null; + + if (!userId) { + logger.warn({ paymentIntentId: paymentIntent.id }, 'Failed auto-replenishment could not resolve user'); + return false; + } + + await prisma.bundleAutoReplenishment.upsert({ + where: { userId }, + update: { + isEnabled: false, + replenishmentInProgress: false, + lastFailedAt: new Date(), + lastFailureReason: + paymentIntent.last_payment_error?.message || 'Automatic bundle replenishment payment failed.', + }, + create: { + userId, + isEnabled: false, + threshold: DEFAULT_BUNDLE_AUTO_REPLENISHMENT_THRESHOLD, + replenishmentInProgress: false, + lastFailedAt: new Date(), + lastFailureReason: + paymentIntent.last_payment_error?.message || 'Automatic bundle replenishment payment failed.', + }, + }); + + return true; + } + static async updateSubscriptionPlan(userId: number, planType: string) { // TODO: Implement plan change logic with Stripe API throw new Error('Plan updates not yet implemented'); @@ -1211,6 +1916,170 @@ export class SubscriptionService { } // Helper methods + private static async getLatestBundlePurchase(userId: number) { + return await prisma.stripeCheckoutFulfillment.findFirst({ + where: { + userId, + fulfillmentType: StripeCheckoutFulfillmentType.BUNDLE_CHATS, + grantedUnits: { + gt: 0, + }, + }, + orderBy: { fulfilledAt: 'desc' }, + select: { + stripePriceId: true, + purchasedUnits: true, + fulfilledAt: true, + }, + }); + } + + private static async getReusablePaymentMethodId(userId: number, customerId: string): Promise { + const stripe = getStripe(); + + try { + const customer = await stripe.customers.retrieve(customerId, { + expand: ['invoice_settings.default_payment_method'], + }); + + if (customer && !customer.deleted) { + const defaultPaymentMethod = (customer as Stripe.Customer).invoice_settings?.default_payment_method; + if (typeof defaultPaymentMethod === 'string') { + return defaultPaymentMethod; + } + if (defaultPaymentMethod && typeof defaultPaymentMethod !== 'string') { + await this.upsertPaymentMethodRecord({ + userId, + stripeCustomerId: customerId, + paymentMethod: defaultPaymentMethod, + isDefault: true, + }); + return defaultPaymentMethod.id; + } + } + } catch (error) { + logger.error({ error, customerId, userId }, 'Failed to read Stripe customer default payment method'); + } + + const latestPaymentMethod = await prisma.paymentMethod.findFirst({ + where: { userId, stripeCustomerId: customerId }, + orderBy: [{ isDefault: 'desc' }, { createdAt: 'desc' }], + select: { stripePaymentMethodId: true }, + }); + + if (latestPaymentMethod?.stripePaymentMethodId) { + return latestPaymentMethod.stripePaymentMethodId; + } + + const stripePaymentMethods = await stripe.paymentMethods.list({ + customer: customerId, + type: 'card', + limit: 1, + }); + + const fallbackPaymentMethod = stripePaymentMethods.data[0]; + if (!fallbackPaymentMethod) { + return null; + } + + await this.upsertPaymentMethodRecord({ + userId, + stripeCustomerId: customerId, + paymentMethod: fallbackPaymentMethod, + isDefault: false, + }); + + return fallbackPaymentMethod.id; + } + + private static async saveBundlePaymentMethodForReuse({ + userId, + customerId, + paymentIntentId, + }: { + userId: number; + customerId: string; + paymentIntentId: string; + }) { + try { + const stripe = getStripe(); + const paymentIntent = await stripe.paymentIntents.retrieve(paymentIntentId); + const paymentMethodId = + typeof paymentIntent.payment_method === 'string' + ? paymentIntent.payment_method + : (paymentIntent.payment_method?.id ?? null); + + if (!paymentMethodId) { + return; + } + + await stripe.customers.update(customerId, { + invoice_settings: { + default_payment_method: paymentMethodId, + }, + }); + + const paymentMethod = + typeof paymentIntent.payment_method === 'string' + ? await stripe.paymentMethods.retrieve(paymentMethodId) + : paymentIntent.payment_method; + + await this.upsertPaymentMethodRecord({ + userId, + stripeCustomerId: customerId, + paymentMethod, + isDefault: true, + }); + } catch (error) { + logger.error({ error, userId, customerId, paymentIntentId }, 'Failed to save bundle payment method for reuse'); + } + } + + private static async upsertPaymentMethodRecord({ + userId, + stripeCustomerId, + paymentMethod, + isDefault, + }: { + userId: number; + stripeCustomerId: string; + paymentMethod: Stripe.PaymentMethod; + isDefault: boolean; + }) { + await prisma.$transaction(async (tx) => { + if (isDefault) { + await tx.paymentMethod.updateMany({ + where: { userId }, + data: { isDefault: false }, + }); + } + + await tx.paymentMethod.upsert({ + where: { stripePaymentMethodId: paymentMethod.id }, + update: { + stripeCustomerId, + type: this.mapStripePaymentMethodType(paymentMethod.type), + last4: paymentMethod.card?.last4, + brand: paymentMethod.card?.brand, + expiryMonth: paymentMethod.card?.exp_month, + expiryYear: paymentMethod.card?.exp_year, + isDefault, + }, + create: { + userId, + stripeCustomerId, + stripePaymentMethodId: paymentMethod.id, + type: this.mapStripePaymentMethodType(paymentMethod.type), + last4: paymentMethod.card?.last4, + brand: paymentMethod.card?.brand, + expiryMonth: paymentMethod.card?.exp_month, + expiryYear: paymentMethod.card?.exp_year, + isDefault, + }, + }); + }); + } + private static async getUserIdFromCustomer(customerId: string): Promise { // First try local DB lookup const subscription = await prisma.subscription.findFirst({ diff --git a/desci-server/src/services/data/processing.ts b/desci-server/src/services/data/processing.ts index cdea8f4f1..0a656e36b 100644 --- a/desci-server/src/services/data/processing.ts +++ b/desci-server/src/services/data/processing.ts @@ -96,6 +96,7 @@ export async function processS3DataToIpfs({ }: ProcessS3DataToIpfsParams): Promise> { let pinResult: IpfsPinnedResult[] = []; let manifestPathsToTypesPrune: Record = {}; + let draftNodeTreePathsAdded: string[] = []; if (contextPath.endsWith('/')) contextPath = contextPath.slice(0, -1); try { await ensureSpaceAvailable(files, user); @@ -128,6 +129,7 @@ export async function processS3DataToIpfs({ user, contextPath, }); + draftNodeTreePathsAdded = draftNodeTreeEntries.map((entry) => entry.path); const addedEntries = await prisma.draftNodeTree.createMany({ data: draftNodeTreeEntries, skipDuplicates: true, @@ -230,9 +232,10 @@ export async function processS3DataToIpfs({ // const manifest = await getLatestManifestFromNode(node); logger.error({ error }, 'Error processing S3 data to IPFS'); if (pinResult.length) { - handleCleanupOnMidProcessingError({ + await handleCleanupOnMidProcessingError({ pinnedFiles: pinResult, manifestPathsToDbComponentTypesMap: manifestPathsToTypesPrune, + draftNodeTreePathsAdded, node, user, }); @@ -682,6 +685,7 @@ interface HandleCleanupOnMidProcessingErrorParams { node: Node; user: User; pinnedFiles: IpfsPinnedResult[]; + draftNodeTreePathsAdded?: string[]; manifestPathsToDbComponentTypesMap: Record; } @@ -690,6 +694,7 @@ interface HandleCleanupOnMidProcessingErrorParams { */ export async function handleCleanupOnMidProcessingError({ pinnedFiles, + draftNodeTreePathsAdded, manifestPathsToDbComponentTypesMap, node, user, @@ -731,6 +736,19 @@ export async function handleCleanupOnMidProcessingError({ ); // In this case, we log the files just incase, no matter how many. } + + if (draftNodeTreePathsAdded?.length) { + const deletedDraftTreeEntries = await prisma.draftNodeTree.deleteMany({ + where: { + nodeId: node.id, + path: { in: draftNodeTreePathsAdded }, + }, + }); + logger.info( + { deletedDraftTreeEntries: deletedDraftTreeEntries.count, draftNodeTreePathsAdded }, + '[UPDATE DATASET E:2] removed DraftNodeTree entries created by failed upload', + ); + } } /** diff --git a/desci-server/src/services/journals/JournalInviteService.ts b/desci-server/src/services/journals/JournalInviteService.ts index cee2d8a9a..a78c02a34 100644 --- a/desci-server/src/services/journals/JournalInviteService.ts +++ b/desci-server/src/services/journals/JournalInviteService.ts @@ -60,6 +60,7 @@ async function inviteJournalEditor({ name: true, description: true, iconCid: true, + imageUrl: true, }, }); @@ -345,6 +346,7 @@ async function resendEditorInvite({ name: true, description: true, iconCid: true, + imageUrl: true, }, }); diff --git a/desci-server/src/utils/generateArweaveKeys.ts b/desci-server/src/utils/generateArweaveKeys.ts index ec73db5e7..a2856ae4c 100755 --- a/desci-server/src/utils/generateArweaveKeys.ts +++ b/desci-server/src/utils/generateArweaveKeys.ts @@ -1,4 +1,4 @@ -import Arweave from 'arweave/node'; +import Arweave from 'arweave'; import dotenv from 'dotenv'; dotenv.config({ path: __dirname + '/.env' }); diff --git a/desci-server/src/utils/manifest.ts b/desci-server/src/utils/manifest.ts index ce3aa3c7d..6d8f37773 100644 --- a/desci-server/src/utils/manifest.ts +++ b/desci-server/src/utils/manifest.ts @@ -18,10 +18,10 @@ const IPFS_RESOLVER_OVERRIDE = process.env.IPFS_RESOLVER_OVERRIDE; export const cleanupManifestUrl = (url: string, gateway?: string) => { if (url && (PUBLIC_IPFS_PATH || gateway)) { const s = url.split('/'); - logger.info({ gateway, PUBLIC_IPFS_PATH }, 'cleanupManifestUrl'); + logger.debug({ gateway, PUBLIC_IPFS_PATH }, 'cleanupManifestUrl'); // TODO: revert back to PUBLIC_IPFS_PATH after testing const res = `${gateway ? gateway : PUBLIC_IPFS_PATH}/${s[s.length - 1]}`; - parentLogger.info({ fn: 'cleanupManifestUrl', url, gateway }, `resolving ${url} => ${res}`); + parentLogger.debug({ fn: 'cleanupManifestUrl', url, gateway }, `resolving ${url} => ${res}`); return res; } return url; diff --git a/desci-server/src/utils/upload.ts b/desci-server/src/utils/upload.ts index 33ebb8f25..5930ec40d 100644 --- a/desci-server/src/utils/upload.ts +++ b/desci-server/src/utils/upload.ts @@ -1,5 +1,3 @@ -import { Blob } from 'buffer'; - export const base64ToBlob = (dataURI: string) => { function dataURItoBlob(dataURI: string) { // convert base64/URLEncoded data component to raw binary data held in a string diff --git a/desci-server/test/integration/bundleAutoReplenishment.test.ts b/desci-server/test/integration/bundleAutoReplenishment.test.ts new file mode 100644 index 000000000..bd5caf0b3 --- /dev/null +++ b/desci-server/test/integration/bundleAutoReplenishment.test.ts @@ -0,0 +1,227 @@ +import { Feature, PaymentMethodType, StripeCheckoutFulfillmentType, User } from '@prisma/client'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const stripeMock = { + customers: { + retrieve: vi.fn(), + update: vi.fn(), + }, + prices: { + retrieve: vi.fn(), + }, + paymentIntents: { + create: vi.fn(), + retrieve: vi.fn(), + }, + paymentMethods: { + list: vi.fn(), + retrieve: vi.fn(), + }, +}; + +import { prisma } from '../../src/client.js'; +import { SCIWEAVE_FREE_LIMIT } from '../../src/config.js'; +import { SubscriptionService } from '../../src/services/SubscriptionService.js'; +import * as stripeUtils from '../../src/utils/stripe.js'; + +describe('Bundle auto-replenishment', () => { + let user: User; + + beforeEach(async () => { + vi.clearAllMocks(); + vi.spyOn(stripeUtils, 'getStripe').mockReturnValue(stripeMock as any); + + await prisma.$queryRaw`TRUNCATE TABLE "BundleAutoReplenishment" CASCADE;`; + await prisma.$queryRaw`TRUNCATE TABLE "StripeCheckoutFulfillment" CASCADE;`; + await prisma.$queryRaw`TRUNCATE TABLE "PaymentMethod" CASCADE;`; + await prisma.$queryRaw`TRUNCATE TABLE "Subscription" CASCADE;`; + await prisma.$queryRaw`TRUNCATE TABLE "UserFeatureLimit" CASCADE;`; + await prisma.$queryRaw`TRUNCATE TABLE "User" CASCADE;`; + + user = await prisma.user.create({ + data: { + email: 'bundle-auto-replenishment-test@desci.com', + name: 'Bundle Auto Replenishment Test User', + stripeUserId: 'cus_bundle_auto_123', + }, + }); + + stripeMock.customers.retrieve.mockResolvedValue({ + id: 'cus_bundle_auto_123', + deleted: false, + invoice_settings: { + default_payment_method: 'pm_bundle_default_123', + }, + }); + stripeMock.customers.update.mockResolvedValue({}); + stripeMock.paymentMethods.list.mockResolvedValue({ data: [] }); + }); + + it('enables auto-replenishment for a user with a previous bundle and saved card', async () => { + await prisma.stripeCheckoutFulfillment.create({ + data: { + userId: user.id, + stripeSessionId: 'cs_bundle_30', + stripePriceId: 'price_bundle_30', + stripePaymentIntentId: 'pi_bundle_30', + stripeCustomerId: 'cus_bundle_auto_123', + fulfillmentType: StripeCheckoutFulfillmentType.BUNDLE_CHATS, + purchasedUnits: 30, + grantedUnits: 30, + }, + }); + + await prisma.paymentMethod.create({ + data: { + userId: user.id, + stripeCustomerId: 'cus_bundle_auto_123', + stripePaymentMethodId: 'pm_bundle_default_123', + type: PaymentMethodType.CARD, + isDefault: true, + brand: 'visa', + last4: '4242', + expiryMonth: 12, + expiryYear: 2030, + }, + }); + + const status = await SubscriptionService.updateBundleAutoReplenishment(user.id, { + enabled: true, + }); + + expect(status.enabled).toBe(true); + expect(status.threshold).toBe(5); + expect(status.latestBundlePurchase?.purchasedUnits).toBe(30); + expect(status.hasSavedPaymentMethod).toBe(true); + }); + + it('charges the most recently purchased bundle when remaining chats reach the threshold', async () => { + await prisma.bundleAutoReplenishment.create({ + data: { + userId: user.id, + isEnabled: true, + threshold: 5, + }, + }); + + await prisma.stripeCheckoutFulfillment.createMany({ + data: [ + { + userId: user.id, + stripeSessionId: 'cs_bundle_old', + stripePriceId: 'price_bundle_10', + stripePaymentIntentId: 'pi_bundle_old', + stripeCustomerId: 'cus_bundle_auto_123', + fulfillmentType: StripeCheckoutFulfillmentType.BUNDLE_CHATS, + purchasedUnits: 10, + grantedUnits: 10, + fulfilledAt: new Date('2026-05-01T00:00:00Z'), + }, + { + userId: user.id, + stripeSessionId: 'cs_bundle_latest', + stripePriceId: 'price_bundle_30', + stripePaymentIntentId: 'pi_bundle_latest', + stripeCustomerId: 'cus_bundle_auto_123', + fulfillmentType: StripeCheckoutFulfillmentType.BUNDLE_CHATS, + purchasedUnits: 30, + grantedUnits: 30, + fulfilledAt: new Date('2026-05-10T00:00:00Z'), + }, + ], + }); + + await prisma.paymentMethod.create({ + data: { + userId: user.id, + stripeCustomerId: 'cus_bundle_auto_123', + stripePaymentMethodId: 'pm_bundle_default_123', + type: PaymentMethodType.CARD, + isDefault: true, + }, + }); + + stripeMock.prices.retrieve.mockResolvedValue({ + id: 'price_bundle_30', + unit_amount: 999, + currency: 'usd', + metadata: { + bundle_chats: '30', + }, + }); + stripeMock.paymentIntents.create.mockResolvedValue({ + id: 'pi_auto_replenishment_123', + status: 'succeeded', + }); + + const triggered = await SubscriptionService.triggerBundleAutoReplenishmentIfNeeded({ + userId: user.id, + remainingUses: 5, + }); + + expect(triggered).toBe(true); + expect(stripeMock.paymentIntents.create).toHaveBeenCalledWith( + expect.objectContaining({ + amount: 999, + customer: 'cus_bundle_auto_123', + payment_method: 'pm_bundle_default_123', + metadata: expect.objectContaining({ + priceId: 'price_bundle_30', + bundleChats: '30', + type: 'bundle_auto_replenishment', + }), + }), + ); + + const settings = await prisma.bundleAutoReplenishment.findUnique({ + where: { userId: user.id }, + }); + expect(settings?.replenishmentInProgress).toBe(true); + }); + + it('fulfills a successful auto-replenishment payment intent and clears the in-progress flag', async () => { + await prisma.bundleAutoReplenishment.create({ + data: { + userId: user.id, + isEnabled: true, + threshold: 5, + replenishmentInProgress: true, + }, + }); + + const handled = await SubscriptionService.handleBundleAutoReplenishmentSucceeded({ + id: 'pi_auto_success_123', + amount: 999, + amount_received: 999, + currency: 'usd', + customer: 'cus_bundle_auto_123', + metadata: { + type: 'bundle_auto_replenishment', + userId: String(user.id), + priceId: 'price_bundle_30', + bundleChats: '30', + quantity: '1', + }, + } as any); + + expect(handled).toBe(true); + + const activeLimit = await prisma.userFeatureLimit.findFirst({ + where: { userId: user.id, feature: Feature.RESEARCH_ASSISTANT, isActive: true }, + orderBy: { createdAt: 'desc' }, + }); + expect(activeLimit?.useLimit).toBe(SCIWEAVE_FREE_LIMIT + 30); + + const fulfillment = await prisma.stripeCheckoutFulfillment.findUnique({ + where: { stripeSessionId: 'auto_replenishment:pi_auto_success_123' }, + }); + expect(fulfillment?.stripePriceId).toBe('price_bundle_30'); + expect(fulfillment?.grantedUnits).toBe(30); + + const settings = await prisma.bundleAutoReplenishment.findUnique({ + where: { userId: user.id }, + }); + expect(settings?.replenishmentInProgress).toBe(false); + expect(settings?.lastSucceededAt).not.toBeNull(); + }); +}); diff --git a/desci-server/test/integration/revenuecatBundles.test.ts b/desci-server/test/integration/revenuecatBundles.test.ts new file mode 100644 index 000000000..f3bc4fdb2 --- /dev/null +++ b/desci-server/test/integration/revenuecatBundles.test.ts @@ -0,0 +1,192 @@ +import { Feature, PlanType, RevenueCatPurchaseFulfillmentType, SubscriptionStatus, User } from '@prisma/client'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('../../src/services/email/email.js', () => ({ + sendEmail: vi.fn().mockResolvedValue(undefined), +})); + +import { prisma } from '../../src/client.js'; +import { MOBILE_CHAT_BUNDLE_PRODUCT_IDS } from '../../src/config/mobileBundles.js'; +import { SCIWEAVE_FREE_LIMIT } from '../../src/config.js'; +import { handleWebhookEvent, type RevenueCatWebhookPayload } from '../../src/services/RevenueCatService.js'; + +describe('RevenueCat bundle fulfillment', () => { + let user: User; + + beforeEach(async () => { + await prisma.$queryRaw`TRUNCATE TABLE "RevenueCatPurchaseFulfillment" CASCADE;`; + await prisma.$queryRaw`TRUNCATE TABLE "Subscription" CASCADE;`; + await prisma.$queryRaw`TRUNCATE TABLE "UserFeatureLimit" CASCADE;`; + await prisma.$queryRaw`TRUNCATE TABLE "User" CASCADE;`; + + user = await prisma.user.create({ + data: { + email: 'revenuecat-bundles-test@desci.com', + name: 'RevenueCat Bundles Test User', + }, + }); + }); + + it('grants chats for a bundle purchase webhook', async () => { + const result = await handleWebhookEvent( + makePayload({ + userId: user.id, + type: 'NON_RENEWING_PURCHASE', + productId: MOBILE_CHAT_BUNDLE_PRODUCT_IDS.CHAT_BUNDLE_10, + transactionId: 'bundle_txn_10', + price: 4.99, + }), + ); + + expect(result).toEqual({ ok: true }); + + const activeLimit = await prisma.userFeatureLimit.findFirst({ + where: { userId: user.id, feature: Feature.RESEARCH_ASSISTANT, isActive: true }, + orderBy: { createdAt: 'desc' }, + }); + expect(activeLimit?.useLimit).toBe(SCIWEAVE_FREE_LIMIT + 10); + + const fulfillment = await prisma.revenueCatPurchaseFulfillment.findUnique({ + where: { revenueCatTransactionId: 'bundle_txn_10' }, + }); + expect(fulfillment?.fulfillmentType).toBe(RevenueCatPurchaseFulfillmentType.BUNDLE_CHATS); + expect(fulfillment?.grantedUnits).toBe(10); + expect(fulfillment?.purchasedUnits).toBe(10); + }); + + it('is idempotent for duplicate bundle webhooks', async () => { + const payload = makePayload({ + userId: user.id, + type: 'NON_RENEWING_PURCHASE', + productId: MOBILE_CHAT_BUNDLE_PRODUCT_IDS.CHAT_BUNDLE_30, + transactionId: 'bundle_txn_duplicate', + price: 9.99, + }); + + await handleWebhookEvent(payload); + await handleWebhookEvent(payload); + + const activeLimit = await prisma.userFeatureLimit.findFirst({ + where: { userId: user.id, feature: Feature.RESEARCH_ASSISTANT, isActive: true }, + orderBy: { createdAt: 'desc' }, + }); + expect(activeLimit?.useLimit).toBe(SCIWEAVE_FREE_LIMIT + 30); + + const fulfillments = await prisma.revenueCatPurchaseFulfillment.findMany({ + where: { revenueCatTransactionId: 'bundle_txn_duplicate' }, + }); + expect(fulfillments).toHaveLength(1); + }); + + it('reverses a bundle grant on cancellation', async () => { + const transactionId = 'bundle_txn_cancel'; + + await handleWebhookEvent( + makePayload({ + userId: user.id, + type: 'NON_RENEWING_PURCHASE', + productId: MOBILE_CHAT_BUNDLE_PRODUCT_IDS.CHAT_BUNDLE_30, + transactionId, + price: 9.99, + }), + ); + + await handleWebhookEvent( + makePayload({ + userId: user.id, + type: 'CANCELLATION', + productId: MOBILE_CHAT_BUNDLE_PRODUCT_IDS.CHAT_BUNDLE_30, + transactionId, + cancelReason: 'CUSTOMER_SUPPORT', + }), + ); + + const activeLimit = await prisma.userFeatureLimit.findFirst({ + where: { userId: user.id, feature: Feature.RESEARCH_ASSISTANT, isActive: true }, + orderBy: { createdAt: 'desc' }, + }); + expect(activeLimit?.useLimit).toBe(SCIWEAVE_FREE_LIMIT); + + const fulfillment = await prisma.revenueCatPurchaseFulfillment.findUnique({ + where: { revenueCatTransactionId: transactionId }, + }); + expect(fulfillment?.reversedAt).not.toBeNull(); + expect(fulfillment?.reversalReason).toBe('CUSTOMER_SUPPORT'); + }); + + it('grants lifetime access for the lifetime product', async () => { + const transactionId = 'lifetime_txn_1'; + + const result = await handleWebhookEvent( + makePayload({ + userId: user.id, + type: 'NON_RENEWING_PURCHASE', + productId: MOBILE_CHAT_BUNDLE_PRODUCT_IDS.LIFETIME, + transactionId, + price: 199.99, + }), + ); + + expect(result).toEqual({ ok: true }); + + const activeLimit = await prisma.userFeatureLimit.findFirst({ + where: { userId: user.id, feature: Feature.RESEARCH_ASSISTANT, isActive: true }, + orderBy: { createdAt: 'desc' }, + }); + expect(activeLimit?.useLimit).toBeNull(); + + const subscription = await prisma.subscription.findFirst({ + where: { stripeSubscriptionId: `revenuecat_lifetime_${transactionId}` }, + }); + expect(subscription?.planType).toBe(PlanType.SCIWEAVE_LIFETIME); + expect(subscription?.status).toBe(SubscriptionStatus.ACTIVE); + + const fulfillment = await prisma.revenueCatPurchaseFulfillment.findUnique({ + where: { revenueCatTransactionId: transactionId }, + }); + expect(fulfillment?.fulfillmentType).toBe(RevenueCatPurchaseFulfillmentType.LIFETIME_UNLOCK); + }); +}); + +function makePayload({ + userId, + type, + productId, + transactionId, + price, + cancelReason, +}: { + userId: number; + type: string; + productId: string; + transactionId: string; + price?: number; + cancelReason?: string; +}): RevenueCatWebhookPayload { + const now = Date.now(); + + return { + api_version: '1.0', + event: { + app_user_id: String(userId), + cancel_reason: cancelReason ?? null, + currency: 'USD', + event_timestamp_ms: now, + expiration_at_ms: now, + id: `${transactionId}_${type.toLowerCase()}`, + original_transaction_id: transactionId, + presented_offering_id: 'chat_bundles', + price: price ?? null, + product_id: productId, + store: 'PLAY_STORE', + subscriber_attributes: { + userId: { + updated_at_ms: now, + value: String(userId), + }, + }, + transaction_id: transactionId, + type, + }, + }; +} diff --git a/desci-server/yarn.lock b/desci-server/yarn.lock index 42128e6b2..ae82b886e 100644 --- a/desci-server/yarn.lock +++ b/desci-server/yarn.lock @@ -3183,13 +3183,6 @@ dependencies: "@opentelemetry/api" "^1.0.0" -"@opentelemetry/api-logs@0.52.1": - version "0.52.1" - resolved "https://registry.yarnpkg.com/@opentelemetry/api-logs/-/api-logs-0.52.1.tgz#52906375da4d64c206b0c4cb8ffa209214654ecc" - integrity sha512-qnSqB2DQ9TPP96dl8cDubDvrUyWc0/sK81xHTK8eSUspzDM3bsewX903qclQFvVhgStjRWdC5bLb3kQqMkfV5A== - dependencies: - "@opentelemetry/api" "^1.0.0" - "@opentelemetry/api-logs@0.53.0": version "0.53.0" resolved "https://registry.yarnpkg.com/@opentelemetry/api-logs/-/api-logs-0.53.0.tgz#c478cbd8120ec2547b64edfa03a552cfe42170be" @@ -3197,6 +3190,20 @@ dependencies: "@opentelemetry/api" "^1.0.0" +"@opentelemetry/api-logs@0.57.1": + version "0.57.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/api-logs/-/api-logs-0.57.1.tgz#97ebd714f0b1fcdf896e85c465ae5c5b22747425" + integrity sha512-I4PHczeujhQAQv6ZBzqHYEUiggZL4IdSMixtVD3EYqbdrjujE7kRfI5QohjlPoJm8BvenoW5YaTMWRrbpot6tg== + dependencies: + "@opentelemetry/api" "^1.3.0" + +"@opentelemetry/api-logs@0.57.2": + version "0.57.2" + resolved "https://registry.yarnpkg.com/@opentelemetry/api-logs/-/api-logs-0.57.2.tgz#d4001b9aa3580367b40fe889f3540014f766cc87" + integrity sha512-uIX52NnTM0iBh84MShlpouI7UKqkZ7MrUszTmaypHBu4r7NofznSnQRfJ+uUeDtQDj6w8eFGg5KBLDAwAPz1+A== + dependencies: + "@opentelemetry/api" "^1.3.0" + "@opentelemetry/api@1.x", "@opentelemetry/api@^1.0.0", "@opentelemetry/api@^1.3.0", "@opentelemetry/api@^1.8", "@opentelemetry/api@^1.8.0", "@opentelemetry/api@^1.9.0": version "1.9.0" resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.9.0.tgz#d03eba68273dc0f7509e2a3d5cba21eae10379fe" @@ -3261,7 +3268,7 @@ resolved "https://registry.yarnpkg.com/@opentelemetry/context-async-hooks/-/context-async-hooks-1.8.0.tgz#6141ff59f7c07fac8eda3f9f208b6aaf06893471" integrity sha512-ueLmocbWDi1aoU4IPdOQyt4qz/Dx+NYyU4qoa3d683usbnkDLUXYXJFfKIMPFV2BbrI5qtnpTtzErCKewoM8aw== -"@opentelemetry/context-async-hooks@^1.25.1": +"@opentelemetry/context-async-hooks@^1.30.1": version "1.30.1" resolved "https://registry.yarnpkg.com/@opentelemetry/context-async-hooks/-/context-async-hooks-1.30.1.tgz#4f76280691a742597fd0bf682982126857622948" integrity sha512-s5vvxXPVdjqS3kTLKMeBMvop9hbWkwzBpu+mUO2M7sZtlkyDJGwFe33wRKnbaYDo8ExRVBIIdwIGrqpxHuKttA== @@ -3280,14 +3287,7 @@ dependencies: "@opentelemetry/semantic-conventions" "1.23.0" -"@opentelemetry/core@1.26.0": - version "1.26.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/core/-/core-1.26.0.tgz#7d84265aaa850ed0ca5813f97d831155be42b328" - integrity sha512-1iKxXXE8415Cdv0yjG3G6hQnB5eVEsJce3QaawX8SjDn0mAS0ZM8fAbZZJD4ajvhC15cePvosSCut404KrIIvQ== - dependencies: - "@opentelemetry/semantic-conventions" "1.27.0" - -"@opentelemetry/core@1.30.1", "@opentelemetry/core@^1.0.0", "@opentelemetry/core@^1.1.0", "@opentelemetry/core@^1.25.1", "@opentelemetry/core@^1.8.0": +"@opentelemetry/core@1.30.1", "@opentelemetry/core@^1.0.0", "@opentelemetry/core@^1.1.0", "@opentelemetry/core@^1.26.0", "@opentelemetry/core@^1.30.1", "@opentelemetry/core@^1.8.0": version "1.30.1" resolved "https://registry.yarnpkg.com/@opentelemetry/core/-/core-1.30.1.tgz#a0b468bb396358df801881709ea38299fc30ab27" integrity sha512-OOCM2C/QIURhJMuKaekP3TRBxBKxG/TWWA0TL2J6nXUtDnuCtccy49LUJF8xPFXMX+0LMcxFpCo8M9cGY1W6rQ== @@ -3495,13 +3495,13 @@ "@opentelemetry/instrumentation" "^0.40.0" "@opentelemetry/semantic-conventions" "^1.0.0" -"@opentelemetry/instrumentation-amqplib@^0.42.0": - version "0.42.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-amqplib/-/instrumentation-amqplib-0.42.0.tgz#b3cab5a7207736a30d769962eed3af3838f986c4" - integrity sha512-fiuU6OKsqHJiydHWgTRQ7MnIrJ2lEqsdgFtNIH4LbAUJl/5XmrIeoDzDnox+hfkgWK65jsleFuQDtYb5hW1koQ== +"@opentelemetry/instrumentation-amqplib@^0.46.0": + version "0.46.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-amqplib/-/instrumentation-amqplib-0.46.1.tgz#7101678488d0e942162ca85c9ac6e93e1f3e0008" + integrity sha512-AyXVnlCf/xV3K/rNumzKxZqsULyITJH6OVLiW6730JPRqWA7Zc9bvYoVNpN6iOpTU8CasH34SU/ksVJmObFibQ== dependencies: "@opentelemetry/core" "^1.8.0" - "@opentelemetry/instrumentation" "^0.53.0" + "@opentelemetry/instrumentation" "^0.57.1" "@opentelemetry/semantic-conventions" "^1.27.0" "@opentelemetry/instrumentation-aws-lambda@^0.35.3": @@ -3541,13 +3541,13 @@ "@opentelemetry/instrumentation" "^0.40.0" "@opentelemetry/semantic-conventions" "^1.0.0" -"@opentelemetry/instrumentation-connect@0.39.0": - version "0.39.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-connect/-/instrumentation-connect-0.39.0.tgz#32bdbaac464cba061c95df6c850ee81efdd86f8b" - integrity sha512-pGBiKevLq7NNglMgqzmeKczF4XQMTOUOTkK8afRHMZMnrK3fcETyTH7lVaSozwiOM3Ws+SuEmXZT7DYrrhxGlg== +"@opentelemetry/instrumentation-connect@0.43.0": + version "0.43.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-connect/-/instrumentation-connect-0.43.0.tgz#411035f4a8f2e498dbfa7300e545c58586a062e2" + integrity sha512-Q57JGpH6T4dkYHo9tKXONgLtxzsh1ZEW5M9A/OwKrZFyEpLqWgjhcZ3hIuVvDlhb426iDF1f9FPToV/mi5rpeA== dependencies: "@opentelemetry/core" "^1.8.0" - "@opentelemetry/instrumentation" "^0.53.0" + "@opentelemetry/instrumentation" "^0.57.0" "@opentelemetry/semantic-conventions" "^1.27.0" "@types/connect" "3.4.36" @@ -3561,12 +3561,12 @@ "@opentelemetry/semantic-conventions" "^1.0.0" "@types/connect" "3.4.35" -"@opentelemetry/instrumentation-dataloader@0.12.0": - version "0.12.0" - resolved "https://registry.npmjs.org/@opentelemetry/instrumentation-dataloader/-/instrumentation-dataloader-0.12.0.tgz" - integrity sha512-pnPxatoFE0OXIZDQhL2okF//dmbiWFzcSc8pUg9TqofCLYZySSxDCgQc69CJBo5JnI3Gz1KP+mOjS4WAeRIH4g== +"@opentelemetry/instrumentation-dataloader@0.16.0": + version "0.16.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-dataloader/-/instrumentation-dataloader-0.16.0.tgz#913345c335f67bf8e17a9b38c227dba741fe488b" + integrity sha512-88+qCHZC02up8PwKHk0UQKLLqGGURzS3hFQBZC7PnGwReuoKjHXS1o29H58S+QkXJpkTr2GACbx8j6mUoGjNPA== dependencies: - "@opentelemetry/instrumentation" "^0.53.0" + "@opentelemetry/instrumentation" "^0.57.0" "@opentelemetry/instrumentation-dataloader@^0.4.3": version "0.4.3" @@ -3584,13 +3584,13 @@ "@opentelemetry/semantic-conventions" "^1.0.0" semver "^7.3.2" -"@opentelemetry/instrumentation-express@0.42.0": - version "0.42.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-express/-/instrumentation-express-0.42.0.tgz#279f195aa66baee2b98623a16666c6229c8e7564" - integrity sha512-YNcy7ZfGnLsVEqGXQPT+S0G1AE46N21ORY7i7yUQyfhGAL4RBjnZUqefMI0NwqIl6nGbr1IpF0rZGoN8Q7x12Q== +"@opentelemetry/instrumentation-express@0.47.0": + version "0.47.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-express/-/instrumentation-express-0.47.0.tgz#f0477db3b1f4b342beb9ecd08edc26c470566724" + integrity sha512-XFWVx6k0XlU8lu6cBlCa29ONtVt6ADEjmxtyAyeF2+rifk8uBJbk1La0yIVfI0DoKURGbaEDTNelaXG9l/lNNQ== dependencies: "@opentelemetry/core" "^1.8.0" - "@opentelemetry/instrumentation" "^0.53.0" + "@opentelemetry/instrumentation" "^0.57.0" "@opentelemetry/semantic-conventions" "^1.27.0" "@opentelemetry/instrumentation-express@^0.32.4": @@ -3603,13 +3603,13 @@ "@opentelemetry/semantic-conventions" "^1.0.0" "@types/express" "4.17.13" -"@opentelemetry/instrumentation-fastify@0.39.0": - version "0.39.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-fastify/-/instrumentation-fastify-0.39.0.tgz#96a040e4944daf77c53a8fe5a128bc3b2568e4aa" - integrity sha512-SS9uSlKcsWZabhBp2szErkeuuBDgxOUlllwkS92dVaWRnMmwysPhcEgHKB8rUe3BHg/GnZC1eo1hbTZv4YhfoA== +"@opentelemetry/instrumentation-fastify@0.44.1": + version "0.44.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-fastify/-/instrumentation-fastify-0.44.1.tgz#c8080f24a6fbdd14689c619ad7b14fe189b10f28" + integrity sha512-RoVeMGKcNttNfXMSl6W4fsYoCAYP1vi6ZAWIGhBY+o7R9Y0afA7f9JJL0j8LHbyb0P0QhSYk+6O56OwI2k4iRQ== dependencies: "@opentelemetry/core" "^1.8.0" - "@opentelemetry/instrumentation" "^0.53.0" + "@opentelemetry/instrumentation" "^0.57.0" "@opentelemetry/semantic-conventions" "^1.27.0" "@opentelemetry/instrumentation-fastify@^0.31.4": @@ -3621,13 +3621,13 @@ "@opentelemetry/instrumentation" "^0.40.0" "@opentelemetry/semantic-conventions" "^1.0.0" -"@opentelemetry/instrumentation-fs@0.15.0": - version "0.15.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-fs/-/instrumentation-fs-0.15.0.tgz#41658507860f39fee5209bca961cea8d24ca2a83" - integrity sha512-JWVKdNLpu1skqZQA//jKOcKdJC66TWKqa2FUFq70rKohvaSq47pmXlnabNO+B/BvLfmidfiaN35XakT5RyMl2Q== +"@opentelemetry/instrumentation-fs@0.19.0": + version "0.19.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-fs/-/instrumentation-fs-0.19.0.tgz#a44807aea97edc64c597d6a5b5b8637b7ab45057" + integrity sha512-JGwmHhBkRT2G/BYNV1aGI+bBjJu4fJUD/5/Jat0EWZa2ftrLV3YE8z84Fiij/wK32oMZ88eS8DI4ecLGZhpqsQ== dependencies: "@opentelemetry/core" "^1.8.0" - "@opentelemetry/instrumentation" "^0.53.0" + "@opentelemetry/instrumentation" "^0.57.0" "@opentelemetry/instrumentation-fs@^0.7.4": version "0.7.4" @@ -3638,12 +3638,12 @@ "@opentelemetry/instrumentation" "^0.40.0" "@opentelemetry/semantic-conventions" "^1.0.0" -"@opentelemetry/instrumentation-generic-pool@0.39.0": - version "0.39.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-generic-pool/-/instrumentation-generic-pool-0.39.0.tgz#2b9af16ad82d5cbe67125c0125753cecd162a728" - integrity sha512-y4v8Y+tSfRB3NNBvHjbjrn7rX/7sdARG7FuK6zR8PGb28CTa0kHpEGCJqvL9L8xkTNvTXo+lM36ajFGUaK1aNw== +"@opentelemetry/instrumentation-generic-pool@0.43.0": + version "0.43.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-generic-pool/-/instrumentation-generic-pool-0.43.0.tgz#b1769eb0e30f2abb764a9cbc811aa3d4560ecc24" + integrity sha512-at8GceTtNxD1NfFKGAuwtqM41ot/TpcLh+YsGe4dhf7gvv1HW/ZWdq6nfRtS6UjIvZJOokViqLPJ3GVtZItAnQ== dependencies: - "@opentelemetry/instrumentation" "^0.53.0" + "@opentelemetry/instrumentation" "^0.57.0" "@opentelemetry/instrumentation-generic-pool@^0.31.4": version "0.31.4" @@ -3654,12 +3654,12 @@ "@opentelemetry/semantic-conventions" "^1.0.0" "@types/generic-pool" "^3.1.9" -"@opentelemetry/instrumentation-graphql@0.43.0": - version "0.43.0" - resolved "https://registry.npmjs.org/@opentelemetry/instrumentation-graphql/-/instrumentation-graphql-0.43.0.tgz" - integrity sha512-aI3YMmC2McGd8KW5du1a2gBA0iOMOGLqg4s9YjzwbjFwjlmMNFSK1P3AIg374GWg823RPUGfVTIgZ/juk9CVOA== +"@opentelemetry/instrumentation-graphql@0.47.0": + version "0.47.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-graphql/-/instrumentation-graphql-0.47.0.tgz#271807e21a6224bd1986a3e9887650f1858ee733" + integrity sha512-Cc8SMf+nLqp0fi8oAnooNEfwZWFnzMiBHCGmDFYqmgjPylyLmi83b+NiTns/rKGwlErpW0AGPt0sMpkbNlzn8w== dependencies: - "@opentelemetry/instrumentation" "^0.53.0" + "@opentelemetry/instrumentation" "^0.57.0" "@opentelemetry/instrumentation-graphql@^0.34.3": version "0.34.3" @@ -3676,13 +3676,13 @@ "@opentelemetry/instrumentation" "0.40.0" "@opentelemetry/semantic-conventions" "1.14.0" -"@opentelemetry/instrumentation-hapi@0.41.0": - version "0.41.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-hapi/-/instrumentation-hapi-0.41.0.tgz#de8711907256d8fae1b5faf71fc825cef4a7ddbb" - integrity sha512-jKDrxPNXDByPlYcMdZjNPYCvw0SQJjN+B1A+QH+sx+sAHsKSAf9hwFiJSrI6C4XdOls43V/f/fkp9ITkHhKFbQ== +"@opentelemetry/instrumentation-hapi@0.45.1": + version "0.45.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-hapi/-/instrumentation-hapi-0.45.1.tgz#5edf982549070d95e20152d568279548ad44d662" + integrity sha512-VH6mU3YqAKTePPfUPwfq4/xr049774qWtfTuJqVHoVspCLiT3bW+fCQ1toZxt6cxRPYASoYaBsMA3CWo8B8rcw== dependencies: "@opentelemetry/core" "^1.8.0" - "@opentelemetry/instrumentation" "^0.53.0" + "@opentelemetry/instrumentation" "^0.57.0" "@opentelemetry/semantic-conventions" "^1.27.0" "@opentelemetry/instrumentation-hapi@^0.31.4": @@ -3695,14 +3695,15 @@ "@opentelemetry/semantic-conventions" "^1.0.0" "@types/hapi__hapi" "20.0.9" -"@opentelemetry/instrumentation-http@0.53.0": - version "0.53.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-http/-/instrumentation-http-0.53.0.tgz#0d806adf1b3aba036bc46e16162e3c0dbb8a6b60" - integrity sha512-H74ErMeDuZfj7KgYCTOFGWF5W9AfaPnqLQQxeFq85+D29wwV2yqHbz2IKLYpkOh7EI6QwDEl7rZCIxjJLyc/CQ== +"@opentelemetry/instrumentation-http@0.57.1": + version "0.57.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-http/-/instrumentation-http-0.57.1.tgz#2d8b395df62191475e76fa0eb7bf60079ea886b9" + integrity sha512-ThLmzAQDs7b/tdKI3BV2+yawuF09jF111OFsovqT1Qj3D8vjwKBwhi/rDE5xethwn4tSXtZcJ9hBsVAlWFQZ7g== dependencies: - "@opentelemetry/core" "1.26.0" - "@opentelemetry/instrumentation" "0.53.0" - "@opentelemetry/semantic-conventions" "1.27.0" + "@opentelemetry/core" "1.30.1" + "@opentelemetry/instrumentation" "0.57.1" + "@opentelemetry/semantic-conventions" "1.28.0" + forwarded-parse "2.1.2" semver "^7.5.2" "@opentelemetry/instrumentation-http@^0.40.0": @@ -3715,12 +3716,12 @@ "@opentelemetry/semantic-conventions" "1.14.0" semver "^7.3.5" -"@opentelemetry/instrumentation-ioredis@0.43.0": - version "0.43.0" - resolved "https://registry.npmjs.org/@opentelemetry/instrumentation-ioredis/-/instrumentation-ioredis-0.43.0.tgz" - integrity sha512-i3Dke/LdhZbiUAEImmRG3i7Dimm/BD7t8pDDzwepSvIQ6s2X6FPia7561gw+64w+nx0+G9X14D7rEfaMEmmjig== +"@opentelemetry/instrumentation-ioredis@0.47.0": + version "0.47.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-ioredis/-/instrumentation-ioredis-0.47.0.tgz#f83bd133d36d137d2d0b58bfbdfe12ed6fe5ab2f" + integrity sha512-4HqP9IBC8e7pW9p90P3q4ox0XlbLGme65YTrA3UTLvqvo4Z6b0puqZQP203YFu8m9rE/luLfaG7/xrwwqMUpJw== dependencies: - "@opentelemetry/instrumentation" "^0.53.0" + "@opentelemetry/instrumentation" "^0.57.0" "@opentelemetry/redis-common" "^0.36.2" "@opentelemetry/semantic-conventions" "^1.27.0" @@ -3734,12 +3735,20 @@ "@opentelemetry/semantic-conventions" "^1.0.0" "@types/ioredis4" "npm:@types/ioredis@^4.28.10" -"@opentelemetry/instrumentation-kafkajs@0.3.0": - version "0.3.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-kafkajs/-/instrumentation-kafkajs-0.3.0.tgz#6687bce4dac8b90ef8ccbf1b662d5d1e95a34414" - integrity sha512-UnkZueYK1ise8FXQeKlpBd7YYUtC7mM8J0wzUSccEfc/G8UqHQqAzIyYCUOUPUKp8GsjLnWOOK/3hJc4owb7Jg== +"@opentelemetry/instrumentation-kafkajs@0.7.0": + version "0.7.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-kafkajs/-/instrumentation-kafkajs-0.7.0.tgz#079b949ec814b42e49d23bb4d4f73735fe460d52" + integrity sha512-LB+3xiNzc034zHfCtgs4ITWhq6Xvdo8bsq7amR058jZlf2aXXDrN9SV4si4z2ya9QX4tz6r4eZJwDkXOp14/AQ== dependencies: - "@opentelemetry/instrumentation" "^0.53.0" + "@opentelemetry/instrumentation" "^0.57.0" + "@opentelemetry/semantic-conventions" "^1.27.0" + +"@opentelemetry/instrumentation-knex@0.44.0": + version "0.44.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-knex/-/instrumentation-knex-0.44.0.tgz#af251ed38f06a2f248812c5addf0266697b6149a" + integrity sha512-SlT0+bLA0Lg3VthGje+bSZatlGHw/vwgQywx0R/5u9QC59FddTQSPJeWNw29M6f8ScORMeUOOTwihlQAn4GkJQ== + dependencies: + "@opentelemetry/instrumentation" "^0.57.0" "@opentelemetry/semantic-conventions" "^1.27.0" "@opentelemetry/instrumentation-knex@^0.31.4": @@ -3750,13 +3759,13 @@ "@opentelemetry/instrumentation" "^0.40.0" "@opentelemetry/semantic-conventions" "^1.0.0" -"@opentelemetry/instrumentation-koa@0.43.0": - version "0.43.0" - resolved "https://registry.npmjs.org/@opentelemetry/instrumentation-koa/-/instrumentation-koa-0.43.0.tgz" - integrity sha512-lDAhSnmoTIN6ELKmLJBplXzT/Jqs5jGZehuG22EdSMaTwgjMpxMDI1YtlKEhiWPWkrz5LUsd0aOO0ZRc9vn3AQ== +"@opentelemetry/instrumentation-koa@0.47.0": + version "0.47.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-koa/-/instrumentation-koa-0.47.0.tgz#a74b35809ba95d0f9db49e8c3f214bde475b095a" + integrity sha512-HFdvqf2+w8sWOuwtEXayGzdZ2vWpCKEQv5F7+2DSA74Te/Cv4rvb2E5So5/lh+ok4/RAIPuvCbCb/SHQFzMmbw== dependencies: "@opentelemetry/core" "^1.8.0" - "@opentelemetry/instrumentation" "^0.53.0" + "@opentelemetry/instrumentation" "^0.57.0" "@opentelemetry/semantic-conventions" "^1.27.0" "@opentelemetry/instrumentation-koa@^0.34.6": @@ -3770,6 +3779,13 @@ "@types/koa" "2.13.6" "@types/koa__router" "8.0.7" +"@opentelemetry/instrumentation-lru-memoizer@0.44.0": + version "0.44.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-lru-memoizer/-/instrumentation-lru-memoizer-0.44.0.tgz#c22e770d950c165f80c657a9c790c9843baaa65b" + integrity sha512-Tn7emHAlvYDFik3vGU0mdwvWJDwtITtkJ+5eT2cUquct6nIs+H8M47sqMJkCpyPe5QIBJoTOHxmc6mj9lz6zDw== + dependencies: + "@opentelemetry/instrumentation" "^0.57.0" + "@opentelemetry/instrumentation-lru-memoizer@^0.32.4": version "0.32.4" resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-lru-memoizer/-/instrumentation-lru-memoizer-0.32.4.tgz#d5bee9d54117064ad535529875d7a55f2cc5f09c" @@ -3786,13 +3802,12 @@ "@opentelemetry/semantic-conventions" "^1.0.0" "@types/memcached" "^2.2.6" -"@opentelemetry/instrumentation-mongodb@0.47.0": - version "0.47.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-mongodb/-/instrumentation-mongodb-0.47.0.tgz#f8107d878281433905e717f223fb4c0f10356a7b" - integrity sha512-yqyXRx2SulEURjgOQyJzhCECSh5i1uM49NUaq9TqLd6fA7g26OahyJfsr9NE38HFqGRHpi4loyrnfYGdrsoVjQ== +"@opentelemetry/instrumentation-mongodb@0.51.0": + version "0.51.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-mongodb/-/instrumentation-mongodb-0.51.0.tgz#8a323c2fb4cb2c93bf95f1b1c0fcb30952d12a08" + integrity sha512-cMKASxCX4aFxesoj3WK8uoQ0YUrRvnfxaO72QWI2xLu5ZtgX/QvdGBlU3Ehdond5eb74c2s1cqRQUIptBnKz1g== dependencies: - "@opentelemetry/instrumentation" "^0.53.0" - "@opentelemetry/sdk-metrics" "^1.9.1" + "@opentelemetry/instrumentation" "^0.57.0" "@opentelemetry/semantic-conventions" "^1.27.0" "@opentelemetry/instrumentation-mongodb@^0.35.0": @@ -3804,13 +3819,13 @@ "@opentelemetry/sdk-metrics" "^1.9.1" "@opentelemetry/semantic-conventions" "^1.0.0" -"@opentelemetry/instrumentation-mongoose@0.42.0": - version "0.42.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-mongoose/-/instrumentation-mongoose-0.42.0.tgz#375afd21adfcd897a8f521c1ffd2d91e6a428705" - integrity sha512-AnWv+RaR86uG3qNEMwt3plKX1ueRM7AspfszJYVkvkehiicC3bHQA6vWdb6Zvy5HAE14RyFbu9+2hUUjR2NSyg== +"@opentelemetry/instrumentation-mongoose@0.46.0": + version "0.46.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-mongoose/-/instrumentation-mongoose-0.46.0.tgz#c3a5f69e1a5b950b542cf84650fbbd3e31bd681e" + integrity sha512-mtVv6UeaaSaWTeZtLo4cx4P5/ING2obSqfWGItIFSunQBrYROfhuVe7wdIrFUs2RH1tn2YYpAJyMaRe/bnTTIQ== dependencies: "@opentelemetry/core" "^1.8.0" - "@opentelemetry/instrumentation" "^0.53.0" + "@opentelemetry/instrumentation" "^0.57.0" "@opentelemetry/semantic-conventions" "^1.27.0" "@opentelemetry/instrumentation-mongoose@^0.32.4": @@ -3822,12 +3837,12 @@ "@opentelemetry/instrumentation" "^0.40.0" "@opentelemetry/semantic-conventions" "^1.0.0" -"@opentelemetry/instrumentation-mysql2@0.41.0": - version "0.41.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-mysql2/-/instrumentation-mysql2-0.41.0.tgz#6377b6e2d2487fd88e1d79aa03658db6c8d51651" - integrity sha512-REQB0x+IzVTpoNgVmy5b+UnH1/mDByrneimP6sbDHkp1j8QOl1HyWOrBH/6YWR0nrbU3l825Em5PlybjT3232g== +"@opentelemetry/instrumentation-mysql2@0.45.0": + version "0.45.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-mysql2/-/instrumentation-mysql2-0.45.0.tgz#95501759d470dbc7038670e91205e8ed601ec402" + integrity sha512-qLslv/EPuLj0IXFvcE3b0EqhWI8LKmrgRPIa4gUd8DllbBpqJAvLNJSv3cC6vWwovpbSI3bagNO/3Q2SuXv2xA== dependencies: - "@opentelemetry/instrumentation" "^0.53.0" + "@opentelemetry/instrumentation" "^0.57.0" "@opentelemetry/semantic-conventions" "^1.27.0" "@opentelemetry/sql-common" "^0.40.1" @@ -3839,12 +3854,12 @@ "@opentelemetry/instrumentation" "^0.40.0" "@opentelemetry/semantic-conventions" "^1.0.0" -"@opentelemetry/instrumentation-mysql@0.41.0": - version "0.41.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-mysql/-/instrumentation-mysql-0.41.0.tgz#2d50691ead5219774bd36d66c35d5b4681485dd7" - integrity sha512-jnvrV6BsQWyHS2qb2fkfbfSb1R/lmYwqEZITwufuRl37apTopswu9izc0b1CYRp/34tUG/4k/V39PND6eyiNvw== +"@opentelemetry/instrumentation-mysql@0.45.0": + version "0.45.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-mysql/-/instrumentation-mysql-0.45.0.tgz#e4df8bc709c0c8b0ff90bbef92fb36e92ebe0d19" + integrity sha512-tWWyymgwYcTwZ4t8/rLDfPYbOTF3oYB8SxnYMtIQ1zEf5uDm90Ku3i6U/vhaMyfHNlIHvDhvJh+qx5Nc4Z3Acg== dependencies: - "@opentelemetry/instrumentation" "^0.53.0" + "@opentelemetry/instrumentation" "^0.57.0" "@opentelemetry/semantic-conventions" "^1.27.0" "@types/mysql" "2.15.26" @@ -3857,12 +3872,12 @@ "@opentelemetry/semantic-conventions" "^1.0.0" "@types/mysql" "2.15.19" -"@opentelemetry/instrumentation-nestjs-core@0.40.0": - version "0.40.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-nestjs-core/-/instrumentation-nestjs-core-0.40.0.tgz#2c0e6405b56caaec32747d55c57ff9a034668ea8" - integrity sha512-WF1hCUed07vKmf5BzEkL0wSPinqJgH7kGzOjjMAiTGacofNXjb/y4KQ8loj2sNsh5C/NN7s1zxQuCgbWbVTGKg== +"@opentelemetry/instrumentation-nestjs-core@0.44.0": + version "0.44.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-nestjs-core/-/instrumentation-nestjs-core-0.44.0.tgz#d2a3631de3bed2b1c0a03afa79c08ae22bef8b6c" + integrity sha512-t16pQ7A4WYu1yyQJZhRKIfUNvl5PAaF2pEteLvgJb/BWdd1oNuU1rOYt4S825kMy+0q4ngiX281Ss9qiwHfxFQ== dependencies: - "@opentelemetry/instrumentation" "^0.53.0" + "@opentelemetry/instrumentation" "^0.57.0" "@opentelemetry/semantic-conventions" "^1.27.0" "@opentelemetry/instrumentation-nestjs-core@^0.32.5": @@ -3881,13 +3896,14 @@ "@opentelemetry/instrumentation" "^0.40.0" "@opentelemetry/semantic-conventions" "^1.0.0" -"@opentelemetry/instrumentation-pg@0.44.0": - version "0.44.0" - resolved "https://registry.npmjs.org/@opentelemetry/instrumentation-pg/-/instrumentation-pg-0.44.0.tgz" - integrity sha512-oTWVyzKqXud1BYEGX1loo2o4k4vaU1elr3vPO8NZolrBtFvQ34nx4HgUaexUDuEog00qQt+MLR5gws/p+JXMLQ== +"@opentelemetry/instrumentation-pg@0.50.0": + version "0.50.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-pg/-/instrumentation-pg-0.50.0.tgz#525ecf683c349529539a14f2be47164f4e3eb0f5" + integrity sha512-TtLxDdYZmBhFswm8UIsrDjh/HFBeDXd4BLmE8h2MxirNHewLJ0VS9UUddKKEverb5Sm2qFVjqRjcU+8Iw4FJ3w== dependencies: - "@opentelemetry/instrumentation" "^0.53.0" - "@opentelemetry/semantic-conventions" "^1.27.0" + "@opentelemetry/core" "^1.26.0" + "@opentelemetry/instrumentation" "^0.57.0" + "@opentelemetry/semantic-conventions" "1.27.0" "@opentelemetry/sql-common" "^0.40.1" "@types/pg" "8.6.1" "@types/pg-pool" "2.0.6" @@ -3910,12 +3926,12 @@ dependencies: "@opentelemetry/instrumentation" "^0.40.0" -"@opentelemetry/instrumentation-redis-4@0.42.0": - version "0.42.0" - resolved "https://registry.npmjs.org/@opentelemetry/instrumentation-redis-4/-/instrumentation-redis-4-0.42.0.tgz" - integrity sha512-NaD+t2JNcOzX/Qa7kMy68JbmoVIV37fT/fJYzLKu2Wwd+0NCxt+K2OOsOakA8GVg8lSpFdbx4V/suzZZ2Pvdjg== +"@opentelemetry/instrumentation-redis-4@0.46.0": + version "0.46.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-redis-4/-/instrumentation-redis-4-0.46.0.tgz#828704b8134f023730ac508bcf3a38ca4d5d697c" + integrity sha512-aTUWbzbFMFeRODn3720TZO0tsh/49T8H3h8vVnVKJ+yE36AeW38Uj/8zykQ/9nO8Vrtjr5yKuX3uMiG/W8FKNw== dependencies: - "@opentelemetry/instrumentation" "^0.53.0" + "@opentelemetry/instrumentation" "^0.57.0" "@opentelemetry/redis-common" "^0.36.2" "@opentelemetry/semantic-conventions" "^1.27.0" @@ -3962,6 +3978,15 @@ "@opentelemetry/instrumentation" "^0.40.0" "@opentelemetry/semantic-conventions" "^1.0.0" +"@opentelemetry/instrumentation-tedious@0.18.0": + version "0.18.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-tedious/-/instrumentation-tedious-0.18.0.tgz#636745423db28e303b4e0289b8f69685cb36f807" + integrity sha512-9zhjDpUDOtD+coeADnYEJQ0IeLVCj7w/hqzIutdp5NqS1VqTAanaEfsEcSypyvYv5DX3YOsTUoF+nr2wDXPETA== + dependencies: + "@opentelemetry/instrumentation" "^0.57.0" + "@opentelemetry/semantic-conventions" "^1.27.0" + "@types/tedious" "^4.0.14" + "@opentelemetry/instrumentation-tedious@^0.5.4": version "0.5.4" resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-tedious/-/instrumentation-tedious-0.5.4.tgz#75d23732196bc6f20aa0ae71f8615ecc3da66287" @@ -3971,13 +3996,13 @@ "@opentelemetry/semantic-conventions" "^1.0.0" "@types/tedious" "^4.0.6" -"@opentelemetry/instrumentation-undici@0.6.0": - version "0.6.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-undici/-/instrumentation-undici-0.6.0.tgz#9436ee155c8dcb0b760b66947c0e0f347688a5ef" - integrity sha512-ABJBhm5OdhGmbh0S/fOTE4N69IZ00CsHC5ijMYfzbw3E5NwLgpQk5xsljaECrJ8wz1SfXbO03FiSuu5AyRAkvQ== +"@opentelemetry/instrumentation-undici@0.10.0": + version "0.10.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-undici/-/instrumentation-undici-0.10.0.tgz#99cba213a6e9d47a82896b6c782c3f2d60e0edb5" + integrity sha512-vm+V255NGw9gaSsPD6CP0oGo8L55BffBc8KnxqsMuc6XiAD1L8SFNzsW0RHhxJFqy9CJaJh+YiJ5EHXuZ5rZBw== dependencies: "@opentelemetry/core" "^1.8.0" - "@opentelemetry/instrumentation" "^0.53.0" + "@opentelemetry/instrumentation" "^0.57.0" "@opentelemetry/instrumentation-winston@^0.31.4": version "0.31.4" @@ -4006,36 +4031,37 @@ semver "^7.3.2" shimmer "^1.2.1" -"@opentelemetry/instrumentation@0.53.0", "@opentelemetry/instrumentation@^0.53.0": - version "0.53.0" - resolved "https://registry.npmjs.org/@opentelemetry/instrumentation/-/instrumentation-0.53.0.tgz" - integrity sha512-DMwg0hy4wzf7K73JJtl95m/e0boSoWhH07rfvHvYzQtBD3Bmv0Wc1x733vyZBqmFm8OjJD0/pfiUg1W3JjFX0A== +"@opentelemetry/instrumentation@0.57.1": + version "0.57.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation/-/instrumentation-0.57.1.tgz#5aea772be8783a35d69d643da46582f381ba1810" + integrity sha512-SgHEKXoVxOjc20ZYusPG3Fh+RLIZTSa4x8QtD3NfgAUDyqdFFS9W1F2ZVbZkqDCdyMcQG02Ok4duUGLHJXHgbA== dependencies: - "@opentelemetry/api-logs" "0.53.0" + "@opentelemetry/api-logs" "0.57.1" "@types/shimmer" "^1.2.0" import-in-the-middle "^1.8.1" require-in-the-middle "^7.1.1" semver "^7.5.2" shimmer "^1.2.1" -"@opentelemetry/instrumentation@^0.46.0": - version "0.46.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation/-/instrumentation-0.46.0.tgz#a8a252306f82e2eace489312798592a14eb9830e" - integrity sha512-a9TijXZZbk0vI5TGLZl+0kxyFfrXHhX6Svtz7Pp2/VBlCSKrazuULEyoJQrOknJyFWNMEmbbJgOciHCCpQcisw== +"@opentelemetry/instrumentation@^0.49 || ^0.50 || ^0.51 || ^0.52.0 || ^0.53.0": + version "0.53.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation/-/instrumentation-0.53.0.tgz#e6369e4015eb5112468a4d45d38dcada7dad892d" + integrity sha512-DMwg0hy4wzf7K73JJtl95m/e0boSoWhH07rfvHvYzQtBD3Bmv0Wc1x733vyZBqmFm8OjJD0/pfiUg1W3JjFX0A== dependencies: - "@types/shimmer" "^1.0.2" - import-in-the-middle "1.7.1" + "@opentelemetry/api-logs" "0.53.0" + "@types/shimmer" "^1.2.0" + import-in-the-middle "^1.8.1" require-in-the-middle "^7.1.1" semver "^7.5.2" shimmer "^1.2.1" -"@opentelemetry/instrumentation@^0.49 || ^0.50 || ^0.51 || ^0.52.0": - version "0.52.1" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation/-/instrumentation-0.52.1.tgz#2e7e46a38bd7afbf03cf688c862b0b43418b7f48" - integrity sha512-uXJbYU/5/MBHjMp1FqrILLRuiJCs3Ofk0MeRDk8g1S1gD47U8X3JnSwcMO1rtRo1x1a7zKaQHaoYu49p/4eSKw== +"@opentelemetry/instrumentation@^0.52.0 || ^0.53.0 || ^0.54.0 || ^0.55.0 || ^0.56.0 || ^0.57.0", "@opentelemetry/instrumentation@^0.57.0", "@opentelemetry/instrumentation@^0.57.1": + version "0.57.2" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation/-/instrumentation-0.57.2.tgz#8924549d7941ba1b5c6f04d5529cf48330456d1d" + integrity sha512-BdBGhQBh8IjZ2oIIX6F2/Q3LKm/FDDKi6ccYKcBTeilh6SNdNKveDOLk73BkSJjQLJk6qe4Yh+hHw1UPhCDdrg== dependencies: - "@opentelemetry/api-logs" "0.52.1" - "@types/shimmer" "^1.0.2" + "@opentelemetry/api-logs" "0.57.2" + "@types/shimmer" "^1.2.0" import-in-the-middle "^1.8.1" require-in-the-middle "^7.1.1" semver "^7.5.2" @@ -4233,7 +4259,7 @@ "@opentelemetry/core" "1.23.0" "@opentelemetry/semantic-conventions" "1.23.0" -"@opentelemetry/resources@1.30.1", "@opentelemetry/resources@^1.0.0", "@opentelemetry/resources@^1.10.0", "@opentelemetry/resources@^1.12.0", "@opentelemetry/resources@^1.23.0", "@opentelemetry/resources@^1.25.1", "@opentelemetry/resources@^1.26.0", "@opentelemetry/resources@^1.8.0": +"@opentelemetry/resources@1.30.1", "@opentelemetry/resources@^1.0.0", "@opentelemetry/resources@^1.10.0", "@opentelemetry/resources@^1.12.0", "@opentelemetry/resources@^1.23.0", "@opentelemetry/resources@^1.30.1", "@opentelemetry/resources@^1.8.0": version "1.30.1" resolved "https://registry.yarnpkg.com/@opentelemetry/resources/-/resources-1.30.1.tgz#a4eae17ebd96947fdc7a64f931ca4b71e18ce964" integrity sha512-5UxZqiAgLYGFjS4s9qm5mBVo433u+dSPUFWVWXmLAD4wB65oMCoXaJP1KJa9DIYYMeHu3z4BZcStG3LC593cWA== @@ -4363,7 +4389,7 @@ "@opentelemetry/resources" "1.8.0" "@opentelemetry/semantic-conventions" "1.8.0" -"@opentelemetry/sdk-trace-base@^1.22", "@opentelemetry/sdk-trace-base@^1.23.0", "@opentelemetry/sdk-trace-base@^1.25.1", "@opentelemetry/sdk-trace-base@^1.26.0", "@opentelemetry/sdk-trace-base@^1.8.0": +"@opentelemetry/sdk-trace-base@^1.22", "@opentelemetry/sdk-trace-base@^1.23.0", "@opentelemetry/sdk-trace-base@^1.30.1", "@opentelemetry/sdk-trace-base@^1.8.0": version "1.30.1" resolved "https://registry.yarnpkg.com/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.30.1.tgz#41a42234096dc98e8f454d24551fc80b816feb34" integrity sha512-jVPgBbH1gCy2Lb7X0AVQ8XAfgg0pJ4nvl8/IiQA6nxOsPvS+0zMJaFSs2ltXe0J6C8dqjcnpyqINDJmU30+uOg== @@ -4421,11 +4447,16 @@ resolved "https://registry.yarnpkg.com/@opentelemetry/semantic-conventions/-/semantic-conventions-1.8.0.tgz#fe2aa90e6df050a11cd57f5c0f47b0641fd2cad3" integrity sha512-TYh1MRcm4JnvpqtqOwT9WYaBYY4KERHdToxs/suDTLviGRsQkIjS5yYROTYTSJQUnYLOn/TuOh5GoMwfLSU+Ew== -"@opentelemetry/semantic-conventions@^1.0.0", "@opentelemetry/semantic-conventions@^1.17.0", "@opentelemetry/semantic-conventions@^1.23.0", "@opentelemetry/semantic-conventions@^1.25.1", "@opentelemetry/semantic-conventions@^1.27.0", "@opentelemetry/semantic-conventions@^1.29.0": +"@opentelemetry/semantic-conventions@^1.0.0", "@opentelemetry/semantic-conventions@^1.23.0", "@opentelemetry/semantic-conventions@^1.27.0", "@opentelemetry/semantic-conventions@^1.29.0": version "1.37.0" resolved "https://registry.yarnpkg.com/@opentelemetry/semantic-conventions/-/semantic-conventions-1.37.0.tgz#aa2b4fa0b910b66a050c5ddfcac1d262e91a321a" integrity sha512-JD6DerIKdJGmRp4jQyX5FlrQjA4tjOw1cvfsPAZXfOOEErMUHjPcPSICS+6WnM0nB0efSFARh0KAZss+bvExOA== +"@opentelemetry/semantic-conventions@^1.28.0": + version "1.40.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/semantic-conventions/-/semantic-conventions-1.40.0.tgz#10b2944ca559386590683392022a897eefd011d3" + integrity sha512-cifvXDhcqMwwTlTK04GBNeIe7yyo28Mfby85QXFe1Yk8nmi36Ab/5UQwptOx84SsoGNRg+EVSjwzfSZMy6pmlw== + "@opentelemetry/sql-common@^0.40.1": version "0.40.1" resolved "https://registry.yarnpkg.com/@opentelemetry/sql-common/-/sql-common-0.40.1.tgz#93fbc48d8017449f5b3c3274f2268a08af2b83b6" @@ -4560,32 +4591,73 @@ dependencies: cross-spawn "^7.0.6" -"@prisma/client@4.10.1": - version "4.10.1" - resolved "https://registry.yarnpkg.com/@prisma/client/-/client-4.10.1.tgz#c47fd54661ee74b174cee63e9dc418ecf57a6ccd" - integrity sha512-VonXLJZybdt8e5XZH5vnIGCRNnIh6OMX1FS3H/yzMGLT3STj5TJ/OkMcednrvELgk8PK89Vo3aSh51MWNO0axA== - dependencies: - "@prisma/engines-version" "4.10.1-2.aead147aa326ccb985dcfed5b065b4fdabd44b19" - -"@prisma/engines-version@4.10.1-2.aead147aa326ccb985dcfed5b065b4fdabd44b19": - version "4.10.1-2.aead147aa326ccb985dcfed5b065b4fdabd44b19" - resolved "https://registry.yarnpkg.com/@prisma/engines-version/-/engines-version-4.10.1-2.aead147aa326ccb985dcfed5b065b4fdabd44b19.tgz#312359d9d00e39e323136d0270876293d315658e" - integrity sha512-tsjTho7laDhf9EJ9EnDxAPEf7yrigSMDhniXeU4YoWc7azHAs4GPxRi2P9LTFonmHkJLMOLjR77J1oIP8Ife1w== - -"@prisma/engines@4.10.1": - version "4.10.1" - resolved "https://registry.yarnpkg.com/@prisma/engines/-/engines-4.10.1.tgz#c7062747f254e5d5fce98a8cae566c25f9f29fb2" - integrity sha512-B3tcTxjx196nuAu1GOTKO9cGPUgTFHYRdkPkTS4m5ptb2cejyBlH9X7GOfSt3xlI7p4zAJDshJP4JJivCg9ouA== - -"@prisma/instrumentation@5.19.1": - version "5.19.1" - resolved "https://registry.yarnpkg.com/@prisma/instrumentation/-/instrumentation-5.19.1.tgz#146319cf85f22b7a43296f0f40cfeac55516e66e" - integrity sha512-VLnzMQq7CWroL5AeaW0Py2huiNKeoMfCH3SUxstdzPrlWQi6UQ9UrfcbUkNHlVFqOMacqy8X/8YtE0kuKDpD9w== +"@prisma/client@6.15.0": + version "6.15.0" + resolved "https://registry.yarnpkg.com/@prisma/client/-/client-6.15.0.tgz#c4166aa8492878c842e63c515853d7a4125e6168" + integrity sha512-wR2LXUbOH4cL/WToatI/Y2c7uzni76oNFND7+23ypLllBmIS8e3ZHhO+nud9iXSXKFt1SoM3fTZvHawg63emZw== + +"@prisma/config@6.15.0": + version "6.15.0" + resolved "https://registry.yarnpkg.com/@prisma/config/-/config-6.15.0.tgz#b9a4fd87620baae3ce71015ee4ad632995838d89" + integrity sha512-KMEoec9b2u6zX0EbSEx/dRpx1oNLjqJEBZYyK0S3TTIbZ7GEGoVyGyFRk4C72+A38cuPLbfQGQvgOD+gBErKlA== + dependencies: + c12 "3.1.0" + deepmerge-ts "7.1.5" + effect "3.16.12" + empathic "2.0.0" + +"@prisma/debug@6.15.0": + version "6.15.0" + resolved "https://registry.yarnpkg.com/@prisma/debug/-/debug-6.15.0.tgz#8595f8ae91eadd20d926398449b8602c74e36bbf" + integrity sha512-y7cSeLuQmyt+A3hstAs6tsuAiVXSnw9T55ra77z0nbNkA8Lcq9rNcQg6PI00by/+WnE/aMRJ/W7sZWn2cgIy1g== + +"@prisma/engines-version@6.15.0-5.85179d7826409ee107a6ba334b5e305ae3fba9fb": + version "6.15.0-5.85179d7826409ee107a6ba334b5e305ae3fba9fb" + resolved "https://registry.yarnpkg.com/@prisma/engines-version/-/engines-version-6.15.0-5.85179d7826409ee107a6ba334b5e305ae3fba9fb.tgz#718b17890287447360107bef3675487d8cb0ae15" + integrity sha512-a/46aK5j6L3ePwilZYEgYDPrhBQ/n4gYjLxT5YncUTJJNRnTCVjPF86QdzUOLRdYjCLfhtZp9aum90W0J+trrg== + +"@prisma/engines@6.15.0": + version "6.15.0" + resolved "https://registry.yarnpkg.com/@prisma/engines/-/engines-6.15.0.tgz#77a220e40f4ed3345ef27407f8f0323e80debea8" + integrity sha512-opITiR5ddFJ1N2iqa7mkRlohCZqVSsHhRcc29QXeldMljOf4FSellLT0J5goVb64EzRTKcIDeIsJBgmilNcKxA== + dependencies: + "@prisma/debug" "6.15.0" + "@prisma/engines-version" "6.15.0-5.85179d7826409ee107a6ba334b5e305ae3fba9fb" + "@prisma/fetch-engine" "6.15.0" + "@prisma/get-platform" "6.15.0" + +"@prisma/fetch-engine@6.15.0": + version "6.15.0" + resolved "https://registry.yarnpkg.com/@prisma/fetch-engine/-/fetch-engine-6.15.0.tgz#ec7e57030789ed42a87bf5b66be4db6e612412e9" + integrity sha512-xcT5f6b+OWBq6vTUnRCc7qL+Im570CtwvgSj+0MTSGA1o9UDSKZ/WANvwtiRXdbYWECpyC3CukoG3A04VTAPHw== + dependencies: + "@prisma/debug" "6.15.0" + "@prisma/engines-version" "6.15.0-5.85179d7826409ee107a6ba334b5e305ae3fba9fb" + "@prisma/get-platform" "6.15.0" + +"@prisma/get-platform@6.15.0": + version "6.15.0" + resolved "https://registry.yarnpkg.com/@prisma/get-platform/-/get-platform-6.15.0.tgz#86a114534a534090b931801b83356b0bbe676f29" + integrity sha512-Jbb+Xbxyp05NSR1x2epabetHiXvpO8tdN2YNoWoA/ZsbYyxxu/CO/ROBauIFuMXs3Ti+W7N7SJtWsHGaWte9Rg== + dependencies: + "@prisma/debug" "6.15.0" + +"@prisma/instrumentation@5.22.0": + version "5.22.0" + resolved "https://registry.yarnpkg.com/@prisma/instrumentation/-/instrumentation-5.22.0.tgz#c39941046e9886e17bdb47dbac45946c24d579aa" + integrity sha512-LxccF392NN37ISGxIurUljZSh1YWnphO34V5a0+T7FVQG2u9bhAXRTJpgmQ3483woVhkraQZFF7cbRrpbw/F4Q== dependencies: "@opentelemetry/api" "^1.8" - "@opentelemetry/instrumentation" "^0.49 || ^0.50 || ^0.51 || ^0.52.0" + "@opentelemetry/instrumentation" "^0.49 || ^0.50 || ^0.51 || ^0.52.0 || ^0.53.0" "@opentelemetry/sdk-trace-base" "^1.22" +"@prisma/instrumentation@6.15.0": + version "6.15.0" + resolved "https://registry.yarnpkg.com/@prisma/instrumentation/-/instrumentation-6.15.0.tgz#40b066dc6b1ea621aa5ae0fd6d54319550b7d8c9" + integrity sha512-6TXaH6OmDkMOQvOxwLZ8XS51hU2v4A3vmE2pSijCIiGRJYyNeMcL6nMHQMyYdZRD8wl7LF3Wzc+AMPMV/9Oo7A== + dependencies: + "@opentelemetry/instrumentation" "^0.52.0 || ^0.53.0 || ^0.54.0 || ^0.55.0 || ^0.56.0 || ^0.57.0" + "@protobufjs/aspromise@^1.1.1", "@protobufjs/aspromise@^1.1.2": version "1.1.2" resolved "https://registry.yarnpkg.com/@protobufjs/aspromise/-/aspromise-1.1.2.tgz#9b8b0cc663d669a7d8f6f5d0893a14d348f30fbf" @@ -5371,127 +5443,66 @@ "@sentry/types" "7.120.4" "@sentry/utils" "7.120.4" -"@sentry/core@8.29.0": - version "8.29.0" - resolved "https://registry.yarnpkg.com/@sentry/core/-/core-8.29.0.tgz#52032ece2d7b60f3775f10189c27e26b1cebdbca" - integrity sha512-scMbZaJ0Ov8NPgWn86EdjhyTLrhvRVbTxjg0imJAvhIvRbblH3xyqye/17Qnk2fOp8TNDOl7TBZHi0NCFQ5HUw== - dependencies: - "@sentry/types" "8.29.0" - "@sentry/utils" "8.29.0" - -"@sentry/core@8.32.0": - version "8.32.0" - resolved "https://registry.yarnpkg.com/@sentry/core/-/core-8.32.0.tgz#7c4b74afa7a15bd31f5e6881aac82ccfd753e1d6" - integrity sha512-+xidTr0lZ0c755tq4k75dXPEb8PA+qvIefW3U9+dQMORLokBrYoKYMf5zZTG2k/OfSJS6OSxatUj36NFuCs3aA== - dependencies: - "@sentry/types" "8.32.0" - "@sentry/utils" "8.32.0" +"@sentry/core@8.55.0": + version "8.55.0" + resolved "https://registry.yarnpkg.com/@sentry/core/-/core-8.55.0.tgz#4964920229fcf649237ef13b1533dfc4b9f6b22e" + integrity sha512-6g7jpbefjHYs821Z+EBJ8r4Z7LT5h80YSWRJaylGS4nW5W5Z2KXzpdnyFarv37O7QjauzVC2E+PABmpkw5/JGA== -"@sentry/node@8.29.0": - version "8.29.0" - resolved "https://registry.yarnpkg.com/@sentry/node/-/node-8.29.0.tgz#6e462b8802356a630c56733dc795a4035464c4ab" - integrity sha512-RCKpWR6DUWmlxtms10MRXwJZRrFt1a2P38FjwEEahcdcK1R6wB8GPf0GO4JnJAiw6oeM0MERSqLIcSLT8+FxtA== +"@sentry/node@8.55.0": + version "8.55.0" + resolved "https://registry.yarnpkg.com/@sentry/node/-/node-8.55.0.tgz#0d0a14777824c341f30c746c4b2939456a09a272" + integrity sha512-h10LJLDTRAzYgay60Oy7moMookqqSZSviCWkkmHZyaDn+4WURnPp5SKhhfrzPRQcXKrweiOwDSHBgn1tweDssg== dependencies: "@opentelemetry/api" "^1.9.0" - "@opentelemetry/context-async-hooks" "^1.25.1" - "@opentelemetry/core" "^1.25.1" - "@opentelemetry/instrumentation" "^0.53.0" - "@opentelemetry/instrumentation-connect" "0.39.0" - "@opentelemetry/instrumentation-express" "0.42.0" - "@opentelemetry/instrumentation-fastify" "0.39.0" - "@opentelemetry/instrumentation-fs" "0.15.0" - "@opentelemetry/instrumentation-generic-pool" "0.39.0" - "@opentelemetry/instrumentation-graphql" "0.43.0" - "@opentelemetry/instrumentation-hapi" "0.41.0" - "@opentelemetry/instrumentation-http" "0.53.0" - "@opentelemetry/instrumentation-ioredis" "0.43.0" - "@opentelemetry/instrumentation-koa" "0.43.0" - "@opentelemetry/instrumentation-mongodb" "0.47.0" - "@opentelemetry/instrumentation-mongoose" "0.42.0" - "@opentelemetry/instrumentation-mysql" "0.41.0" - "@opentelemetry/instrumentation-mysql2" "0.41.0" - "@opentelemetry/instrumentation-nestjs-core" "0.40.0" - "@opentelemetry/instrumentation-pg" "0.44.0" - "@opentelemetry/instrumentation-redis-4" "0.42.0" - "@opentelemetry/resources" "^1.25.1" - "@opentelemetry/sdk-trace-base" "^1.25.1" - "@opentelemetry/semantic-conventions" "^1.25.1" - "@prisma/instrumentation" "5.19.1" - "@sentry/core" "8.29.0" - "@sentry/opentelemetry" "8.29.0" - "@sentry/types" "8.29.0" - "@sentry/utils" "8.29.0" - import-in-the-middle "^1.11.0" - optionalDependencies: - opentelemetry-instrumentation-fetch-node "1.2.3" - -"@sentry/node@8.32.0": - version "8.32.0" - resolved "https://registry.yarnpkg.com/@sentry/node/-/node-8.32.0.tgz#68822b3246fb2ed7418f21795ed539a18058cfa8" - integrity sha512-a2PoFA9j/HmJVGF/zXJhLP6QhRHGye/2EznQdHOELsH1BkeMgBaXl7D52r2E/b7qki647lXrdbspB6jid8NycA== - dependencies: - "@opentelemetry/api" "^1.9.0" - "@opentelemetry/context-async-hooks" "^1.25.1" - "@opentelemetry/core" "^1.25.1" - "@opentelemetry/instrumentation" "^0.53.0" - "@opentelemetry/instrumentation-amqplib" "^0.42.0" - "@opentelemetry/instrumentation-connect" "0.39.0" - "@opentelemetry/instrumentation-dataloader" "0.12.0" - "@opentelemetry/instrumentation-express" "0.42.0" - "@opentelemetry/instrumentation-fastify" "0.39.0" - "@opentelemetry/instrumentation-fs" "0.15.0" - "@opentelemetry/instrumentation-generic-pool" "0.39.0" - "@opentelemetry/instrumentation-graphql" "0.43.0" - "@opentelemetry/instrumentation-hapi" "0.41.0" - "@opentelemetry/instrumentation-http" "0.53.0" - "@opentelemetry/instrumentation-ioredis" "0.43.0" - "@opentelemetry/instrumentation-kafkajs" "0.3.0" - "@opentelemetry/instrumentation-koa" "0.43.0" - "@opentelemetry/instrumentation-mongodb" "0.47.0" - "@opentelemetry/instrumentation-mongoose" "0.42.0" - "@opentelemetry/instrumentation-mysql" "0.41.0" - "@opentelemetry/instrumentation-mysql2" "0.41.0" - "@opentelemetry/instrumentation-nestjs-core" "0.40.0" - "@opentelemetry/instrumentation-pg" "0.44.0" - "@opentelemetry/instrumentation-redis-4" "0.42.0" - "@opentelemetry/instrumentation-undici" "0.6.0" - "@opentelemetry/resources" "^1.26.0" - "@opentelemetry/sdk-trace-base" "^1.26.0" - "@opentelemetry/semantic-conventions" "^1.27.0" - "@prisma/instrumentation" "5.19.1" - "@sentry/core" "8.32.0" - "@sentry/opentelemetry" "8.32.0" - "@sentry/types" "8.32.0" - "@sentry/utils" "8.32.0" - import-in-the-middle "^1.11.0" - -"@sentry/opentelemetry@8.29.0": - version "8.29.0" - resolved "https://registry.yarnpkg.com/@sentry/opentelemetry/-/opentelemetry-8.29.0.tgz#6ae54c640155925fc42ac86f37d125f9bb983794" - integrity sha512-MtfjDMUuKFYlyw9hZohp9xnphz+6QosyHb2zCV3e/fANoA53FDxmg/Co7haMohUCiOwwJPytUmSQQaP0nyL2Uw== - dependencies: - "@sentry/core" "8.29.0" - "@sentry/types" "8.29.0" - "@sentry/utils" "8.29.0" - -"@sentry/opentelemetry@8.32.0": - version "8.32.0" - resolved "https://registry.yarnpkg.com/@sentry/opentelemetry/-/opentelemetry-8.32.0.tgz#4af02c17102635e4b34942d2e82d3620ddb7d95a" - integrity sha512-YCD8EnwJJ2ab3zWWtu5VrvHP/6Ss6GGQH0TYx2cfeGG3c0wTA/5zYx9JR4i3hUtOh1pifN34HlY0yyQHD4yctg== - dependencies: - "@sentry/core" "8.32.0" - "@sentry/types" "8.32.0" - "@sentry/utils" "8.32.0" - -"@sentry/profiling-node@8.32.0": - version "8.32.0" - resolved "https://registry.yarnpkg.com/@sentry/profiling-node/-/profiling-node-8.32.0.tgz#cb4bf0512dc7a6f0a9058529f3c4dbb8963a597d" - integrity sha512-wKE2JAvDM3DjVNFKd2YLzwhXFS3SurLxDPpvA1EjMQ9Me7sR5ssO9jFn3e6et5oT3e3EzGDyTfbrJ81BNQuZhQ== - dependencies: - "@sentry/core" "8.32.0" - "@sentry/node" "8.32.0" - "@sentry/types" "8.32.0" - "@sentry/utils" "8.32.0" + "@opentelemetry/context-async-hooks" "^1.30.1" + "@opentelemetry/core" "^1.30.1" + "@opentelemetry/instrumentation" "^0.57.1" + "@opentelemetry/instrumentation-amqplib" "^0.46.0" + "@opentelemetry/instrumentation-connect" "0.43.0" + "@opentelemetry/instrumentation-dataloader" "0.16.0" + "@opentelemetry/instrumentation-express" "0.47.0" + "@opentelemetry/instrumentation-fastify" "0.44.1" + "@opentelemetry/instrumentation-fs" "0.19.0" + "@opentelemetry/instrumentation-generic-pool" "0.43.0" + "@opentelemetry/instrumentation-graphql" "0.47.0" + "@opentelemetry/instrumentation-hapi" "0.45.1" + "@opentelemetry/instrumentation-http" "0.57.1" + "@opentelemetry/instrumentation-ioredis" "0.47.0" + "@opentelemetry/instrumentation-kafkajs" "0.7.0" + "@opentelemetry/instrumentation-knex" "0.44.0" + "@opentelemetry/instrumentation-koa" "0.47.0" + "@opentelemetry/instrumentation-lru-memoizer" "0.44.0" + "@opentelemetry/instrumentation-mongodb" "0.51.0" + "@opentelemetry/instrumentation-mongoose" "0.46.0" + "@opentelemetry/instrumentation-mysql" "0.45.0" + "@opentelemetry/instrumentation-mysql2" "0.45.0" + "@opentelemetry/instrumentation-nestjs-core" "0.44.0" + "@opentelemetry/instrumentation-pg" "0.50.0" + "@opentelemetry/instrumentation-redis-4" "0.46.0" + "@opentelemetry/instrumentation-tedious" "0.18.0" + "@opentelemetry/instrumentation-undici" "0.10.0" + "@opentelemetry/resources" "^1.30.1" + "@opentelemetry/sdk-trace-base" "^1.30.1" + "@opentelemetry/semantic-conventions" "^1.28.0" + "@prisma/instrumentation" "5.22.0" + "@sentry/core" "8.55.0" + "@sentry/opentelemetry" "8.55.0" + import-in-the-middle "^1.11.2" + +"@sentry/opentelemetry@8.55.0": + version "8.55.0" + resolved "https://registry.yarnpkg.com/@sentry/opentelemetry/-/opentelemetry-8.55.0.tgz#d4827d52ae9f1fb9352729250b980f9f2caa1877" + integrity sha512-UvatdmSr3Xf+4PLBzJNLZ2JjG1yAPWGe/VrJlJAqyTJ2gKeTzgXJJw8rp4pbvNZO8NaTGEYhhO+scLUj0UtLAQ== + dependencies: + "@sentry/core" "8.55.0" + +"@sentry/profiling-node@8.55.0": + version "8.55.0" + resolved "https://registry.yarnpkg.com/@sentry/profiling-node/-/profiling-node-8.55.0.tgz#bb7d6735ba6dc7fe294c5039b1f06997134d6e33" + integrity sha512-rYrlxbMlfQLHhkBUEC7bviuja1rojCb4+TtXi4NGnB4PppZeveGeuVTdJDWt3Ed6IBd20EEYoXv4+0aETbEnpw== + dependencies: + "@sentry/core" "8.55.0" + "@sentry/node" "8.55.0" detect-libc "^2.0.2" node-abi "^3.61.0" @@ -5507,16 +5518,6 @@ resolved "https://registry.yarnpkg.com/@sentry/types/-/types-7.120.4.tgz#8fab8dceeec4bda079fc6e8e380b982f766de354" integrity sha512-cUq2hSSe6/qrU6oZsEP4InMI5VVdD86aypE+ENrQ6eZEVLTCYm1w6XhW1NvIu3UuWh7gZec4a9J7AFpYxki88Q== -"@sentry/types@8.29.0": - version "8.29.0" - resolved "https://registry.yarnpkg.com/@sentry/types/-/types-8.29.0.tgz#c19e43524b8e7766028f4da8f02eddcc33518541" - integrity sha512-j4gX3ctzgD4xVWllXAhm6M+kHFEvrFoUPFq60X/pgkjsWCocGuhtNfB0rW43ICG8hCnlz8IYl7O7b8V8qY7SPg== - -"@sentry/types@8.32.0": - version "8.32.0" - resolved "https://registry.yarnpkg.com/@sentry/types/-/types-8.32.0.tgz#dfd8aa9449a5f793b9c720888819a74a11f1790d" - integrity sha512-hxckvN2MzS5SgGDgVQ0/QpZXk13Vrq4BtZLwXhPhyeTmZtUiUfWvcL5TFQqLinfKdTKPe9q2MxeAJ0D4LalhMg== - "@sentry/utils@7.120.4": version "7.120.4" resolved "https://registry.yarnpkg.com/@sentry/utils/-/utils-7.120.4.tgz#8995637fc4742ee75df347dcd0f08ee137968c78" @@ -5524,20 +5525,6 @@ dependencies: "@sentry/types" "7.120.4" -"@sentry/utils@8.29.0": - version "8.29.0" - resolved "https://registry.yarnpkg.com/@sentry/utils/-/utils-8.29.0.tgz#d4a36643369e30ba62ef8f40f149420a100f64bb" - integrity sha512-nb93/m3SjQChQJFqJj3oNW3Rz/12yrT7jypTCire3c2hpYWG2uR5n8VY9UUMTA6HLNvdom6tckK7p3bXGXlF0w== - dependencies: - "@sentry/types" "8.29.0" - -"@sentry/utils@8.32.0": - version "8.32.0" - resolved "https://registry.yarnpkg.com/@sentry/utils/-/utils-8.32.0.tgz#99a4298ee8fd7208ade470931c19d71c571dfce8" - integrity sha512-t1WVERhgmYURxbBj9J4/H2P2X+VKqm7B3ce9iQyrZbdf5NekhcU4jHIecPUWCPHjQkFIqkVTorqeBmDTlg/UmQ== - dependencies: - "@sentry/types" "8.32.0" - "@sideway/address@^4.1.5": version "4.1.5" resolved "https://registry.yarnpkg.com/@sideway/address/-/address-4.1.5.tgz#4bc149a0076623ced99ca8208ba780d65a99b9d5" @@ -6390,6 +6377,11 @@ c32check "^2.0.0" lodash.clonedeep "^4.5.0" +"@standard-schema/spec@^1.0.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@standard-schema/spec/-/spec-1.1.0.tgz#a79b55dbaf8604812f52d140b2c9ab41bc150bb8" + integrity sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w== + "@swc/core-darwin-arm64@1.3.101": version "1.3.101" resolved "https://registry.yarnpkg.com/@swc/core-darwin-arm64/-/core-darwin-arm64-1.3.101.tgz#9ffdc0e77c31b20877fa7405c82905e0c76738d0" @@ -7165,7 +7157,7 @@ "@types/express" "*" "@types/serve-static" "*" -"@types/tedious@^4.0.6": +"@types/tedious@^4.0.14", "@types/tedious@^4.0.6": version "4.0.14" resolved "https://registry.yarnpkg.com/@types/tedious/-/tedious-4.0.14.tgz#868118e7a67808258c05158e9cad89ca58a2aec1" integrity sha512-KHPsfX/FoVbUGbyYvk1q9MMQHLPeRZhRJZdO45Q4YjvFkv4hMNghCWTvy7rdKessBsmtz4euWCWAB6/tVpI1Iw== @@ -7562,11 +7554,6 @@ accepts@~1.3.4, accepts@~1.3.8: mime-types "~2.1.34" negotiator "0.6.3" -acorn-import-assertions@^1.9.0: - version "1.9.0" - resolved "https://registry.yarnpkg.com/acorn-import-assertions/-/acorn-import-assertions-1.9.0.tgz#507276249d684797c84e0734ef84860334cfb1ac" - integrity sha512-cmMwop9x+8KFhxvKrKfPYmN6/pKTYYHBqLa0DfvVZcKMJWNyWLnaqND7dx/qn66R7ewM1UX5XMaDVP5wlVTaVA== - acorn-import-attributes@^1.9.5: version "1.9.5" resolved "https://registry.yarnpkg.com/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz#7eb1557b1ba05ef18b5ed0ec67591bfab04688ef" @@ -7594,7 +7581,7 @@ acorn@^7.4.0, acorn@^7.4.1: resolved "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz" integrity sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A== -acorn@^8.11.0, acorn@^8.14.0, acorn@^8.15.0, acorn@^8.4.1, acorn@^8.8.2: +acorn@^8.11.0, acorn@^8.14.0, acorn@^8.15.0, acorn@^8.4.1: version "8.15.0" resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.15.0.tgz#a360898bc415edaac46c8241f6383975b930b816" integrity sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg== @@ -8454,6 +8441,24 @@ bytes@3.1.2: resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.2.tgz#8b0beeb98605adf1b128fa4386403c009e0221a5" integrity sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg== +c12@3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/c12/-/c12-3.1.0.tgz#9e237970e1d3b74ebae51d25945cb59664c12c89" + integrity sha512-uWoS8OU1MEIsOv8p/5a82c3H31LsWVR5qiyXVfBNOzfffjUWtPnhAb4BYI2uG2HfGmZmFjCtui5XNWaps+iFuw== + dependencies: + chokidar "^4.0.3" + confbox "^0.2.2" + defu "^6.1.4" + dotenv "^16.6.1" + exsolve "^1.0.7" + giget "^2.0.0" + jiti "^2.4.2" + ohash "^2.0.11" + pathe "^2.0.3" + perfect-debounce "^1.0.0" + pkg-types "^2.2.0" + rc9 "^2.1.2" + c32check@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/c32check/-/c32check-2.0.0.tgz#b9365618b2fb135c0783d03f00605b7b0f90c659" @@ -8711,6 +8716,13 @@ chokidar@^3.5.2, chokidar@^3.5.3: optionalDependencies: fsevents "~2.3.2" +chokidar@^4.0.3: + version "4.0.3" + resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-4.0.3.tgz#7be37a4c03c9aee1ecfe862a4a23b2c70c205d30" + integrity sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA== + dependencies: + readdirp "^4.0.1" + chownr@^1.1.1: version "1.1.4" resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b" @@ -8734,6 +8746,18 @@ cipher-base@^1.0.0, cipher-base@^1.0.1, cipher-base@^1.0.3: inherits "^2.0.4" safe-buffer "^5.2.1" +citty@^0.1.6: + version "0.1.6" + resolved "https://registry.yarnpkg.com/citty/-/citty-0.1.6.tgz#0f7904da1ed4625e1a9ea7e0fa780981aab7c5e4" + integrity sha512-tskPPKEs8D2KPafUypv2gxwJP8h/OaJmC82QQGGDQcHvXX43xF2VDACcJVmZ0EuSxkpO9Kc4MlrA3q0+FG58AQ== + dependencies: + consola "^3.2.3" + +citty@^0.2.0: + version "0.2.2" + resolved "https://registry.yarnpkg.com/citty/-/citty-0.2.2.tgz#92d3f7d13868a730ab06c420bb10bded06cf259f" + integrity sha512-+6vJA3L98yv+IdfKGZHBNiGW5KHn22e/JwID0Strsz8h4S/csAu/OuICwxrg44k5MRiZHWIo8XXuJgQTriRP4w== + cjs-module-lexer@^1.2.2: version "1.4.3" resolved "https://registry.yarnpkg.com/cjs-module-lexer/-/cjs-module-lexer-1.4.3.tgz#0f79731eb8cfe1ec72acd4066efac9d61991b00d" @@ -8974,6 +8998,11 @@ concurrently@^8.2.0: tree-kill "^1.2.2" yargs "^17.7.2" +confbox@^0.2.2: + version "0.2.4" + resolved "https://registry.yarnpkg.com/confbox/-/confbox-0.2.4.tgz#592e7be71f882a4a874e3c88f0ac1ef6f7da1ce5" + integrity sha512-ysOGlgTFbN2/Y6Cg3Iye8YKulHw+R2fNXHrgSmXISQdMnomY6eNDprVdW9R5xBguEqI954+S6709UyiO7B+6OQ== + config-chain@^1.1.13: version "1.1.13" resolved "https://registry.yarnpkg.com/config-chain/-/config-chain-1.1.13.tgz#fad0795aa6a6cdaff9ed1b68e9dff94372c232f4" @@ -8982,6 +9011,11 @@ config-chain@^1.1.13: ini "^1.3.4" proto-list "~1.2.1" +consola@^3.2.3, consola@^3.4.0: + version "3.4.2" + resolved "https://registry.yarnpkg.com/consola/-/consola-3.4.2.tgz#5af110145397bb67afdab77013fdc34cae590ea7" + integrity sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA== + console-control-strings@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/console-control-strings/-/console-control-strings-1.1.0.tgz#3d7cf4464db6446ea644bf4b39507f9851008e8e" @@ -9370,6 +9404,11 @@ deep-is@^0.1.3: resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.4.tgz#a6f2dce612fadd2ef1f519b73551f17e85199831" integrity sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ== +deepmerge-ts@7.1.5: + version "7.1.5" + resolved "https://registry.yarnpkg.com/deepmerge-ts/-/deepmerge-ts-7.1.5.tgz#ff818564007f5c150808d2b7b732cac83aa415ab" + integrity sha512-HOJkrhaYsweh+W+e74Yn7YStZOilkoPb6fycpwNLKzSPtruFs48nYis0zy5yJz1+ktUhHxoRDJ27RQAWLIJVJw== + deepmerge@^4.2.2, deepmerge@^4.3.1: version "4.3.1" resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-4.3.1.tgz#44b5f2147cd3b00d4b56137685966f26fd25dd4a" @@ -9400,6 +9439,11 @@ define-properties@^1.2.1: has-property-descriptors "^1.0.0" object-keys "^1.1.1" +defu@^6.1.4: + version "6.1.7" + resolved "https://registry.yarnpkg.com/defu/-/defu-6.1.7.tgz#72543567c8e9f97ff13ce402b6dbe09ac5ae4d23" + integrity sha512-7z22QmUWiQ/2d0KkdYmANbRUVABpZ9SNYyH5vx6PZ+nE5bcC0l7uFvEfHlyld/HcGBFTL536ClDt3DEcSlEJAQ== + delay@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/delay/-/delay-5.0.0.tgz#137045ef1b96e5071060dd5be60bf9334436bd1d" @@ -9425,6 +9469,11 @@ depd@~1.1.2: resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.2.tgz#9bcd52e14c097763e749b274c4346ed2e560b5a9" integrity sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ== +destr@^2.0.3: + version "2.0.5" + resolved "https://registry.yarnpkg.com/destr/-/destr-2.0.5.tgz#7d112ff1b925fb8d2079fac5bdb4a90973b51fdb" + integrity sha512-ugFTXCtDZunbzasqBxrK93Ik/DRYsO6S/fedkWEMKqt04xZ4csmnmwGDBAb07QWNaGMAmnTIemsYZCksjATwsA== + destroy@1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/destroy/-/destroy-1.2.0.tgz#4803735509ad8be552934c67df614f94e66fa015" @@ -9595,7 +9644,7 @@ dotenv@16.0.3: resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.0.3.tgz#115aec42bac5053db3c456db30cc243a5a836a07" integrity sha512-7GO6HghkA5fYG9TYnNxi14/7K9f5occMlp3zXAuSxn7CKCxt9xbNWG7yF8hTCSUchlfWSe3uLmlPfigevRItzQ== -dotenv@^16.4.7, dotenv@^16.5.0: +dotenv@^16.4.7, dotenv@^16.5.0, dotenv@^16.6.1: version "16.6.1" resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.6.1.tgz#773f0e69527a8315c7285d5ee73c4459d20a8020" integrity sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow== @@ -9663,6 +9712,14 @@ ee-first@1.1.1: resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" integrity sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow== +effect@3.16.12: + version "3.16.12" + resolved "https://registry.yarnpkg.com/effect/-/effect-3.16.12.tgz#3762f745846cfa4905512e397e17f683438addbe" + integrity sha512-N39iBk0K71F9nb442TLbTkjl24FLUzuvx2i1I2RsEAQsdAdUTuUoW0vlfUXgkMTUOnYqKnWcFfqw4hK4Pw27hg== + dependencies: + "@standard-schema/spec" "^1.0.0" + fast-check "^3.23.1" + electron-fetch@^1.7.2: version "1.9.1" resolved "https://registry.yarnpkg.com/electron-fetch/-/electron-fetch-1.9.1.tgz#e28bfe78d467de3f2dec884b1d72b8b05322f30f" @@ -9698,6 +9755,11 @@ emoji-regex@^9.2.2: resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-9.2.2.tgz#840c8803b0d8047f4ff0cf963176b32d4ef3ed72" integrity sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg== +empathic@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/empathic/-/empathic-2.0.0.tgz#71d3c2b94fad49532ef98a6c34be0386659f6131" + integrity sha512-i6UzDscO/XfAcNYD75CfICkmfLedpyPDdozrLMmQc5ORaQcdMoc21OnlEylMIqI7U8eniKrPMxxtj8k0vhmJhA== + encodeurl@~1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59" @@ -10405,6 +10467,11 @@ express@^4.17.1: utils-merge "1.0.1" vary "~1.1.2" +exsolve@^1.0.7: + version "1.0.8" + resolved "https://registry.yarnpkg.com/exsolve/-/exsolve-1.0.8.tgz#7f5e34da61cd1116deda5136e62292c096f50613" + integrity sha512-LmDxfWXwcTArk8fUEnOfSZpHOJ6zOMUJKOtFLFqJLoKJetuQG874Uc7/Kki7zFLzYybmZhp1M7+98pfMqeX8yA== + extend@^3.0.2, extend@~3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" @@ -10420,6 +10487,13 @@ extsprintf@^1.2.0: resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.4.1.tgz#8d172c064867f235c0c84a596806d279bf4bcc07" integrity sha512-Wrk35e8ydCKDj/ArClo1VrPVmN8zph5V4AtHwIuHhvMXsKf73UT3BOD+azBIW+3wOJ4FhEH7zyaJCFvChjYvMA== +fast-check@^3.23.1: + version "3.23.2" + resolved "https://registry.yarnpkg.com/fast-check/-/fast-check-3.23.2.tgz#0129f1eb7e4f500f58e8290edc83c670e4a574a2" + integrity sha512-h5+1OzzfCC3Ef7VbtKdcv7zsstUQwUDlYpUTvjeUsJAssPgLn7QzbboPtL5ro04Mq0rPOsMzl7q5hIbRs2wD1A== + dependencies: + pure-rand "^6.1.0" + fast-copy@^3.0.0: version "3.0.2" resolved "https://registry.yarnpkg.com/fast-copy/-/fast-copy-3.0.2.tgz#59c68f59ccbcac82050ba992e0d5c389097c9d35" @@ -10673,6 +10747,11 @@ formidable@^2.1.2: once "^1.4.0" qs "^6.11.0" +forwarded-parse@2.1.2: + version "2.1.2" + resolved "https://registry.yarnpkg.com/forwarded-parse/-/forwarded-parse-2.1.2.tgz#08511eddaaa2ddfd56ba11138eee7df117a09325" + integrity sha512-alTFZZQDKMporBH77856pXgzhEzaUVmLCDk+egLgIgHst3Tpndzz8MnKe+GzRJRfvVdn69HhpW7cmXzvtLvJAw== + forwarded@0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.2.0.tgz#2269936428aad4c15c7ebe9779a84bf0b2a81811" @@ -10912,6 +10991,18 @@ getpass@^0.1.1: dependencies: assert-plus "^1.0.0" +giget@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/giget/-/giget-2.0.0.tgz#395fc934a43f9a7a29a29d55b99f23e30c14f195" + integrity sha512-L5bGsVkxJbJgdnwyuheIunkGatUF/zssUoxxjACCseZYAVbaqdh9Tsmmlkl8vYan09H7sbvKt4pS8GqKLBrEzA== + dependencies: + citty "^0.1.6" + consola "^3.4.0" + defu "^6.1.4" + node-fetch-native "^1.6.6" + nypm "^0.6.0" + pathe "^2.0.3" + github-from-package@0.0.0: version "0.0.0" resolved "https://registry.yarnpkg.com/github-from-package/-/github-from-package-0.0.0.tgz#97fb5d96bfde8973313f20e8288ef9a167fa64ce" @@ -11421,17 +11512,17 @@ import-in-the-middle@1.3.5: dependencies: module-details-from-path "^1.0.3" -import-in-the-middle@1.7.1: - version "1.7.1" - resolved "https://registry.yarnpkg.com/import-in-the-middle/-/import-in-the-middle-1.7.1.tgz#3e111ff79c639d0bde459bd7ba29dd9fdf357364" - integrity sha512-1LrZPDtW+atAxH42S6288qyDFNQ2YCty+2mxEPRtfazH6Z5QwkaBSTS2ods7hnVJioF6rkRfNoA6A/MstpFXLg== +import-in-the-middle@^1.11.2: + version "1.15.0" + resolved "https://registry.yarnpkg.com/import-in-the-middle/-/import-in-the-middle-1.15.0.tgz#9e20827a322bbadaeb5e3bac49ea8f6d4685fdd8" + integrity sha512-bpQy+CrsRmYmoPMAE/0G33iwRqwW4ouqdRg8jgbH3aKuCtOc8lxgmYXg2dMM92CRiGP660EtBcymH/eVUpCSaA== dependencies: - acorn "^8.8.2" - acorn-import-assertions "^1.9.0" + acorn "^8.14.0" + acorn-import-attributes "^1.9.5" cjs-module-lexer "^1.2.2" module-details-from-path "^1.0.3" -import-in-the-middle@^1.11.0, import-in-the-middle@^1.8.1: +import-in-the-middle@^1.8.1: version "1.14.2" resolved "https://registry.yarnpkg.com/import-in-the-middle/-/import-in-the-middle-1.14.2.tgz#283661625a88ff7c0462bd2984f77715c3bc967c" integrity sha512-5tCuY9BV8ujfOpwtAGgsTx9CGUapcFMEEyByLv1B+v2+6DhAcw+Zr0nhQT7uwaZ7DiourxFEscghOR8e1aPLQw== @@ -12091,6 +12182,11 @@ jiti@^1.19.1: resolved "https://registry.yarnpkg.com/jiti/-/jiti-1.21.7.tgz#9dd81043424a3d28458b193d965f0d18a2300ba9" integrity sha512-/imKNG4EbWNrVjoNC/1H5/9GFy+tqjGBHCaSsN+P2RnPqjsLmv6UD3Ej+Kj8nBWaRAwyk7kK5ZUc+OEatnTR3A== +jiti@^2.4.2: + version "2.6.1" + resolved "https://registry.yarnpkg.com/jiti/-/jiti-2.6.1.tgz#178ef2fc9a1a594248c20627cd820187a4d78d92" + integrity sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ== + jmespath@0.16.0: version "0.16.0" resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.16.0.tgz#b15b0a85dfd4d930d43e69ed605943c802785076" @@ -13299,6 +13395,11 @@ node-domexception@^1.0.0: resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5" integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ== +node-fetch-native@^1.6.6: + version "1.6.7" + resolved "https://registry.yarnpkg.com/node-fetch-native/-/node-fetch-native-1.6.7.tgz#9d09ca63066cc48423211ed4caf5d70075d76a71" + integrity sha512-g9yhqoedzIUm0nTnTqAQvueMPVOuIY16bqgAJJC8XOOubYFNwz6IER9qs0Gq2Xd0+CecCKFjtdDTMA4u4xG06Q== + node-fetch@^2.6.1, node-fetch@^2.6.7, node-fetch@^2.6.8, node-fetch@^2.6.9, node-fetch@^2.7.0: version "2.7.0" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" @@ -13438,6 +13539,15 @@ npmlog@^6.0.0: gauge "^4.0.3" set-blocking "^2.0.0" +nypm@^0.6.0: + version "0.6.5" + resolved "https://registry.yarnpkg.com/nypm/-/nypm-0.6.5.tgz#5edd97310ee468fa3306b9ef5fe82b8ef6605b57" + integrity sha512-K6AJy1GMVyfyMXRVB88700BJqNUkByijGJM8kEHpLdcAt+vSQAVfkWWHYzuRXHSY6xA2sNc5RjTj0p9rE2izVQ== + dependencies: + citty "^0.2.0" + pathe "^2.0.3" + tinyexec "^1.0.2" + o3@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/o3/-/o3-1.0.3.tgz#192ce877a882dfa6751f0412a865fafb2da1dac0" @@ -13523,6 +13633,11 @@ object.values@^1.2.1: define-properties "^1.2.1" es-object-atoms "^1.0.0" +ohash@^2.0.11: + version "2.0.11" + resolved "https://registry.yarnpkg.com/ohash/-/ohash-2.0.11.tgz#60b11e8cff62ca9dee88d13747a5baa145f5900b" + integrity sha512-RdR9FQrFwNBNXAr4GixM8YaRZRJ5PUWbKYbE5eOsrwAjJW0q2REGcf79oYPsLyskQCZG1PLN+S/K1V00joZAoQ== + on-exit-leak-free@^2.1.0: version "2.1.2" resolved "https://registry.yarnpkg.com/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz#fed195c9ebddb7d9e4c3842f93f281ac8dadd3b8" @@ -13569,14 +13684,6 @@ openid-client@^6.6.4: jose "^6.1.0" oauth4webapi "^3.8.0" -opentelemetry-instrumentation-fetch-node@1.2.3: - version "1.2.3" - resolved "https://registry.yarnpkg.com/opentelemetry-instrumentation-fetch-node/-/opentelemetry-instrumentation-fetch-node-1.2.3.tgz#beb24048bdccb1943ba2a5bbadca68020e448ea7" - integrity sha512-Qb11T7KvoCevMaSeuamcLsAD+pZnavkhDnlVL0kRozfhl42dKG5Q3anUklAFKJZjY3twLR+BnRa6DlwwkIE/+A== - dependencies: - "@opentelemetry/instrumentation" "^0.46.0" - "@opentelemetry/semantic-conventions" "^1.17.0" - opentracing@^0.14.4: version "0.14.7" resolved "https://registry.yarnpkg.com/opentracing/-/opentracing-0.14.7.tgz#25d472bd0296dc0b64d7b94cbc995219031428f5" @@ -13795,6 +13902,11 @@ pend@~1.2.0: resolved "https://registry.yarnpkg.com/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50" integrity sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg== +perfect-debounce@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/perfect-debounce/-/perfect-debounce-1.0.0.tgz#9c2e8bc30b169cc984a58b7d5b28049839591d2a" + integrity sha512-xCy9V055GLEqoFaHoC1SoLIaLmWctgCUaBaWxDZ7/Zx4CTyX7cJQLJOok/orfjZAh9kEYpjJa4d0KcJmCbctZA== + performance-now@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" @@ -13969,6 +14081,15 @@ pirates@^4.0.1: resolved "https://registry.yarnpkg.com/pirates/-/pirates-4.0.7.tgz#643b4a18c4257c8a65104b73f3049ce9a0a15e22" integrity sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA== +pkg-types@^2.2.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/pkg-types/-/pkg-types-2.3.0.tgz#037f2c19bd5402966ff6810e32706558cb5b5726" + integrity sha512-SIqCzDRg0s9npO5XQ3tNZioRY1uK06lA41ynBC1YmFTmnY6FjUjVt6s4LoADmwoig1qqD0oK8h1p/8mlMx8Oig== + dependencies: + confbox "^0.2.2" + exsolve "^1.0.7" + pathe "^2.0.3" + please-upgrade-node@^3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/please-upgrade-node/-/please-upgrade-node-3.2.0.tgz#aeddd3f994c933e4ad98b99d9a556efa0e2fe942" @@ -14137,12 +14258,13 @@ prism-react-renderer@2.1.0: "@types/prismjs" "^1.26.0" clsx "^1.2.1" -prisma@4.10.1: - version "4.10.1" - resolved "https://registry.yarnpkg.com/prisma/-/prisma-4.10.1.tgz#88084695d7b364ae6bebf93d5006f84439c4e7d1" - integrity sha512-0jDxgg+DruB1kHVNlcspXQB9au62IFfVg9drkhzXudszHNUAQn0lVuu+T8np0uC2z1nKD5S3qPeCyR8u5YFLnA== +prisma@6.15.0: + version "6.15.0" + resolved "https://registry.yarnpkg.com/prisma/-/prisma-6.15.0.tgz#198f95a4468aba8f3c95b58bf3400f65daeb8942" + integrity sha512-E6RCgOt+kUVtjtZgLQDBJ6md2tDItLJNExwI0XJeBc1FKL+Vwb+ovxXxuok9r8oBgsOXBA33fGDuE/0qDdCWqQ== dependencies: - "@prisma/engines" "4.10.1" + "@prisma/config" "6.15.0" + "@prisma/engines" "6.15.0" prismjs@1.29.0: version "1.29.0" @@ -14285,6 +14407,11 @@ punycode@^2.1.0, punycode@^2.1.1, punycode@^2.3.1: resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.3.1.tgz#027422e2faec0b25e1549c3e1bd8309b9133b6e5" integrity sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg== +pure-rand@^6.1.0: + version "6.1.0" + resolved "https://registry.yarnpkg.com/pure-rand/-/pure-rand-6.1.0.tgz#d173cf23258231976ccbdb05247c9787957604f2" + integrity sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA== + qs@6.13.0: version "6.13.0" resolved "https://registry.yarnpkg.com/qs/-/qs-6.13.0.tgz#6ca3bd58439f7e245655798997787b0d88a51906" @@ -14363,6 +14490,14 @@ raw-body@2.5.2: iconv-lite "0.4.24" unpipe "1.0.0" +rc9@^2.1.2: + version "2.1.2" + resolved "https://registry.yarnpkg.com/rc9/-/rc9-2.1.2.tgz#6282ff638a50caa0a91a31d76af4a0b9cbd1080d" + integrity sha512-btXCnMmRIBINM2LDZoEmOogIZU7Qe7zn4BpomSKZ/ykbLObuBdvG+mFq11DL6fjH1DRwHhrlgtYWG96bJiC7Cg== + dependencies: + defu "^6.1.4" + destr "^2.0.3" + rc@^1.2.7: version "1.2.8" resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.8.tgz#cd924bf5200a075b83c188cd6b9e211b7fc0d3ed" @@ -14561,6 +14696,11 @@ readable-stream@~1.0.31: isarray "0.0.1" string_decoder "~0.10.x" +readdirp@^4.0.1: + version "4.1.2" + resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-4.1.2.tgz#eb85801435fbf2a7ee58f19e0921b068fc69948d" + integrity sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg== + readdirp@~3.6.0: version "3.6.0" resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-3.6.0.tgz#74a370bd857116e245b29cc97340cd431a02a6c7" @@ -16035,6 +16175,11 @@ tinyexec@^0.3.2: resolved "https://registry.yarnpkg.com/tinyexec/-/tinyexec-0.3.2.tgz#941794e657a85e496577995c6eef66f53f42b3d2" integrity sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA== +tinyexec@^1.0.2: + version "1.1.1" + resolved "https://registry.yarnpkg.com/tinyexec/-/tinyexec-1.1.1.tgz#e1ff45dfa60d1dedb91b734956b78f6c2a3e821b" + integrity sha512-VKS/ZaQhhkKFMANmAOhhXVoIfBXblQxGX1myCQ2faQrfmobMftXeJPcZGp0gS07ocvGJWDLZGyOZDadDBqYIJg== + tinyglobby@^0.2.14: version "0.2.14" resolved "https://registry.yarnpkg.com/tinyglobby/-/tinyglobby-0.2.14.tgz#5280b0cf3f972b050e74ae88406c0a6a58f4079d" diff --git a/nodes-lib/test/root.spec.ts b/nodes-lib/test/root.spec.ts index 265b0f709..0bd5f87e5 100644 --- a/nodes-lib/test/root.spec.ts +++ b/nodes-lib/test/root.spec.ts @@ -830,6 +830,17 @@ describe("nodes-lib", () => { describe("external CID", async () => { test("can be added", async () => { + // Add a small file to the test IPFS node so it's resolvable as an "external" CID + const testContent = Buffer.from("test-cat-image-data"); + const ipfsUrl = "http://host.docker.internal:5003"; + const formData = new FormData(); + formData.append("file", new Blob([testContent]), "cat.jpg"); + const ipfsRes = await fetch(`${ipfsUrl}/api/v0/add?cid-version=1`, { + method: "POST", + body: formData, + }); + const { Hash: catCid } = await ipfsRes.json() as { Hash: string }; + const { node: { uuid }, } = await createBoilerplateNode(); @@ -838,8 +849,6 @@ describe("nodes-lib", () => { contextPath: "root", newFolderName: "catpics", }); - const catCid = - "bafkreidivzimqfqtoqxkrpge6bjyhlvxqs3rhe73owtmdulaxr5do5in7u"; const addResult = await addExternalCid({ uuid, externalCids: [ diff --git a/nodes-media/src/controllers/services/shareImagePuppeteer.ts b/nodes-media/src/controllers/services/shareImagePuppeteer.ts index 90812ab7b..a48d234b0 100644 --- a/nodes-media/src/controllers/services/shareImagePuppeteer.ts +++ b/nodes-media/src/controllers/services/shareImagePuppeteer.ts @@ -86,6 +86,28 @@ function escapeHtml(text: string): string { .replace(/`/g, '`'); } +// Inline tags we allow to "leak through" the HTML escape so chemistry/biology +// names (e.g. "NAD+", "H2O", italicized species names) +// render correctly inside the question and citation titles. None of these tags +// can carry attributes through this escape path, so there's no XSS surface. +const INLINE_FORMATTING_TAGS = ['sup', 'sub', 'i', 'em', 'b', 'strong'] as const; + +/** + * Like escapeHtml, but un-escapes a small allowlist of attribute-less inline + * formatting tags so titles like "NAD+" don't render as literal + * text. Use this for any user/LLM-controlled text rendered into the template + * that may legitimately contain inline scientific notation. + */ +function escapeHtmlAllowInline(text: string): string { + let escaped = escapeHtml(text); + for (const tag of INLINE_FORMATTING_TAGS) { + escaped = escaped + .split(`<${tag}>`).join(`<${tag}>`) + .split(`</${tag}>`).join(``); + } + return escaped; +} + /** * Custom error to indicate that SVG fallback is needed */ @@ -320,11 +342,17 @@ function getReferencedCitations(answer: string, citations: Citation[]): any[] { * Configures marked for proper markdown rendering */ function configureMarkdown() { - // Configure marked for proper HTML rendering + // Configure marked for proper HTML rendering. + // headerIds:false prevents marked from emitting `

` — the class-injection + // regex below was written for `

` literal tags, so without this the .heading-N CSS + // never applies and headings collapse to body-text size. mangle:false silences a noisy + // deprecation warning about email obfuscation we don't use. marked.setOptions({ breaks: true, gfm: true, - }); + headerIds: false, + mangle: false, + } as any); } /** @@ -333,18 +361,24 @@ function configureMarkdown() { function processSimpleMarkdownToHTML(text: string): string { if (!text) return ''; + // Ensure marked options match the main path (in particular, headerIds:false). + configureMarkdown(); // Use marked library for proper markdown parsing let html = marked.parse(text); - // Add custom classes to elements for styling + // Add custom classes to elements for styling. Tolerate any pre-existing + // attributes (e.g. id="..." that older marked versions emit on headings). html = html - .replace(/

/g, '

') - .replace(/

/g, '

') - .replace(/

/g, '

') - .replace(//g, '') - .replace(/

/g, '

') - .replace(/

    /g, '
      ') - .replace(/
    • /g, '
    • '); + .replace(/]*)?>/g, '

      ') + .replace(/]*)?>/g, '

      ') + .replace(/]*)?>/g, '

      ') + .replace(/]*)?>/g, '

      ') + .replace(/]*)?>/g, '
      ') + .replace(/]*)?>/g, '
      ') + .replace(/]*)?>/g, '') + .replace(/]*)?>/g, '

      ') + .replace(/]*)?>/g, '

        ') + .replace(/]*)?>/g, '
      • '); return html; } @@ -400,17 +434,23 @@ function processMarkdownToHTML(text: string): string { }, 'Generated HTML'); } - // Add custom classes to elements for styling + // Add custom classes to elements for styling. Tolerate any pre-existing + // attributes on the opening tag (e.g. id="..." that older marked versions + // emit on headings) — without this, the .heading-N classes never apply + // and the headings collapse to body-text size. html = html - .replace(/

        /g, '

        ') - .replace(/

        /g, '

        ') - .replace(/

        /g, '

        ') - .replace(//g, '') - .replace(//g, '') - .replace(/

        /g, '

        ') - .replace(/

          /g, '
            ') - .replace(/
              /g, '
                ') - .replace(/
              1. /g, '
              2. ') + .replace(/]*)?>/g, '

                ') + .replace(/]*)?>/g, '

                ') + .replace(/]*)?>/g, '

                ') + .replace(/]*)?>/g, '

                ') + .replace(/]*)?>/g, '
                ') + .replace(/]*)?>/g, '
                ') + .replace(/]*)?>/g, '') + .replace(/]*)?>/g, '') + .replace(/]*)?>/g, '

                ') + .replace(/]*)?>/g, '

                  ') + .replace(/]*)?>/g, '
                    ') + .replace(/]*)?>/g, '
                  1. ') .replace(/
                    /g, '
                    '); return html; @@ -671,7 +711,7 @@ function generateHTML(data: ShareImageData): string {
                    -
                    ${escapeHtml(data.text)}
                    +
                    ${escapeHtmlAllowInline(data.text)}
                    @@ -691,8 +731,8 @@ function generateHTML(data: ShareImageData): string {
                    [${citation.number}]
                    -
                    ${escapeHtml(citation.title)}
                    -
                    ${escapeHtml(citation.metadata)}
                    +
                    ${escapeHtmlAllowInline(citation.title)}
                    +
                    ${escapeHtmlAllowInline(citation.metadata)}
                    `, @@ -838,6 +878,14 @@ async function useSvgFallback(req: Request, res: async function generateImageFromData(res: Response, data: ShareImageData) { let browser: puppeteer.Browser | null = null; + // Per-step timing — helps diagnose why a single render is 10–45s. + // Each step is logged in milliseconds; the final summary is a single line for grep-ability. + const t0 = Date.now(); + const timings: Record = {}; + const mark = (name: string, since: number) => { + timings[name] = Date.now() - since; + }; + try { // Set response headers for PNG image res.setHeader('Content-Type', 'image/png'); @@ -848,12 +896,15 @@ async function generateImageFromData(res: Response, data: ShareImageData) { : 'public, max-age=21600' // Cache for 6 hours in production ); - // Generate HTML content + // Generate HTML content (markdown processing + template assembly + base64 background image) + const tHtml = Date.now(); const htmlContent = generateHTML(data); + mark('html_build_ms', tHtml); logger.info('Attempting to launch Puppeteer browser...'); // Launch browser with Docker-optimized settings and timeout + const tLaunch = Date.now(); browser = await puppeteer.launch({ headless: true, timeout: 30000, @@ -877,9 +928,11 @@ async function generateImageFromData(res: Response, data: ShareImageData) { // Try to use system Chrome if available, fallback to bundled executablePath: process.env.CHROME_EXECUTABLE_PATH || undefined, }); + mark('puppeteer_launch_ms', tLaunch); logger.info('Browser launched successfully'); + const tNewPage = Date.now(); const page = await browser.newPage(); // Set viewport to match our design @@ -888,26 +941,34 @@ async function generateImageFromData(res: Response, data: ShareImageData) { height: 1024, deviceScaleFactor: 1, }); + mark('new_page_ms', tNewPage); logger.info('Setting page content...'); // Set content and wait for rendering + const tSetContent = Date.now(); await page.setContent(htmlContent, { waitUntil: 'networkidle0', timeout: 30000, }); + mark('set_content_ms', tSetContent); logger.info('Taking screenshot...'); // Take screenshot + const tShot = Date.now(); const screenshot = await page.screenshot({ type: 'png', fullPage: false, }); + mark('screenshot_ms', tShot); // Send the PNG image res.send(screenshot); + timings.total_ms = Date.now() - t0; + const screenshotBytes = (screenshot as any).length ?? (screenshot as any).byteLength ?? 0; + logger.info({ timings, html_chars: htmlContent.length, screenshot_bytes: screenshotBytes }, '[ShareImage] timings'); logger.info('Successfully generated share image with Puppeteer'); } catch (error) { logger.error( diff --git a/openalex-importer/Dockerfile b/openalex-importer/Dockerfile index 9909586a2..4ad690efb 100644 --- a/openalex-importer/Dockerfile +++ b/openalex-importer/Dockerfile @@ -1,4 +1,4 @@ -FROM node:20.18.1-bullseye-slim +FROM node:20.19.1-bullseye-slim RUN apt-get update && apt-get install -y \ jq \ @@ -15,6 +15,7 @@ WORKDIR /app COPY . . RUN npm ci +RUN npm test RUN npm run build CMD [ "npm", "start" ] diff --git a/openalex-importer/drizzle/batches-schema.ts b/openalex-importer/drizzle/batches-schema.ts index 01a3b0930..f2594d5de 100644 --- a/openalex-importer/drizzle/batches-schema.ts +++ b/openalex-importer/drizzle/batches-schema.ts @@ -10,7 +10,9 @@ export const batchesInOpenAlex = openAlexSchema.table("batch", { query_type: text().notNull(), query_from: timestamp({ mode: "date" }).notNull(), query_to: timestamp({ mode: "date" }).notNull(), -}); +}, (table) => [ + index("batch_cleanup_idx").on(table.query_type, table.query_from, table.query_to, table.finished_at), +]); export const workBatchesInOpenAlex = openAlexSchema.table("works_batch", { work_id: text().references(() => worksInOpenalex.id, { diff --git a/openalex-importer/drizzle/migrations/0011_batch_cleanup_idx.sql b/openalex-importer/drizzle/migrations/0011_batch_cleanup_idx.sql new file mode 100644 index 000000000..8aa63ecc4 --- /dev/null +++ b/openalex-importer/drizzle/migrations/0011_batch_cleanup_idx.sql @@ -0,0 +1 @@ +CREATE INDEX "batch_cleanup_idx" ON "openalex"."batch" USING btree ("query_type","query_from","query_to","finished_at"); \ No newline at end of file diff --git a/openalex-importer/drizzle/migrations/meta/0011_snapshot.json b/openalex-importer/drizzle/migrations/meta/0011_snapshot.json new file mode 100644 index 000000000..fb9f156c5 --- /dev/null +++ b/openalex-importer/drizzle/migrations/meta/0011_snapshot.json @@ -0,0 +1,2042 @@ +{ + "id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "prevId": "93eb937d-cf30-42ec-b4eb-70a198ca9681", + "version": "7", + "dialect": "postgresql", + "tables": { + "openalex.authors": { + "name": "authors", + "schema": "openalex", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "orcid": { + "name": "orcid", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "display_name_alternatives": { + "name": "display_name_alternatives", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "works_count": { + "name": "works_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "cited_by_count": { + "name": "cited_by_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_known_institution": { + "name": "last_known_institution", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "works_api_url": { + "name": "works_api_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_date": { + "name": "updated_date", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "openalex.authors_counts_by_year": { + "name": "authors_counts_by_year", + "schema": "openalex", + "columns": { + "author_id": { + "name": "author_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "year": { + "name": "year", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "works_count": { + "name": "works_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "cited_by_count": { + "name": "cited_by_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "oa_works_count": { + "name": "oa_works_count", + "type": "integer", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "authors_counts_by_year_author_id_year_pk": { + "name": "authors_counts_by_year_author_id_year_pk", + "columns": [ + "author_id", + "year" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "openalex.authors_ids": { + "name": "authors_ids", + "schema": "openalex", + "columns": { + "author_id": { + "name": "author_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "openalex": { + "name": "openalex", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "orcid": { + "name": "orcid", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scopus": { + "name": "scopus", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "twitter": { + "name": "twitter", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "wikipedia": { + "name": "wikipedia", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "mag": { + "name": "mag", + "type": "bigint", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "openalex.concepts": { + "name": "concepts", + "schema": "openalex", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "wikidata": { + "name": "wikidata", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "level": { + "name": "level", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "works_count": { + "name": "works_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "cited_by_count": { + "name": "cited_by_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "image_url": { + "name": "image_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "image_thumbnail_url": { + "name": "image_thumbnail_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "works_api_url": { + "name": "works_api_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_date": { + "name": "updated_date", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "descriptions_embeddings": { + "name": "descriptions_embeddings", + "type": "vector(768)", + "primaryKey": false, + "notNull": false + }, + "name_embeddings": { + "name": "name_embeddings", + "type": "vector(768)", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "openalex.concepts_ancestors": { + "name": "concepts_ancestors", + "schema": "openalex", + "columns": { + "concept_id": { + "name": "concept_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ancestor_id": { + "name": "ancestor_id", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "openalex.concepts_counts_by_year": { + "name": "concepts_counts_by_year", + "schema": "openalex", + "columns": { + "concept_id": { + "name": "concept_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "year": { + "name": "year", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "works_count": { + "name": "works_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "cited_by_count": { + "name": "cited_by_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "oa_works_count": { + "name": "oa_works_count", + "type": "integer", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "concepts_counts_by_year_concept_id_year_pk": { + "name": "concepts_counts_by_year_concept_id_year_pk", + "columns": [ + "concept_id", + "year" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "openalex.concepts_ids": { + "name": "concepts_ids", + "schema": "openalex", + "columns": { + "concept_id": { + "name": "concept_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "openalex": { + "name": "openalex", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "wikidata": { + "name": "wikidata", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "wikipedia": { + "name": "wikipedia", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "umls_aui": { + "name": "umls_aui", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "umls_cui": { + "name": "umls_cui", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "mag": { + "name": "mag", + "type": "bigint", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "openalex.concepts_related_concepts": { + "name": "concepts_related_concepts", + "schema": "openalex", + "columns": { + "concept_id": { + "name": "concept_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "related_concept_id": { + "name": "related_concept_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "score": { + "name": "score", + "type": "real", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "concepts_related_concepts_concept_id_related_concept_id_pk": { + "name": "concepts_related_concepts_concept_id_related_concept_id_pk", + "columns": [ + "concept_id", + "related_concept_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "openalex.institutions": { + "name": "institutions", + "schema": "openalex", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "ror": { + "name": "ror", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "country_code": { + "name": "country_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "homepage_url": { + "name": "homepage_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "image_url": { + "name": "image_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "image_thumbnail_url": { + "name": "image_thumbnail_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "display_name_acronyms": { + "name": "display_name_acronyms", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "display_name_alternatives": { + "name": "display_name_alternatives", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "works_count": { + "name": "works_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "cited_by_count": { + "name": "cited_by_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "works_api_url": { + "name": "works_api_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_date": { + "name": "updated_date", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "openalex.institutions_associated_institutions": { + "name": "institutions_associated_institutions", + "schema": "openalex", + "columns": { + "institution_id": { + "name": "institution_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "associated_institution_id": { + "name": "associated_institution_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "relationship": { + "name": "relationship", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "institutions_associated_institutions_institution_id_associated_institution_id_pk": { + "name": "institutions_associated_institutions_institution_id_associated_institution_id_pk", + "columns": [ + "institution_id", + "associated_institution_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "openalex.institutions_counts_by_year": { + "name": "institutions_counts_by_year", + "schema": "openalex", + "columns": { + "institution_id": { + "name": "institution_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "year": { + "name": "year", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "works_count": { + "name": "works_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "cited_by_count": { + "name": "cited_by_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "oa_works_count": { + "name": "oa_works_count", + "type": "integer", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "institutions_counts_by_year_institution_id_year_pk": { + "name": "institutions_counts_by_year_institution_id_year_pk", + "columns": [ + "institution_id", + "year" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "openalex.institutions_geo": { + "name": "institutions_geo", + "schema": "openalex", + "columns": { + "institution_id": { + "name": "institution_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "city": { + "name": "city", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "geonames_city_id": { + "name": "geonames_city_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "country_code": { + "name": "country_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "country": { + "name": "country", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "latitude": { + "name": "latitude", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "longitude": { + "name": "longitude", + "type": "real", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "openalex.institutions_ids": { + "name": "institutions_ids", + "schema": "openalex", + "columns": { + "institution_id": { + "name": "institution_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "openalex": { + "name": "openalex", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ror": { + "name": "ror", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "grid": { + "name": "grid", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "wikipedia": { + "name": "wikipedia", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "wikidata": { + "name": "wikidata", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "mag": { + "name": "mag", + "type": "bigint", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "openalex.publishers": { + "name": "publishers", + "schema": "openalex", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "alternate_titles": { + "name": "alternate_titles", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "country_codes": { + "name": "country_codes", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "hierarchy_level": { + "name": "hierarchy_level", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "parent_publisher": { + "name": "parent_publisher", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "works_count": { + "name": "works_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "cited_by_count": { + "name": "cited_by_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "sources_api_url": { + "name": "sources_api_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_date": { + "name": "updated_date", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "openalex.publishers_counts_by_year": { + "name": "publishers_counts_by_year", + "schema": "openalex", + "columns": { + "publisher_id": { + "name": "publisher_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "year": { + "name": "year", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "works_count": { + "name": "works_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "cited_by_count": { + "name": "cited_by_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "oa_works_count": { + "name": "oa_works_count", + "type": "integer", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "publishers_counts_by_year_publisher_id_year_pk": { + "name": "publishers_counts_by_year_publisher_id_year_pk", + "columns": [ + "publisher_id", + "year" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "openalex.publishers_ids": { + "name": "publishers_ids", + "schema": "openalex", + "columns": { + "publisher_id": { + "name": "publisher_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "openalex": { + "name": "openalex", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ror": { + "name": "ror", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "wikidata": { + "name": "wikidata", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "openalex.sources": { + "name": "sources", + "schema": "openalex", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "issn_l": { + "name": "issn_l", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "issn": { + "name": "issn", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "publisher": { + "name": "publisher", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "works_count": { + "name": "works_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "cited_by_count": { + "name": "cited_by_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "is_oa": { + "name": "is_oa", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "is_in_doaj": { + "name": "is_in_doaj", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "homepage_url": { + "name": "homepage_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "works_api_url": { + "name": "works_api_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_date": { + "name": "updated_date", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "openalex.sources_counts_by_year": { + "name": "sources_counts_by_year", + "schema": "openalex", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "year": { + "name": "year", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "works_count": { + "name": "works_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "cited_by_count": { + "name": "cited_by_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "oa_works_count": { + "name": "oa_works_count", + "type": "integer", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "sources_counts_by_year_source_id_year_pk": { + "name": "sources_counts_by_year_source_id_year_pk", + "columns": [ + "source_id", + "year" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "openalex.sources_ids": { + "name": "sources_ids", + "schema": "openalex", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "openalex": { + "name": "openalex", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "issn_l": { + "name": "issn_l", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "issn": { + "name": "issn", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "mag": { + "name": "mag", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "wikidata": { + "name": "wikidata", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "fatcat": { + "name": "fatcat", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "openalex.topics": { + "name": "topics", + "schema": "openalex", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "subfield_id": { + "name": "subfield_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "subfield_display_name": { + "name": "subfield_display_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "field_id": { + "name": "field_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "field_display_name": { + "name": "field_display_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "domain_id": { + "name": "domain_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "domain_display_name": { + "name": "domain_display_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "keywords": { + "name": "keywords", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "works_api_url": { + "name": "works_api_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "wikipedia_id": { + "name": "wikipedia_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "works_count": { + "name": "works_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "cited_by_count": { + "name": "cited_by_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "updated_date": { + "name": "updated_date", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "siblings": { + "name": "siblings", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "openalex.works": { + "name": "works", + "schema": "openalex", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "doi": { + "name": "doi", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "publication_year": { + "name": "publication_year", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "publication_date": { + "name": "publication_date", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cited_by_count": { + "name": "cited_by_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "is_retracted": { + "name": "is_retracted", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "is_paratext": { + "name": "is_paratext", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "cited_by_api_url": { + "name": "cited_by_api_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "abstract_inverted_index": { + "name": "abstract_inverted_index", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "language": { + "name": "language", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "openalex.works_authorships": { + "name": "works_authorships", + "schema": "openalex", + "columns": { + "work_id": { + "name": "work_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "author_position": { + "name": "author_position", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "author_id": { + "name": "author_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "institution_ids": { + "name": "institution_ids", + "type": "text[]", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "raw_affiliation_string": { + "name": "raw_affiliation_string", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "works_authorships_work_id_author_id_pk": { + "name": "works_authorships_work_id_author_id_pk", + "columns": [ + "work_id", + "author_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "openalex.works_best_oa_locations": { + "name": "works_best_oa_locations", + "schema": "openalex", + "columns": { + "work_id": { + "name": "work_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "landing_page_url": { + "name": "landing_page_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "pdf_url": { + "name": "pdf_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_oa": { + "name": "is_oa", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "license": { + "name": "license", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "openalex.works_biblio": { + "name": "works_biblio", + "schema": "openalex", + "columns": { + "work_id": { + "name": "work_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "volume": { + "name": "volume", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "issue": { + "name": "issue", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "first_page": { + "name": "first_page", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_page": { + "name": "last_page", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "openalex.works_concepts": { + "name": "works_concepts", + "schema": "openalex", + "columns": { + "work_id": { + "name": "work_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "concept_id": { + "name": "concept_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "score": { + "name": "score", + "type": "real", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "works_concepts_work_id_concept_id_pk": { + "name": "works_concepts_work_id_concept_id_pk", + "columns": [ + "work_id", + "concept_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "openalex.works_ids": { + "name": "works_ids", + "schema": "openalex", + "columns": { + "work_id": { + "name": "work_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "openalex": { + "name": "openalex", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "doi": { + "name": "doi", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "mag": { + "name": "mag", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "pmid": { + "name": "pmid", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "pmcid": { + "name": "pmcid", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "openalex.works_locations": { + "name": "works_locations", + "schema": "openalex", + "columns": { + "work_id": { + "name": "work_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "landing_page_url": { + "name": "landing_page_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "pdf_url": { + "name": "pdf_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_oa": { + "name": "is_oa", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "license": { + "name": "license", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "locations_work_id_idx": { + "name": "locations_work_id_idx", + "columns": [ + { + "expression": "work_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "openalex.works_mesh": { + "name": "works_mesh", + "schema": "openalex", + "columns": { + "work_id": { + "name": "work_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "descriptor_ui": { + "name": "descriptor_ui", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "descriptor_name": { + "name": "descriptor_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "qualifier_ui": { + "name": "qualifier_ui", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "qualifier_name": { + "name": "qualifier_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_major_topic": { + "name": "is_major_topic", + "type": "boolean", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "works_mesh_work_id_descriptor_ui_qualifier_ui_pk": { + "name": "works_mesh_work_id_descriptor_ui_qualifier_ui_pk", + "columns": [ + "work_id", + "descriptor_ui", + "qualifier_ui" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "openalex.works_open_access": { + "name": "works_open_access", + "schema": "openalex", + "columns": { + "work_id": { + "name": "work_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "is_oa": { + "name": "is_oa", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "oa_status": { + "name": "oa_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "oa_url": { + "name": "oa_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "any_repository_has_fulltext": { + "name": "any_repository_has_fulltext", + "type": "boolean", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "works_open_access_is_oa_idx": { + "name": "works_open_access_is_oa_idx", + "columns": [ + { + "expression": "is_oa", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "openalex.works_primary_locations": { + "name": "works_primary_locations", + "schema": "openalex", + "columns": { + "work_id": { + "name": "work_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "landing_page_url": { + "name": "landing_page_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "pdf_url": { + "name": "pdf_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_oa": { + "name": "is_oa", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "license": { + "name": "license", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "openalex.works_referenced_works": { + "name": "works_referenced_works", + "schema": "openalex", + "columns": { + "work_id": { + "name": "work_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "referenced_work_id": { + "name": "referenced_work_id", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "works_referenced_works_work_id_referenced_work_id_pk": { + "name": "works_referenced_works_work_id_referenced_work_id_pk", + "columns": [ + "work_id", + "referenced_work_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "openalex.works_related_works": { + "name": "works_related_works", + "schema": "openalex", + "columns": { + "work_id": { + "name": "work_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "related_work_id": { + "name": "related_work_id", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "works_related_works_work_id_related_work_id_pk": { + "name": "works_related_works_work_id_related_work_id_pk", + "columns": [ + "work_id", + "related_work_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "openalex.works_topics": { + "name": "works_topics", + "schema": "openalex", + "columns": { + "work_id": { + "name": "work_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "topic_id": { + "name": "topic_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "score": { + "name": "score", + "type": "real", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "works_topics_work_id_topic_id_pk": { + "name": "works_topics_work_id_topic_id_pk", + "columns": [ + "work_id", + "topic_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "openalex.batch": { + "name": "batch", + "schema": "openalex", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "query_type": { + "name": "query_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "query_from": { + "name": "query_from", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "query_to": { + "name": "query_to", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "batch_cleanup_idx": { + "name": "batch_cleanup_idx", + "columns": [ + { + "expression": "query_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "query_from", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "query_to", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "finished_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "openalex.works_batch": { + "name": "works_batch", + "schema": "openalex", + "columns": { + "work_id": { + "name": "work_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "batch_id": { + "name": "batch_id", + "type": "integer", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "works_batch_work_id_works_id_fk": { + "name": "works_batch_work_id_works_id_fk", + "tableFrom": "works_batch", + "tableTo": "works", + "schemaTo": "openalex", + "columnsFrom": [ + "work_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "cascade" + }, + "works_batch_batch_id_batch_id_fk": { + "name": "works_batch_batch_id_batch_id_fk", + "tableFrom": "works_batch", + "tableTo": "batch", + "schemaTo": "openalex", + "columnsFrom": [ + "batch_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "works_batch_work_id_batch_id_pk": { + "name": "works_batch_work_id_batch_id_pk", + "columns": [ + "work_id", + "batch_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": { + "openalex": "openalex" + }, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/openalex-importer/drizzle/migrations/meta/_journal.json b/openalex-importer/drizzle/migrations/meta/_journal.json index c599ff119..06dbbaffe 100644 --- a/openalex-importer/drizzle/migrations/meta/_journal.json +++ b/openalex-importer/drizzle/migrations/meta/_journal.json @@ -78,6 +78,13 @@ "when": 1747667888986, "tag": "0010_nappy_dormammu", "breakpoints": true + }, + { + "idx": 11, + "version": "7", + "when": 1749177600000, + "tag": "0011_batch_cleanup_idx", + "breakpoints": true } ] } \ No newline at end of file diff --git a/openalex-importer/index.ts b/openalex-importer/index.ts index ec96ea7e8..d158280f9 100644 --- a/openalex-importer/index.ts +++ b/openalex-importer/index.ts @@ -1,13 +1,14 @@ import 'dotenv/config'; -import { addDays, differenceInDays, endOfDay, isAfter, isSameDay, startOfDay } from 'date-fns'; +import { addDays, differenceInDays, endOfDay, isAfter, isSameDay, startOfDay, subDays } from 'date-fns'; import { logger } from './src/logger.js'; import { db, getNextDayToImport, type OaDb } from './src/db/index.js'; import { type QueryInfo } from './src/db/types.js'; -import { type Optional, parseDate } from './src/util.js'; +import { type Optional, parseDate, dropTime } from './src/util.js'; import { errWithCause } from 'pino-std-serializers'; import { UTCDate } from '@date-fns/utc'; import { runImportPipeline } from './src/pipeline.js'; import { Cron } from 'croner'; +import { markImporting, notifyCaughtUp, notifyError, sendDailyDigest, sendTelegram, startCommandListener, stopCommandListener } from './src/notifier.js'; export const MAX_PAGES_TO_FETCH = parseInt(process.env.MAX_PAGES_TO_FETCH || '100'); export const IS_DEV = process.env.NODE_ENV === 'development'; @@ -24,9 +25,13 @@ const runImportTask = async (db: OaDb, query_type: QueryInfo['query_type']) => { const currentDate: UTCDate = new UTCDate(); if (isSameDay(nextDay, currentDate) || isAfter(nextDay, currentDate)) { logger.info({ nextDay, currentDate }, '💤 Next day to import is today or in the future, snoozing...'); + const syncedTo = dropTime(subDays(nextDay, 1).toISOString()); + await notifyCaughtUp(syncedTo); return; } + markImporting(); + const importParams: QueryInfo = { query_from: startOfDay(nextDay), query_to: endOfDay(nextDay), @@ -64,14 +69,21 @@ async function main(): Promise { protect: () => { logger.info('💤 Recurring task invoked while an import is already running, snoozing...'); }, - // For some reason this doesn't trigger if the stream callbacks catches errors, but the app exits anyway so OK - catch: (e, job) => { - logger.error({ error: errWithCause(e as Error) }, '💥 Cron job caught an error'); - job.stop(); - throw e; + catch: async (e) => { + const err = e as Error; + logger.error({ error: errWithCause(err) }, '💥 Cron job caught an error, will retry on next schedule tick'); + await notifyError(err, 'Recurring import task'); }, }); + const digestSchedule = process.env.DIGEST_SCHEDULE ?? '0 9 * * *'; + const _digestJob = new Cron(digestSchedule, async () => { + logger.info('📊 Sending daily digest...'); + await sendDailyDigest(db); + }, { timezone: 'UTC' }); + + startCommandListener(db); + // Kick off first run right away void job.trigger(); } else if (args.query_from) { @@ -201,37 +213,45 @@ const getRuntimeArgs = (): RuntimeArgs => { return args as RuntimeArgs; }; -let isPoolEnded = false; +let isShuttingDown = false; -const endPool = async () => { - if (!isPoolEnded) { - isPoolEnded = true; - await db.$pool.end(); - } +const withTimeout = (promise: Promise, ms: number): Promise => + Promise.race([promise, new Promise(r => setTimeout(r, ms))]); + +const shutdown = async (reason: string, exitCode: number) => { + if (isShuttingDown) return; + isShuttingDown = true; + + stopCommandListener(); + await withTimeout( + sendTelegram(`🔴 OpenAlex Importer shutting down\nReason: ${reason}`), + 5_000, + ).catch(() => {}); + await withTimeout(db.$pool.end(), 5_000).catch(() => {}); + process.exit(exitCode); }; process.on('uncaughtException', async (err) => { logger.fatal(errWithCause(err), 'uncaught exception'); - await endPool(); - process.exit(1); + await withTimeout(notifyError(err, 'Uncaught exception'), 5_000).catch(() => {}); + await shutdown('uncaught exception', 1); }); -process.on("SIGTERM", async () => { - logger.info("Received SIGTERM signal. Shutting down pool..."); - await endPool(); - process.exit(1); +process.on('unhandledRejection', async (reason) => { + const err = reason instanceof Error ? reason : new Error(String(reason)); + logger.fatal({ err }, 'unhandled rejection'); + await withTimeout(notifyError(err, 'Unhandled rejection'), 5_000).catch(() => {}); + await shutdown('unhandled rejection', 1); }); -process.on("SIGINT", async () => { - logger.info("Received SIGINT signal. Shutting down pool..."); - await endPool(); - process.exit(1); +process.on("SIGTERM", () => { + logger.info("Received SIGTERM signal. Shutting down..."); + void shutdown('SIGTERM (K8s rollout or scale-down)', 0); }); -process.on('beforeExit', async () => { - logger.info('Process exiting, shutting down pool...') - await endPool(); - process.exit(0); +process.on("SIGINT", () => { + logger.info("Received SIGINT signal. Shutting down..."); + void shutdown('SIGINT', 0); }); /** diff --git a/openalex-importer/src/__fixtures__/works.ts b/openalex-importer/src/__fixtures__/works.ts new file mode 100644 index 000000000..0cdff51f1 --- /dev/null +++ b/openalex-importer/src/__fixtures__/works.ts @@ -0,0 +1,283 @@ +import type { Work } from '../types/works.js'; + +/** + * Minimal valid Work as returned by the OpenAlex API. + * All required arrays are present; optional sub-objects are populated. + */ +export const minimalWork: Work = { + id: 'https://openalex.org/W1234567890', + doi: 'https://doi.org/10.1234/test.2024.001', + title: 'A Minimal Test Work', + display_name: 'A Minimal Test Work', + publication_year: 2024, + publication_date: '2024-03-15', + language: 'en', + type: 'journal-article', + type_crossref: 'journal-article', + cited_by_count: 5, + is_retracted: false, + is_paratext: false, + cited_by_api_url: 'https://api.openalex.org/works?filter=cites:W1234567890', + abstract_inverted_index: { test: [0], abstract: [1] }, + ids: { + openalex: 'https://openalex.org/W1234567890', + doi: 'https://doi.org/10.1234/test.2024.001', + mag: 1234567890, + pmid: null, + pmcid: null, + }, + primary_location: { + is_oa: true, + landing_page_url: 'https://example.com/article', + pdf_url: 'https://example.com/article.pdf', + source: { + id: 'https://openalex.org/S100', + display_name: 'Test Journal', + issn_l: '1234-5678', + issn: ['1234-5678'], + is_oa: true, + is_in_doaj: true, + is_core: false, + host_organization: 'https://openalex.org/P1', + host_organization_name: 'Test Publisher', + host_organization_lineage: [], + host_organization_lineage_names: [], + type: 'journal', + }, + license: 'cc-by', + license_id: 'https://openalex.org/licenses/cc-by', + version: 'publishedVersion', + is_accepted: true, + is_published: true, + }, + best_oa_location: { + is_oa: true, + landing_page_url: 'https://example.com/article', + pdf_url: 'https://example.com/article.pdf', + source: { + id: 'https://openalex.org/S100', + display_name: 'Test Journal', + issn_l: null, + issn: null, + is_oa: true, + is_in_doaj: true, + is_core: false, + host_organization: null, + host_organization_name: null, + host_organization_lineage: [], + host_organization_lineage_names: [], + type: 'journal', + }, + license: 'cc-by', + license_id: 'https://openalex.org/licenses/cc-by', + version: 'publishedVersion', + is_accepted: true, + is_published: true, + }, + open_access: { + is_oa: true, + oa_status: 'gold', + oa_url: 'https://example.com/article', + any_repository_has_fulltext: false, + }, + authorships: [ + { + author_position: 'first', + author: { + id: 'https://openalex.org/A001', + display_name: 'Jane Doe', + orcid: 'https://orcid.org/0000-0001-2345-6789', + twitter: null, + scopus: null, + wikipedia: null, + mag: null, + }, + institutions: [ + { + id: 'https://openalex.org/I001', + ror: 'https://ror.org/test001', + display_name: 'Test University', + country_code: 'US', + type: 'education', + type_id: 'https://openalex.org/institution-types/education', + lineage: [], + homepage_url: 'https://test.edu', + image_url: '', + image_thumbnail_url: '', + display_name_acronyms: [], + display_name_alternatives: [], + repositories: [], + works_count: 1000, + cited_by_count: 5000, + summary_stats: { '2yr_mean_citedness': 2.5, h_index: 50, i10_index: 100 }, + ids: { openalex: 'I001' }, + geo: { city: 'Test City', geonames_city_id: '123', region: null, country_code: 'US', country: 'United States', latitude: 40.0, longitude: -74.0 }, + international: { display_name: {} as any }, + associated_institutions: [], + counts_by_year: [], + roles: [], + topics: [], + topic_share: [], + x_concepts: [], + is_super_system: false, + works_api_url: '', + updated_date: '2024-01-01', + created_date: '2020-01-01', + }, + ], + countries: ['US'], + is_corresponding: true, + raw_author_name: 'Jane Doe', + raw_affiliation_strings: [], + affiliations: [], + }, + ], + biblio: { + volume: '42', + issue: '3', + first_page: '100', + last_page: '110', + }, + concepts: [ + { id: 'https://openalex.org/C100', wikidata: null, display_name: 'Computer Science', level: 0, score: 0.85, description: null, works_count: null, cited_by_count: null, image_url: null, image_thumbnail_url: null, works_api_url: null, updated_date: null }, + ], + topics: [ + { id: 'https://openalex.org/T100', display_name: 'Machine Learning', count: 100, subfield: { id: 'S1', display_name: 'AI' }, field: { id: 'F1', display_name: 'CS' }, domain: { id: 'D1', display_name: 'Sciences' }, score: 0.9 } as any, + ], + mesh: [ + { descriptor_ui: 'D000001', descriptor_name: 'Calcimycin', qualifier_ui: 'Q000031', qualifier_name: 'analysis', is_major_topic: true }, + ], + locations: [ + { + is_oa: true, + landing_page_url: 'https://example.com/article', + pdf_url: 'https://example.com/article.pdf', + source: { id: 'https://openalex.org/S100', display_name: 'Test Journal', issn_l: null, issn: null, is_oa: true, is_in_doaj: true, is_core: false, host_organization: null, host_organization_name: null, host_organization_lineage: [], host_organization_lineage_names: [], type: 'journal' }, + license: 'cc-by', + license_id: 'cc-by', + version: 'publishedVersion', + is_accepted: true, + is_published: true, + }, + ], + referenced_works: ['https://openalex.org/W9999'], + related_works: ['https://openalex.org/W8888'], + indexed_in: ['crossref'], + countries_distinct_count: 1, + institutions_distinct_count: 1, + corresponding_author_ids: [], + corresponding_institution_ids: [], + apc_list: null, + apc_paid: null, + fwci: 1.5, + has_fulltext: true, + fulltext_origin: 'pdf', + cited_by_percentile_year: { min: 80, max: 90 }, + primary_topic: null, + keywords: [], + locations_count: 1, + sustainable_development_goals: [], + grants: [], + datasets: [], + versions: [], + referenced_works_count: 1, + ngrams_url: '', + counts_by_year: [{ year: 2024, cited_by_count: 5 }], + updated_date: '2024-03-20', + created_date: '2024-03-15', +}; + +/** + * Reproduces the exact crash scenario: mesh entries where OpenAlex returns + * null/empty qualifier_ui values. The transformer passes these through as-is, + * and the DB layer must handle the resulting null PKs gracefully. + */ +export const workWithNullMeshQualifiers: Work = { + ...minimalWork, + id: 'https://openalex.org/W1111111111', + doi: 'https://doi.org/10.1234/null-mesh', + mesh: [ + { descriptor_ui: 'D000001', descriptor_name: 'Calcimycin', qualifier_ui: null as any, qualifier_name: null as any, is_major_topic: true }, + { descriptor_ui: 'D000002', descriptor_name: 'Temefos', qualifier_ui: null as any, qualifier_name: null as any, is_major_topic: false }, + { descriptor_ui: null as any, descriptor_name: null as any, qualifier_ui: 'Q000031', qualifier_name: 'analysis', is_major_topic: true }, + ], +}; + +/** + * Work with no primary_location or best_oa_location (common for preprints/datasets). + */ +export const workWithNoPrimaryLocation: Work = { + ...minimalWork, + id: 'https://openalex.org/W2222222222', + doi: null as any, + primary_location: null as any, + best_oa_location: null as any, + locations: [], +}; + +/** + * Work with empty arrays for all child entities — the "bare bones" response. + */ +export const workWithEmptyArrays: Work = { + ...minimalWork, + id: 'https://openalex.org/W3333333333', + authorships: [], + concepts: [], + topics: [], + mesh: [], + locations: [], + referenced_works: [], + related_works: [], +}; + +/** + * Work with duplicate authorships (same author appears twice, e.g. multi-affiliation). + */ +export const workWithDuplicateAuthorships: Work = { + ...minimalWork, + id: 'https://openalex.org/W4444444444', + authorships: [ + minimalWork.authorships[0], + { + ...minimalWork.authorships[0], + institutions: [ + { + ...minimalWork.authorships[0].institutions[0], + id: 'https://openalex.org/I002', + display_name: 'Second University', + }, + ], + }, + ], +}; + +/** + * Work with concepts/topics that have null IDs (seen in older OpenAlex data). + */ +export const workWithNullConceptAndTopicIds: Work = { + ...minimalWork, + id: 'https://openalex.org/W5555555555', + concepts: [ + { id: null as any, wikidata: null, display_name: 'Unknown', level: 0, score: 0.5, description: null, works_count: null, cited_by_count: null, image_url: null, image_thumbnail_url: null, works_api_url: null, updated_date: null }, + { id: 'https://openalex.org/C200', wikidata: null, display_name: 'Biology', level: 0, score: 0.8, description: null, works_count: null, cited_by_count: null, image_url: null, image_thumbnail_url: null, works_api_url: null, updated_date: null }, + ], + topics: [ + { id: null as any, display_name: 'Unknown Topic', count: 0, subfield: { id: 'S1', display_name: 'AI' }, field: { id: 'F1', display_name: 'CS' }, domain: { id: 'D1', display_name: 'Sciences' }, score: 0.3 } as any, + { id: 'https://openalex.org/T200', display_name: 'Genomics', count: 50, subfield: { id: 'S2', display_name: 'Bio' }, field: { id: 'F2', display_name: 'Life' }, domain: { id: 'D2', display_name: 'Life Sciences' }, score: 0.7 } as any, + ], +}; + +/** + * Work where authorships have null author IDs (corrupt data from API). + */ +export const workWithNullAuthorIds: Work = { + ...minimalWork, + id: 'https://openalex.org/W6666666666', + authorships: [ + { + ...minimalWork.authorships[0], + author: { ...minimalWork.authorships[0].author, id: null as any }, + }, + minimalWork.authorships[0], + ], +}; diff --git a/openalex-importer/src/db/dedup.test.ts b/openalex-importer/src/db/dedup.test.ts index a5ba9547b..c8decd85c 100644 --- a/openalex-importer/src/db/dedup.test.ts +++ b/openalex-importer/src/db/dedup.test.ts @@ -83,6 +83,17 @@ describe('deduplicateWorksConcepts', () => { expect(deduplicateWorksConcepts([])).toEqual([]); }); + it('returns empty array when all rows have null primary keys', () => { + const data = [ + { work_id: null, concept_id: 'C1', score: 0.5 }, + { work_id: 'W1', concept_id: null, score: 0.6 }, + { work_id: null, concept_id: null, score: 0.7 }, + ] as any; + + const result = deduplicateWorksConcepts(data); + expect(result).toEqual([]); + }); + it('filters out rows with null work_id', () => { const data = [ { work_id: null, concept_id: 'C1', score: 0.5 }, @@ -169,6 +180,16 @@ describe('deduplicateWorksTopics', () => { it('returns empty array for empty input', () => { expect(deduplicateWorksTopics([])).toEqual([]); }); + + it('returns empty array when all rows have null primary keys', () => { + const data = [ + { work_id: null, topic_id: 'T1', score: 0.5 }, + { work_id: 'W1', topic_id: null, score: 0.6 }, + ] as any; + + const result = deduplicateWorksTopics(data); + expect(result).toEqual([]); + }); }); describe('deduplicateWorksMesh', () => { @@ -221,4 +242,15 @@ describe('deduplicateWorksMesh', () => { it('returns empty array for empty input', () => { expect(deduplicateWorksMesh([])).toEqual([]); }); + + it('returns empty array when all rows have null primary keys', () => { + const data = [ + { work_id: null, descriptor_ui: 'D001', descriptor_name: 'Name1', qualifier_ui: 'Q001', qualifier_name: 'QName1', is_major_topic: true }, + { work_id: 'W1', descriptor_ui: null, descriptor_name: 'Name1', qualifier_ui: 'Q001', qualifier_name: 'QName1', is_major_topic: true }, + { work_id: 'W1', descriptor_ui: 'D001', descriptor_name: 'Name1', qualifier_ui: null, qualifier_name: 'QName1', is_major_topic: true }, + ] as any; + + const result = deduplicateWorksMesh(data); + expect(result).toEqual([]); + }); }); diff --git a/openalex-importer/src/db/index.ts b/openalex-importer/src/db/index.ts index 23fcf324f..e8f78c824 100644 --- a/openalex-importer/src/db/index.ts +++ b/openalex-importer/src/db/index.ts @@ -95,16 +95,64 @@ export const db = pgp(dbInfo); export type OaDb = typeof db; -export const createBatch = async (tx: pgPromise.ITask, queryInfo: QueryInfo) => { - const savedBatch = await tx.one( +export const createBatch = async (db: OaDb, queryInfo: QueryInfo) => { + const savedBatch = await db.one( 'INSERT INTO openalex.batch (query_type, query_from, query_to) VALUES ($1, $2, $3) RETURNING id', [queryInfo.query_type, queryInfo.query_from, queryInfo.query_to] ); return savedBatch.id; }; -export const finalizeBatch = async (tx: pgPromise.ITask, batchId: number) => - await tx.none('UPDATE openalex.batch SET finished_at = $1 WHERE id = $2', [new UTCDate(), batchId]); +export const finalizeBatch = async (db: OaDb, batchId: number) => + await db.none('UPDATE openalex.batch SET finished_at = $1 WHERE id = $2', [new UTCDate(), batchId]); + +/** + * Check if there are unfinished batches from a previous crash (without cleaning them up). + * Intentionally global (not scoped to a specific day) — used only at startup to detect + * any crash evidence and send the "recovering" notification. Cleanup is day-scoped separately. + */ +export const hasUnfinishedBatches = async (db: OaDb): Promise => { + const result = await db.oneOrNone( + 'SELECT 1 FROM openalex.batch WHERE finished_at IS NULL LIMIT 1', + ); + return result !== null; +}; + +/** + * Clean up unfinished batches for a specific day only. + * Intentionally scoped to queryInfo's day — other days may be in-progress + * via time-travel mode or a parallel process. + * + * Uses FOR UPDATE SKIP LOCKED to avoid deleting batches that are actively + * being written to by a concurrent process (e.g. during rolling restarts). + * Only targets batches older than 5 minutes to avoid racing with freshly + * created batches. + */ +export const cleanupUnfinishedBatches = async (db: OaDb, queryInfo: QueryInfo) => { + await db.tx(async (tx) => { + const unfinished = await tx.manyOrNone( + `SELECT id FROM openalex.batch + WHERE query_type = $1 AND query_from = $2 AND query_to = $3 + AND finished_at IS NULL + AND started_at < NOW() - INTERVAL '5 minutes' + FOR UPDATE SKIP LOCKED`, + [queryInfo.query_type, queryInfo.query_from, queryInfo.query_to] + ); + + if (unfinished.length === 0) return; + + const batchIds = unfinished.map(r => r.id); + logger.info({ batchIds, queryInfo }, 'Cleaning up unfinished batches from previous crash'); + + // Delete works_batch rows for these batches first (avoid orphaned NULL rows) + await tx.none('DELETE FROM openalex.works_batch WHERE batch_id IN ($1:list)', [batchIds]); + + // Then delete the batch records themselves + await tx.none('DELETE FROM openalex.batch WHERE id IN ($1:list)', [batchIds]); + + logger.info({ batchIds }, 'Cleanup complete'); + }); +}; export const saveData = async (tx: pgPromise.ITask, batchId: number, models: DataModels) => { const counts = Object.entries(models).reduce((acc, [k, a]) => ({ ...acc, [k]: a.length}), {}); @@ -398,6 +446,8 @@ const updateWorksConcepts = async (tx: pgPromise.ITask, data: DataModels['w if (!data.length) return; const deduped = deduplicateWorksConcepts(data); + if (!deduped.length) return; + const columns = getColumnSet(works_conceptsInOpenalex); const query = pgp.helpers.insert(deduped.sort(sortWorksConcepts), columns) + ' ON CONFLICT (work_id, concept_id) DO UPDATE SET ' + @@ -423,6 +473,8 @@ const updateWorksMesh = async (tx: pgPromise.ITask, data: DataModels['works if (!data.length) return; const deduped = deduplicateWorksMesh(data); + if (!deduped.length) return; + const columns = getColumnSet(works_meshInOpenalex); const query = pgp.helpers.insert(deduped.sort(sortWorksMesh), columns) + ' ON CONFLICT (work_id, descriptor_ui, qualifier_ui) DO UPDATE SET ' + @@ -444,6 +496,8 @@ const updateWorksTopics = async (tx: pgPromise.ITask, data: DataModels['wor if (!data.length) return; const deduped = deduplicateWorksTopics(data); + if (!deduped.length) return; + const columns = getColumnSet(works_topicsInOpenalex); const query = pgp.helpers.insert(deduped.sort(sortWorksTopics), columns) + ' ON CONFLICT (work_id, topic_id) DO UPDATE SET ' + @@ -457,7 +511,7 @@ const updateWorksTopics = async (tx: pgPromise.ITask, data: DataModels['wor */ export const getNextDayToImport = async (queryType: QueryInfo['query_type']): Promise => { const lastBatchEnd = await db.oneOrNone( - 'SELECT query_to FROM openalex.batch WHERE query_type = $1 ORDER BY query_to DESC LIMIT 1', + 'SELECT query_to FROM openalex.batch WHERE query_type = $1 AND finished_at IS NOT NULL ORDER BY query_to DESC LIMIT 1', [queryType] ); diff --git a/openalex-importer/src/db/saveData.test.ts b/openalex-importer/src/db/saveData.test.ts new file mode 100644 index 000000000..41449960f --- /dev/null +++ b/openalex-importer/src/db/saveData.test.ts @@ -0,0 +1,98 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { transformDataModel, type DataModels } from '../transformers.js'; +import { + minimalWork, + workWithNullMeshQualifiers, + workWithNoPrimaryLocation, + workWithEmptyArrays, + workWithNullConceptAndTopicIds, + workWithNullAuthorIds, +} from '../__fixtures__/works.js'; + +vi.mock('../logger.js', () => ({ + logger: { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + debug: vi.fn(), + }, +})); + +/** + * These tests verify that saveData doesn't throw on edge-case DataModels. + * + * We use the real pgp.helpers.insert() for SQL generation, but swap the DB + * transaction with a mock so no actual database is needed. saveData receives + * mockTx directly, so the generated SQL is passed to mockTx.none() and discarded. + */ + +const mockTxNone = vi.fn().mockResolvedValue(undefined); +const mockTxOne = vi.fn().mockResolvedValue({ id: 1 }); +const mockTxOneOrNone = vi.fn().mockResolvedValue(null); + +const mockTx = { + none: mockTxNone, + one: mockTxOne, + oneOrNone: mockTxOneOrNone, +} as any; + +vi.mock('pg-promise', () => vi.importActual('pg-promise')); + +let saveData: typeof import('./index.js')['saveData']; + +beforeEach(async () => { + vi.clearAllMocks(); + mockTxNone.mockResolvedValue(undefined); + + const mod = await import('./index.js'); + saveData = mod.saveData; +}); + +describe('saveData guards', () => { + it('handles a normal work batch without throwing', async () => { + const models = transformDataModel([minimalWork]); + await expect(saveData(mockTx, 1, models)).resolves.toBeUndefined(); + expect(mockTxNone).toHaveBeenCalled(); + }); + + it('handles a batch where all mesh rows have null PKs (the production crash scenario)', async () => { + const models = transformDataModel([workWithNullMeshQualifiers]); + await expect(saveData(mockTx, 1, models)).resolves.toBeUndefined(); + }); + + it('handles a work with no primary_location or best_oa_location', async () => { + const models = transformDataModel([workWithNoPrimaryLocation]); + await expect(saveData(mockTx, 1, models)).resolves.toBeUndefined(); + }); + + it('handles a work with all empty child arrays', async () => { + const models = transformDataModel([workWithEmptyArrays]); + await expect(saveData(mockTx, 1, models)).resolves.toBeUndefined(); + }); + + it('handles works with null concept and topic IDs', async () => { + const models = transformDataModel([workWithNullConceptAndTopicIds]); + await expect(saveData(mockTx, 1, models)).resolves.toBeUndefined(); + }); + + it('handles works with null author IDs', async () => { + const models = transformDataModel([workWithNullAuthorIds]); + await expect(saveData(mockTx, 1, models)).resolves.toBeUndefined(); + }); + + it('handles completely empty DataModels', async () => { + const models = transformDataModel([]); + await expect(saveData(mockTx, 1, models)).resolves.toBeUndefined(); + }); + + it('handles a mixed batch of clean and problematic works', async () => { + const models = transformDataModel([ + minimalWork, + workWithNullMeshQualifiers, + workWithNoPrimaryLocation, + workWithEmptyArrays, + workWithNullConceptAndTopicIds, + ]); + await expect(saveData(mockTx, 1, models)).resolves.toBeUndefined(); + }); +}); diff --git a/openalex-importer/src/notifier.test.ts b/openalex-importer/src/notifier.test.ts new file mode 100644 index 000000000..350ea94a4 --- /dev/null +++ b/openalex-importer/src/notifier.test.ts @@ -0,0 +1,439 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; + +vi.mock('./logger.js', () => ({ + logger: { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + debug: vi.fn(), + }, +})); + +const fetchSpy = vi.fn().mockResolvedValue({ ok: true, text: async () => '{}' }); +vi.stubGlobal('fetch', fetchSpy); + +let sendTelegram: typeof import('./notifier.js')['sendTelegram']; +let notifyError: typeof import('./notifier.js')['notifyError']; +let notifyCaughtUp: typeof import('./notifier.js')['notifyCaughtUp']; +let markImporting: typeof import('./notifier.js')['markImporting']; +let sendDailyDigest: typeof import('./notifier.js')['sendDailyDigest']; +let buildDigestMessage: typeof import('./notifier.js')['buildDigestMessage']; +let buildPipelineStatus: typeof import('./notifier.js')['buildPipelineStatus']; +let startCommandListener: typeof import('./notifier.js')['startCommandListener']; +let stopCommandListener: typeof import('./notifier.js')['stopCommandListener']; + +beforeEach(async () => { + vi.resetModules(); + vi.stubEnv('TELEGRAM_BOT_TOKEN', 'test-token'); + vi.stubEnv('TELEGRAM_CHAT_ID', '-100123456'); + fetchSpy.mockReset(); + fetchSpy.mockResolvedValue({ ok: true, text: async () => '{}' }); + + const mod = await import('./notifier.js'); + sendTelegram = mod.sendTelegram; + notifyError = mod.notifyError; + notifyCaughtUp = mod.notifyCaughtUp; + markImporting = mod.markImporting; + sendDailyDigest = mod.sendDailyDigest; + buildDigestMessage = mod.buildDigestMessage; + buildPipelineStatus = mod.buildPipelineStatus; + startCommandListener = mod.startCommandListener; + stopCommandListener = mod.stopCommandListener; +}); + +afterEach(() => { + vi.unstubAllEnvs(); +}); + +describe('sendTelegram', () => { + it('sends a message when env vars are configured', async () => { + const result = await sendTelegram('test message'); + expect(result).toBe(true); + expect(fetchSpy).toHaveBeenCalledOnce(); + + const [url, opts] = fetchSpy.mock.calls[0]; + expect(url).toContain('test-token'); + const body = JSON.parse(opts.body); + expect(body.chat_id).toBe('-100123456'); + expect(body.text).toBe('test message'); + expect(body.parse_mode).toBe('HTML'); + }); + + it('includes message_thread_id when TELEGRAM_THREAD_ID is set', async () => { + vi.stubEnv('TELEGRAM_THREAD_ID', '13983'); + vi.resetModules(); + const mod = await import('./notifier.js'); + + await mod.sendTelegram('threaded message'); + const body = JSON.parse(fetchSpy.mock.calls[0][1].body); + expect(body.message_thread_id).toBe(13983); + }); + + it('no-ops and returns false when env vars are missing', async () => { + vi.stubEnv('TELEGRAM_BOT_TOKEN', ''); + vi.resetModules(); + const mod = await import('./notifier.js'); + + const result = await mod.sendTelegram('should not send'); + expect(result).toBe(false); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it('returns false on API error without throwing', async () => { + fetchSpy.mockResolvedValueOnce({ ok: false, status: 403, text: async () => 'Forbidden' }); + const result = await sendTelegram('test'); + expect(result).toBe(false); + }); + + it('returns false on network error without throwing', async () => { + fetchSpy.mockRejectedValueOnce(new Error('network down')); + const result = await sendTelegram('test'); + expect(result).toBe(false); + }); +}); + +describe('notifyError', () => { + it('sends on first error', async () => { + await notifyError(new Error('test failure')); + expect(fetchSpy).toHaveBeenCalledOnce(); + const body = JSON.parse(fetchSpy.mock.calls[0][1].body); + expect(body.text).toContain('test failure'); + expect(body.text).toContain('Error'); + }); + + it('includes context in the message', async () => { + await notifyError(new Error('boom'), 'during mesh insert'); + const body = JSON.parse(fetchSpy.mock.calls[0][1].body); + expect(body.text).toContain('during mesh insert'); + }); + + it('rate-limits: second error within cooldown is suppressed', async () => { + await notifyError(new Error('first')); + await notifyError(new Error('second')); + expect(fetchSpy).toHaveBeenCalledOnce(); + }); + + it('sends again after cooldown expires', async () => { + const baseTime = 1_700_000_000_000; + const dateNowSpy = vi.spyOn(Date, 'now'); + + dateNowSpy.mockReturnValue(baseTime); + await notifyError(new Error('first')); + expect(fetchSpy).toHaveBeenCalledOnce(); + + dateNowSpy.mockReturnValue(baseTime + 61 * 60 * 1000); + await notifyError(new Error('after cooldown')); + expect(fetchSpy).toHaveBeenCalledTimes(2); + + dateNowSpy.mockRestore(); + }); + + it('truncates very long error messages', async () => { + const longMsg = 'x'.repeat(500); + await notifyError(new Error(longMsg)); + const body = JSON.parse(fetchSpy.mock.calls[0][1].body); + expect(body.text.length).toBeLessThan(500); + }); +}); + +describe('notifyCaughtUp', () => { + it('does not notify when already caught up (initial state)', async () => { + await notifyCaughtUp('2024-03-15'); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it('notifies on transition from importing to caught up', async () => { + markImporting(); + await notifyCaughtUp('2024-03-15'); + expect(fetchSpy).toHaveBeenCalledOnce(); + const body = JSON.parse(fetchSpy.mock.calls[0][1].body); + expect(body.text).toContain('2024-03-15'); + expect(body.text).toContain('caught up'); + }); + + it('does not double-notify if called again while still caught up', async () => { + markImporting(); + await notifyCaughtUp('2024-03-15'); + await notifyCaughtUp('2024-03-15'); + expect(fetchSpy).toHaveBeenCalledOnce(); + }); + + it('notifies again after another import cycle', async () => { + markImporting(); + await notifyCaughtUp('2024-03-15'); + markImporting(); + await notifyCaughtUp('2024-03-16'); + expect(fetchSpy).toHaveBeenCalledTimes(2); + }); +}); + +describe('sendDailyDigest', () => { + it('sends a digest with data from DB queries', async () => { + const mockDb = { + oneOrNone: vi.fn() + .mockResolvedValueOnce({ query_to: new Date('2024-03-15') }) + .mockResolvedValueOnce({ days_imported: '3', total_duration_sec: '125' }) + .mockResolvedValueOnce({ failed_count: '0' }) + .mockResolvedValueOnce({ finished_at: new Date('2024-03-15T10:30:00Z') }) + .mockResolvedValueOnce({ total_works: '5000000', works_24h: '45000', works_30d: '900000' }), + } as any; + + await sendDailyDigest(mockDb); + expect(fetchSpy).toHaveBeenCalledOnce(); + const body = JSON.parse(fetchSpy.mock.calls[0][1].body); + expect(body.text).toContain('Status'); + expect(body.text).toContain('2024-03-15'); + expect(body.text).toContain('3 day'); + expect(body.text).toContain('Last import'); + expect(body.text).toContain('45.0K works'); + expect(body.text).toContain('900.0K works'); + expect(body.text).toContain('5.0M works'); + }); + + it('shows warning for failed batches', async () => { + const mockDb = { + oneOrNone: vi.fn() + .mockResolvedValueOnce({ query_to: new Date('2024-03-10') }) + .mockResolvedValueOnce({ days_imported: '1', total_duration_sec: '60' }) + .mockResolvedValueOnce({ failed_count: '5' }) + .mockResolvedValueOnce({ finished_at: new Date('2024-03-10T08:00:00Z') }) + .mockResolvedValueOnce({ total_works: '1000000', works_24h: '10000', works_30d: '200000' }), + } as any; + + await sendDailyDigest(mockDb); + const body = JSON.parse(fetchSpy.mock.calls[0][1].body); + expect(body.text).toContain('Failed batches'); + expect(body.text).toContain('5'); + }); + + it('handles no sync position gracefully', async () => { + const mockDb = { + oneOrNone: vi.fn().mockResolvedValue(null), + } as any; + + await sendDailyDigest(mockDb); + expect(fetchSpy).toHaveBeenCalledOnce(); + const body = JSON.parse(fetchSpy.mock.calls[0][1].body); + expect(body.text).toContain('unknown'); + }); + + it('sends fallback message if DB query fails', async () => { + const mockDb = { + oneOrNone: vi.fn().mockRejectedValue(new Error('DB down')), + } as any; + + await sendDailyDigest(mockDb); + expect(fetchSpy).toHaveBeenCalledOnce(); + const body = JSON.parse(fetchSpy.mock.calls[0][1].body); + expect(body.text).toContain('Failed to generate daily digest'); + }); + + it('warns about stalled imports when behind with no recent completions', async () => { + const mockDb = { + oneOrNone: vi.fn() + .mockResolvedValueOnce({ query_to: new Date('2024-01-01') }) + .mockResolvedValueOnce({ days_imported: '0', total_duration_sec: '0' }) + .mockResolvedValueOnce({ failed_count: '0' }) + .mockResolvedValueOnce({ finished_at: new Date('2024-01-01T12:00:00Z') }) + .mockResolvedValueOnce({ total_works: '100000', works_24h: '0', works_30d: '5000' }), + } as any; + + await sendDailyDigest(mockDb); + const body = JSON.parse(fetchSpy.mock.calls[0][1].body); + expect(body.text).toContain('No imports completed'); + expect(body.text).toContain('check pod health'); + }); +}); + +describe('buildDigestMessage', () => { + it('includes pod uptime, last import with time ago, and record counts', async () => { + const recentDate = new Date(Date.now() - 3600_000); // 1h ago + const mockDb = { + oneOrNone: vi.fn() + .mockResolvedValueOnce({ query_to: new Date('2024-03-15') }) + .mockResolvedValueOnce({ days_imported: '1', total_duration_sec: '60' }) + .mockResolvedValueOnce({ failed_count: '0' }) + .mockResolvedValueOnce({ finished_at: recentDate }) + .mockResolvedValueOnce({ total_works: '12500000', works_24h: '8500', works_30d: '250000' }), + } as any; + + const message = await buildDigestMessage(mockDb); + expect(message).toContain('Pod uptime'); + expect(message).toContain('Last import'); + expect(message).toContain('ago)'); + expect(message).toContain('24h:'); + expect(message).toContain('8.5K works'); + expect(message).toContain('30d:'); + expect(message).toContain('250.0K works'); + expect(message).toContain('~12.5M works'); + }); + + it('shows caught up status for recent sync', async () => { + const yesterday = new Date(); + yesterday.setDate(yesterday.getDate() - 1); + const mockDb = { + oneOrNone: vi.fn() + .mockResolvedValueOnce({ query_to: yesterday }) + .mockResolvedValueOnce({ days_imported: '1', total_duration_sec: '30' }) + .mockResolvedValueOnce({ failed_count: '0' }) + .mockResolvedValueOnce({ finished_at: yesterday }) + .mockResolvedValueOnce({ total_works: '1000', works_24h: '500', works_30d: '800' }), + } as any; + + const message = await buildDigestMessage(mockDb); + expect(message).toContain('Caught up'); + }); + + it('shows "never" when no successful imports exist', async () => { + const mockDb = { + oneOrNone: vi.fn().mockResolvedValue(null), + } as any; + + const message = await buildDigestMessage(mockDb); + expect(message).toContain('never'); + }); +}); + +describe('buildPipelineStatus', () => { + it('shows overall health and per-pipeline details', async () => { + const mockDb = { + oneOrNone: vi.fn().mockResolvedValueOnce({ max_batch: '14611' }), + manyOrNone: vi.fn().mockResolvedValueOnce([ + { service: 'pg-to-es-batch-openalex', value: '14527', updated_at: new Date(Date.now() - 3600_000) }, + { service: 'ml-novelty-batch-openalex', value: '14527', updated_at: new Date(Date.now() - 3600_000) }, + { service: 'pg-to-vector-db-batch-openalex', value: '6451', updated_at: new Date('2026-02-27') }, + ]), + } as any; + + fetchSpy + .mockResolvedValueOnce({ + ok: true, + json: async () => [ + { id: 'deploy-es', name: 'pg_to_es_batch_import_deployment' }, + { id: 'deploy-nov', name: 'batch_novelty_openalex_deployment' }, + { id: 'deploy-qdr', name: 'pg_to_vector_db_batch_import_deployment' }, + ], + }) + .mockResolvedValueOnce({ + ok: true, + json: async () => [ + { id: 'run1', deployment_id: 'deploy-es', state: { type: 'COMPLETED', name: 'Completed' }, start_time: new Date(Date.now() - 7200_000).toISOString(), total_run_time: 221 }, + { id: 'run2', deployment_id: 'deploy-nov', state: { type: 'COMPLETED', name: 'Completed' }, start_time: new Date(Date.now() - 7200_000).toISOString(), total_run_time: 1540 }, + { id: 'run3', deployment_id: 'deploy-qdr', state: { type: 'COMPLETED', name: 'Completed' }, start_time: new Date(Date.now() - 86400_000).toISOString(), total_run_time: 3.8 }, + ], + }); + + const message = await buildPipelineStatus(mockDb); + expect(message).toContain('Pipeline Health'); + expect(message).toContain('action needed'); + expect(message).toContain('✅'); + expect(message).toContain('PG → Elasticsearch'); + expect(message).toContain('Healthy'); + expect(message).toContain('PG → Qdrant'); + expect(message).toContain('Stalled'); + expect(message).toContain('6,451'); + expect(message).toContain('no data processed'); + expect(message).toContain('Importer at batch'); + }); + + it('shows all healthy when pipelines are caught up', async () => { + const mockDb = { + oneOrNone: vi.fn().mockResolvedValueOnce({ max_batch: '14611' }), + manyOrNone: vi.fn().mockResolvedValueOnce([ + { service: 'pg-to-es-batch-openalex', value: '14611', updated_at: new Date() }, + { service: 'ml-novelty-batch-openalex', value: '14611', updated_at: new Date() }, + { service: 'pg-to-vector-db-batch-openalex', value: '14611', updated_at: new Date() }, + ]), + } as any; + + fetchSpy + .mockResolvedValueOnce({ + ok: true, + json: async () => [ + { id: 'd1', name: 'pg_to_es_batch_import_deployment' }, + { id: 'd2', name: 'batch_novelty_openalex_deployment' }, + { id: 'd3', name: 'pg_to_vector_db_batch_import_deployment' }, + ], + }) + .mockResolvedValueOnce({ + ok: true, + json: async () => [ + { id: 'r1', deployment_id: 'd1', state: { type: 'COMPLETED', name: 'Completed' }, start_time: new Date().toISOString(), total_run_time: 300 }, + { id: 'r2', deployment_id: 'd2', state: { type: 'COMPLETED', name: 'Completed' }, start_time: new Date().toISOString(), total_run_time: 1200 }, + { id: 'r3', deployment_id: 'd3', state: { type: 'COMPLETED', name: 'Completed' }, start_time: new Date().toISOString(), total_run_time: 600 }, + ], + }); + + const message = await buildPipelineStatus(mockDb); + expect(message).toContain('All pipelines healthy'); + expect(message).toContain('caught up'); + }); + + it('handles Prefect API failure gracefully', async () => { + const mockDb = { + oneOrNone: vi.fn().mockResolvedValueOnce({ max_batch: '14611' }), + manyOrNone: vi.fn().mockResolvedValueOnce([ + { service: 'pg-to-es-batch-openalex', value: '14527', updated_at: new Date() }, + ]), + } as any; + + fetchSpy.mockRejectedValueOnce(new Error('connection refused')); + + const message = await buildPipelineStatus(mockDb); + expect(message).toContain('Pipeline Health'); + expect(message).toContain('No recent runs found'); + expect(message).toContain('14,527'); + }); + + it('handles empty export_metadata gracefully', async () => { + const mockDb = { + oneOrNone: vi.fn().mockResolvedValueOnce({ max_batch: '100' }), + manyOrNone: vi.fn().mockResolvedValueOnce([]), + } as any; + + fetchSpy.mockRejectedValueOnce(new Error('connection refused')); + + const message = await buildPipelineStatus(mockDb); + expect(message).toContain('no tracking data'); + }); + + it('shows failing status for crashed runs', async () => { + const mockDb = { + oneOrNone: vi.fn().mockResolvedValueOnce({ max_batch: '100' }), + manyOrNone: vi.fn().mockResolvedValueOnce([ + { service: 'pg-to-es-batch-openalex', value: '90', updated_at: new Date() }, + ]), + } as any; + + fetchSpy + .mockResolvedValueOnce({ + ok: true, + json: async () => [{ id: 'd1', name: 'pg_to_es_batch_import_deployment' }], + }) + .mockResolvedValueOnce({ + ok: true, + json: async () => [ + { id: 'r1', deployment_id: 'd1', state: { type: 'CRASHED', name: 'Crashed' }, start_time: new Date().toISOString(), total_run_time: 2 }, + ], + }); + + const message = await buildPipelineStatus(mockDb); + expect(message).toContain('Last run failed'); + expect(message).toContain('action needed'); + }); +}); + +describe('startCommandListener', () => { + it('does not start polling when env vars are missing', async () => { + vi.stubEnv('TELEGRAM_BOT_TOKEN', ''); + vi.resetModules(); + const mod = await import('./notifier.js'); + + mod.startCommandListener({} as any); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it('stops cleanly via stopCommandListener', () => { + stopCommandListener(); + }); +}); diff --git a/openalex-importer/src/notifier.ts b/openalex-importer/src/notifier.ts new file mode 100644 index 000000000..6077d9c2c --- /dev/null +++ b/openalex-importer/src/notifier.ts @@ -0,0 +1,593 @@ +import { logger } from './logger.js'; +import { differenceInCalendarDays, formatDistanceToNowStrict } from 'date-fns'; +import { UTCDate } from '@date-fns/utc'; +import { hasUnfinishedBatches, type OaDb } from './db/index.js'; + +const TELEGRAM_API = 'https://api.telegram.org'; +const ERROR_COOLDOWN_MS = 60 * 60 * 1000; // 1 hour between error notifications +const POLL_INTERVAL_MS = 5_000; +const PREFECT_API = process.env.PREFECT_API_URL || 'http://alexandria.desci.com:4200'; + +const DOWNSTREAM_PIPELINES = [ + { deployment: 'pg_to_es_batch_import_deployment', service: 'pg-to-es-batch-openalex', label: 'PG → Elasticsearch' }, + { deployment: 'batch_novelty_openalex_deployment', service: 'ml-novelty-batch-openalex', label: 'Batch Novelty' }, + { deployment: 'pg_to_vector_db_batch_import_deployment', service: 'pg-to-vector-db-batch-openalex', label: 'PG → Qdrant' }, +] as const; + +let lastErrorNotifyMs = 0; +let wasCaughtUp = true; +let pollAbort: AbortController | null = null; +let digestPaused = false; + +const getAdminUserIds = (): Set => { + const raw = process.env.TELEGRAM_ADMIN_IDS ?? ''; + return new Set(raw.split(',').map(s => Number(s.trim())).filter(n => !Number.isNaN(n) && n > 0)); +}; + +export const isDigestPaused = (): boolean => digestPaused; + +const getConfig = () => { + const token = process.env.TELEGRAM_BOT_TOKEN; + const chatId = process.env.TELEGRAM_CHAT_ID; + const threadId = process.env.TELEGRAM_THREAD_ID; + return token && chatId ? { token, chatId, threadId } : null; +}; + +export const sendTelegram = async (message: string): Promise => { + const config = getConfig(); + if (!config) return false; + + try { + const body: Record = { + chat_id: config.chatId, + text: message, + parse_mode: 'HTML', + }; + if (config.threadId) { + body.message_thread_id = parseInt(config.threadId, 10); + } + + const res = await fetch(`${TELEGRAM_API}/bot${config.token}/sendMessage`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }); + + if (!res.ok) { + const text = await res.text(); + logger.warn({ status: res.status, body: text }, 'Telegram API returned non-OK'); + return false; + } + return true; + } catch (err) { + logger.warn({ err }, 'Failed to send Telegram notification'); + return false; + } +}; + +/** + * Rate-limited error notification. Sends at most 1 error message per hour + * (in-memory cooldown). On crash-loop restarts, the first error after each + * restart gets through, but K8s exponential backoff (up to 5min) limits the + * volume naturally. + */ +export const notifyError = async (error: Error, context?: string): Promise => { + const now = Date.now(); + if (now - lastErrorNotifyMs < ERROR_COOLDOWN_MS) return; + lastErrorNotifyMs = now; + + const contextLine = context ? `\nContext: ${context}` : ''; + await sendTelegram( + `🔴 OpenAlex Importer Error${contextLine}\n${error.message.slice(0, 300)}`, + ); +}; + +/** + * Notify when the importer finishes catching up — only fires on the + * state transition from "behind" to "caught up", not on every idle tick. + */ +export const notifyCaughtUp = async (syncedTo: string): Promise => { + if (wasCaughtUp) return; + wasCaughtUp = true; + await sendTelegram( + `✅ OpenAlex Importer caught up\nSynced through ${syncedTo}`, + ); +}; + +/** + * Mark that we're actively importing (behind). Called before each import + * so notifyCaughtUp knows when the transition happens. + */ +export const markImporting = (): void => { + wasCaughtUp = false; +}; + +/** + * Builds the status/digest message by querying the batch table. + * Shared between the daily digest cron and the /status bot command. + */ +export const buildDigestMessage = async (db: OaDb): Promise => { + const syncPosition = await db.oneOrNone<{ query_to: Date }>( + `SELECT query_to FROM openalex.batch + WHERE query_type = 'updated' AND finished_at IS NOT NULL + ORDER BY query_to DESC LIMIT 1`, + ); + + const last24h = await db.oneOrNone<{ days_imported: string; total_duration_sec: string }>( + `SELECT + COUNT(*) AS days_imported, + COALESCE(EXTRACT(EPOCH FROM SUM(finished_at - started_at)), 0)::int AS total_duration_sec + FROM openalex.batch + WHERE finished_at > NOW() - INTERVAL '24 hours' + AND query_type = 'updated'`, + ); + + const recentErrors = await db.oneOrNone<{ failed_count: string }>( + `SELECT COUNT(*) AS failed_count + FROM openalex.batch + WHERE finished_at IS NULL + AND started_at > NOW() - INTERVAL '24 hours' + AND query_type = 'updated'`, + ); + + const lastSuccess = await db.oneOrNone<{ finished_at: Date }>( + `SELECT finished_at FROM openalex.batch + WHERE finished_at IS NOT NULL AND query_type = 'updated' + ORDER BY finished_at DESC LIMIT 1`, + ); + + const recordCounts = await db.oneOrNone<{ total_works: string; works_24h: string; works_30d: string }>( + `SELECT + COALESCE((SELECT reltuples::bigint FROM pg_class + WHERE oid = 'openalex.works'::regclass), 0)::text AS total_works, + COALESCE((SELECT COUNT(*) FROM openalex.works_batch wb + JOIN openalex.batch b ON b.id = wb.batch_id + WHERE b.finished_at > NOW() - INTERVAL '24 hours' + AND b.query_type = 'updated'), 0)::text AS works_24h, + COALESCE((SELECT COUNT(*) FROM openalex.works_batch wb + JOIN openalex.batch b ON b.id = wb.batch_id + WHERE b.finished_at > NOW() - INTERVAL '30 days' + AND b.query_type = 'updated'), 0)::text AS works_30d`, + ); + + const syncDate = syncPosition?.query_to + ? new UTCDate(syncPosition.query_to).toISOString().split('T')[0] + : 'unknown'; + + const daysImported = parseInt(last24h?.days_imported ?? '0'); + const totalDuration = parseInt(last24h?.total_duration_sec ?? '0'); + const failedBatches = parseInt(recentErrors?.failed_count ?? '0'); + const daysBehind = syncPosition?.query_to + ? differenceInCalendarDays(new UTCDate(), new UTCDate(syncPosition.query_to)) + : null; + + const durationMin = Math.floor(totalDuration / 60); + const durationSec = totalDuration % 60; + + let status: string; + if (daysBehind === null) status = '⚠️ Unknown'; + else if (daysBehind <= 1) status = '✅ Caught up'; + else if (daysBehind <= 7) status = `🟡 ${daysBehind} days behind`; + else status = `🔴 ${daysBehind} days behind`; + + const uptime = Math.floor(process.uptime()); + const uptimeH = Math.floor(uptime / 3600); + const uptimeM = Math.floor((uptime % 3600) / 60); + + const lastSuccessTs = lastSuccess?.finished_at + ? new UTCDate(lastSuccess.finished_at).toISOString().replace('T', ' ').slice(0, 19) + ' UTC' + : null; + const lastSuccessAgo = lastSuccess?.finished_at + ? formatDistanceToNowStrict(new UTCDate(lastSuccess.finished_at), { addSuffix: true }) + : null; + const lastSuccessStr = lastSuccessTs + ? `${lastSuccessTs} (${lastSuccessAgo})` + : 'never'; + + const totalWorks = parseInt(recordCounts?.total_works ?? '0'); + const works24h = parseInt(recordCounts?.works_24h ?? '0'); + const works30d = parseInt(recordCounts?.works_30d ?? '0'); + + const fmt = (n: number) => n >= 1_000_000 + ? `${(n / 1_000_000).toFixed(1)}M` + : n >= 1_000 ? `${(n / 1_000).toFixed(1)}K` : `${n}`; + + const lines = [ + `📊 OpenAlex Importer — Status`, + ``, + `Status: ${status}`, + `Synced through: ${syncDate}${daysBehind !== null && daysBehind > 1 ? ` (${daysBehind} days behind)` : ''}`, + `Last import: ${lastSuccessStr}`, + `Throughput: ${daysImported} day${daysImported !== 1 ? 's' : ''} imported in ${durationMin}m ${durationSec}s (last 24h)`, + ``, + `Records ingested`, + ` 24h: ${fmt(works24h)} works`, + ` 30d: ${fmt(works30d)} works`, + ` total: ~${fmt(totalWorks)} works`, + ``, + `Pod uptime: ${uptimeH}h ${uptimeM}m`, + ]; + + if (failedBatches > 0) { + lines.push(`⚠️ Failed batches: ${failedBatches}`); + } + + if (daysImported === 0 && (daysBehind ?? 0) > 1) { + lines.push(`\n⚠️ No imports completed in 24h but still behind — check pod health`); + } + + return lines.join('\n'); +}; + +export const sendDailyDigest = async (db: OaDb): Promise => { + if (digestPaused) { + logger.info({ paused: true }, 'Daily digest skipped (paused)'); + return; + } + try { + const message = await buildDigestMessage(db); + await sendTelegram(`📅 Daily Update\n\n${message}`); + } catch (err) { + logger.warn({ err }, 'Failed to build daily digest'); + await sendTelegram('⚠️ OpenAlex Importer\nFailed to generate daily digest — check logs'); + } +}; + +interface PrefectFlowRun { + id: string; + deployment_id: string; + state: { type: string; name: string }; + start_time: string | null; + total_run_time: number; +} + +interface PrefectDeployment { + id: string; + name: string; + schedules: Array<{ schedule: { cron: string; timezone?: string }; active: boolean }>; +} + +/** + * Queries the Prefect API and export_metadata to build a status summary + * for all downstream pipelines (ES, novelty, Qdrant). + */ +export const buildPipelineStatus = async (db: OaDb): Promise => { + const importerBatch = await db.oneOrNone<{ max_batch: string }>( + `SELECT MAX(id)::text AS max_batch FROM openalex.batch WHERE finished_at IS NOT NULL`, + ); + const currentBatch = parseInt(importerBatch?.max_batch ?? '0', 10) || 0; + + const exportRows = await db.manyOrNone<{ service: string; value: string; updated_at: Date }>( + `SELECT service, value, updated_at FROM openalex.export_metadata + WHERE service IN ($1:csv)`, + [DOWNSTREAM_PIPELINES.map(p => p.service)], + ); + const exportByService = new Map(exportRows.map(r => [r.service, r])); + + let deploymentMap = new Map(); + let latestRuns = new Map(); + + try { + const deployRes = await fetch(`${PREFECT_API}/api/deployments/filter`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + deployments: { name: { any_: DOWNSTREAM_PIPELINES.map(p => p.deployment) } }, + }), + signal: AbortSignal.timeout(10_000), + }); + + if (deployRes.ok) { + const deploys = await deployRes.json() as PrefectDeployment[]; + deploymentMap = new Map(deploys.map(d => [d.name, d.id])); + + if (deploymentMap.size > 0) { + const runsRes = await fetch(`${PREFECT_API}/api/flow_runs/filter`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + sort: 'START_TIME_DESC', + limit: 30, + flow_runs: { + state: { type: { any_: ['COMPLETED', 'FAILED', 'CRASHED'] } }, + deployment_id: { any_: [...deploymentMap.values()] }, + }, + }), + signal: AbortSignal.timeout(10_000), + }); + + if (runsRes.ok) { + const runs = await runsRes.json() as PrefectFlowRun[]; + for (const run of runs) { + if (!latestRuns.has(run.deployment_id)) { + latestRuns.set(run.deployment_id, run); + } + } + } else { + logger.warn({ status: runsRes.status }, 'Prefect flow_runs/filter returned non-OK'); + } + } + } else { + logger.warn({ status: deployRes.status }, 'Prefect deployments/filter returned non-OK'); + } + } catch (err) { + logger.warn({ err }, 'Failed to query Prefect API for pipeline status'); + } + + type PipelineHealth = 'healthy' | 'stalled' | 'failing' | 'unknown'; + + const statuses: Array<{ label: string; health: PipelineHealth; lines: string[] }> = []; + + for (const pipeline of DOWNSTREAM_PIPELINES) { + const deployId = deploymentMap.get(pipeline.deployment); + const run = deployId ? latestRuns.get(deployId) : undefined; + const exportRow = exportByService.get(pipeline.service); + + const parsedBatch = exportRow ? parseInt(exportRow.value, 10) : NaN; + const batch = Number.isFinite(parsedBatch) ? parsedBatch : null; + const behind = batch !== null ? currentBatch - batch : null; + const pct = batch !== null && currentBatch > 0 + ? ((batch / currentBatch) * 100).toFixed(1) + : null; + + const ranRecently = run?.start_time + ? (Date.now() - new UTCDate(run.start_time).getTime()) < 24 * 60 * 60 * 1000 + : false; + const didWork = run ? run.total_run_time > 30 : false; + + let health: PipelineHealth = 'unknown'; + if (run?.state.type === 'FAILED' || run?.state.type === 'CRASHED') { + health = 'failing'; + } else if (ranRecently && didWork) { + health = 'healthy'; + } else if (run) { + health = 'stalled'; + } + + const icon = { healthy: '✅', stalled: '🔴', failing: '❌', unknown: '❓' }[health]; + const verdict = { + healthy: 'Healthy', + stalled: 'Stalled', + failing: 'Last run failed', + unknown: 'Unknown', + }[health]; + + const detail: string[] = []; + + if (run) { + const ago = run.start_time + ? formatDistanceToNowStrict(new UTCDate(run.start_time), { addSuffix: true }) + : 'unknown'; + const duration = run.total_run_time < 60 + ? `${run.total_run_time.toFixed(1)}s` + : `${Math.floor(run.total_run_time / 60)}m ${Math.round(run.total_run_time % 60)}s`; + const didWork = run.total_run_time > 30; + detail.push(` Ran ${ago} · ${duration}${!didWork ? ' (no data processed)' : ''}`); + } else { + detail.push(` No recent runs found`); + } + + if (batch !== null) { + if (behind !== null && behind <= 0) { + detail.push(` Progress: ${batch.toLocaleString()} / ${currentBatch.toLocaleString()} — caught up`); + } else if (behind !== null) { + detail.push(` Progress: ${batch.toLocaleString()} / ${currentBatch.toLocaleString()} (${pct}%) — ${behind.toLocaleString()} batches behind`); + } + if (health === 'stalled' && exportRow) { + const stalledAgo = formatDistanceToNowStrict(new UTCDate(exportRow.updated_at), { addSuffix: true }); + detail.push(` ⚠️ No progress since ${stalledAgo}`); + } + } else { + detail.push(` Progress: no tracking data`); + } + + statuses.push({ label: pipeline.label, health, lines: [`${icon} ${pipeline.label} — ${verdict}`, ...detail] }); + } + + const healthy = statuses.filter(s => s.health === 'healthy').length; + const total = statuses.length; + let overallIcon: string; + let overallVerdict: string; + if (healthy === total) { + overallIcon = '✅'; + overallVerdict = 'All pipelines healthy'; + } else if (statuses.some(s => s.health === 'stalled' || s.health === 'failing')) { + overallIcon = '🔴'; + overallVerdict = `${healthy}/${total} healthy — action needed`; + } else { + overallIcon = '🟡'; + overallVerdict = `${healthy}/${total} healthy`; + } + + const lines = [ + `🔗 Pipeline Health — ${overallIcon} ${overallVerdict}`, + ``, + ...statuses.flatMap(s => [...s.lines, ``]), + `Importer at batch ${currentBatch.toLocaleString()}`, + ]; + + return lines.join('\n'); +}; + +interface TelegramUpdate { + update_id: number; + message?: { + message_id: number; + from?: { id: number }; + chat: { id: number }; + message_thread_id?: number; + text?: string; + }; +} + +const replyToMessage = async (chatId: number, messageId: number, text: string, threadId?: number): Promise => { + const config = getConfig(); + if (!config) return; + + const body: Record = { + chat_id: chatId, + text, + parse_mode: 'HTML', + reply_to_message_id: messageId, + }; + if (threadId) { + body.message_thread_id = threadId; + } + + try { + await fetch(`${TELEGRAM_API}/bot${config.token}/sendMessage`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }); + } catch (err) { + logger.warn({ err }, 'Failed to reply to Telegram command'); + } +}; + +/** + * Polls Telegram for bot commands. Responds to: + * /status — current sync position, last 24h stats, pod uptime + * /help — list available commands + */ +export const startCommandListener = (db: OaDb): void => { + const config = getConfig(); + if (!config) { + logger.info('Telegram bot commands disabled (no TELEGRAM_BOT_TOKEN)'); + return; + } + + let offset = 0; + pollAbort = new AbortController(); + + void (async () => { + try { + const crashRecovery = await hasUnfinishedBatches(db); + if (crashRecovery) { + const message = await buildDigestMessage(db); + await sendTelegram(`🔄 OpenAlex Importer back online (recovering from crash)\n\n${message}`); + } else { + await sendTelegram(`🟢 OpenAlex Importer online\nType /help for commands.`); + } + } catch { + await sendTelegram('🟢 OpenAlex Importer online\nFailed to fetch initial status — type /status to retry.'); + } + })(); + + const poll = async () => { + while (!pollAbort?.signal.aborted) { + try { + const res = await fetch( + `${TELEGRAM_API}/bot${config.token}/getUpdates?offset=${offset}&timeout=30`, + { signal: pollAbort?.signal ?? null }, + ); + + if (!res.ok) { + logger.warn({ status: res.status }, 'Telegram getUpdates failed'); + await new Promise(r => setTimeout(r, POLL_INTERVAL_MS)); + continue; + } + + const data = await res.json() as { ok: boolean; result: TelegramUpdate[] }; + if (!data.ok || !data.result.length) continue; + + for (const update of data.result) { + offset = update.update_id + 1; + const text = update.message?.text?.trim(); + if (!text || !update.message) continue; + + const command = text.split(/\s+/)[0].split('@')[0].toLowerCase(); + + if (command === '/status') { + try { + const message = await buildDigestMessage(db); + await replyToMessage( + update.message.chat.id, + update.message.message_id, + message, + update.message.message_thread_id, + ); + } catch (err) { + logger.warn({ err }, 'Failed to build status for /status command'); + await replyToMessage( + update.message.chat.id, + update.message.message_id, + '⚠️ Failed to fetch status — check logs', + update.message.message_thread_id, + ); + } + } else if (command === '/pipelines') { + try { + const message = await buildPipelineStatus(db); + await replyToMessage( + update.message.chat.id, + update.message.message_id, + message, + update.message.message_thread_id, + ); + } catch (err) { + logger.warn({ err }, 'Failed to build pipeline status'); + await replyToMessage( + update.message.chat.id, + update.message.message_id, + '⚠️ Failed to fetch pipeline status — check logs', + update.message.message_thread_id, + ); + } + } else if (command === '/stopupdate' || command === '/startupdate') { + const adminIds = getAdminUserIds(); + const senderId = update.message.from?.id; + // When TELEGRAM_ADMIN_IDS is unset, allow all users (backwards-compatible dev default) + if (adminIds.size > 0 && (!senderId || !adminIds.has(senderId))) { + logger.warn({ senderId, command }, 'Unauthorized command attempt'); + await replyToMessage( + update.message.chat.id, + update.message.message_id, + '🚫 You are not authorized to use this command.', + update.message.message_thread_id, + ); + } else { + digestPaused = command === '/stopupdate'; + logger.info({ command, senderId }, `Daily digest ${digestPaused ? 'paused' : 'resumed'} via ${command}`); + await replyToMessage( + update.message.chat.id, + update.message.message_id, + digestPaused + ? '⏸️ Daily updates paused. Use /startupdate to resume.' + : '▶️ Daily updates resumed.', + update.message.message_thread_id, + ); + } + } else if (command === '/help') { + await replyToMessage( + update.message.chat.id, + update.message.message_id, + [ + `OpenAlex Importer Bot`, + ``, + `/status — importer sync position, records, uptime`, + `/pipelines — downstream pipeline health (ES, novelty, Qdrant)`, + `/stopupdate — pause daily digest`, + `/startupdate — resume daily digest`, + `/help — show this message`, + ].join('\n'), + update.message.message_thread_id, + ); + } + } + } catch (err) { + if (pollAbort?.signal.aborted) break; + logger.warn({ err }, 'Telegram poll error, retrying...'); + await new Promise(r => setTimeout(r, POLL_INTERVAL_MS)); + } + } + }; + + logger.info('Telegram bot command listener started'); + void poll(); +}; + +export const stopCommandListener = (): void => { + pollAbort?.abort(); + pollAbort = null; +}; diff --git a/openalex-importer/src/pipeline.ts b/openalex-importer/src/pipeline.ts index 069429285..5ca0bcd49 100644 --- a/openalex-importer/src/pipeline.ts +++ b/openalex-importer/src/pipeline.ts @@ -1,6 +1,7 @@ import { Readable, Transform } from 'stream'; import { pipeline } from 'stream/promises'; import { + cleanupUnfinishedBatches, createBatch, finalizeBatch, type OaDb, @@ -16,7 +17,6 @@ import { getDuration, sleep } from './util.js'; import { Writable } from 'node:stream'; import { IS_DEV, MAX_PAGES_TO_FETCH, SKIP_LOG_WRITE } from '../index.js'; import { RateLimiter } from './rateLimiter.js'; -import * as pgPromise from 'pg-promise'; const MAX_RETRIES = 10; const BASE_DELAY = 1_000; @@ -178,14 +178,16 @@ const createLogStream = (): Transform => { }); }; -const createSaveStream = (tx: pgPromise.ITask, batchId: number): Writable => { +const createSaveStream = (db: OaDb, batchId: number): Writable => { return new Writable({ highWaterMark: 1, objectMode: true, async write(chunk: DataModels, _encoding, callback) { try { const start = Date.now(); - await saveData(tx, batchId, chunk); + await db.tx(async (tx) => { + await saveData(tx, batchId, chunk); + }); logger.info({ duration: Date.now() - start }, 'Saved chunk to database'); callback(); } catch (error) { @@ -202,18 +204,29 @@ export const runImportPipeline = async (db: OaDb, queryInfo: QueryInfo): Promise await nukeOldLogs(); const filter = filterFromQueryInfo(queryInfo); - await db.tx(async (tx) => { - const batchId = await createBatch(tx, queryInfo); + // Clean up any unfinished batch from a previous crash for this day + await cleanupUnfinishedBatches(db, queryInfo); + + // Create batch record in its own transaction + const batchId = await createBatch(db, queryInfo); + logger.info({ batchId, queryInfo }, 'Created batch record'); + + // Each chunk saves in its own transaction — progress survives crashes + try { await pipeline( createWorksAPIStream(filter), createBufferStream(), createTransformStream(), createLogStream(), - createSaveStream(tx, batchId), + createSaveStream(db, batchId), ); + } catch (err) { + logger.error({ batchId, queryInfo, err: errWithCause(err as Error) }, 'Import pipeline failed — batch left unfinished for recovery'); + throw err; + } - await finalizeBatch(tx, batchId); - }); + // Mark the batch as complete + await finalizeBatch(db, batchId); const duration = getDuration(startTime, Date.now()); logger.info({ duration: `${duration} s`, queryInfo }, 'Import pipeline finished!'); diff --git a/openalex-importer/src/transformers.test.ts b/openalex-importer/src/transformers.test.ts new file mode 100644 index 000000000..ac1c4fd88 --- /dev/null +++ b/openalex-importer/src/transformers.test.ts @@ -0,0 +1,163 @@ +import { describe, it, expect } from 'vitest'; +import { transformDataModel } from './transformers.js'; +import { + minimalWork, + workWithNullMeshQualifiers, + workWithNoPrimaryLocation, + workWithEmptyArrays, + workWithDuplicateAuthorships, + workWithNullConceptAndTopicIds, + workWithNullAuthorIds, +} from './__fixtures__/works.js'; + +const ALL_DATA_MODEL_KEYS = [ + 'authors', + 'works_authorships', + 'authors_ids', + 'works', + 'works_id', + 'works_biblio', + 'works_concepts', + 'works_topics', + 'works_locations', + 'works_mesh', + 'works_open_access', + 'works_primary_locations', + 'works_referenced_works', + 'works_related_works', + 'works_best_oa_locations', +] as const; + +describe('transformDataModel', () => { + it('produces all expected keys from a valid Work', () => { + const result = transformDataModel([minimalWork]); + for (const key of ALL_DATA_MODEL_KEYS) { + expect(result).toHaveProperty(key); + expect(Array.isArray(result[key])).toBe(true); + } + }); + + it('strips OpenAlex URL prefixes from IDs', () => { + const result = transformDataModel([minimalWork]); + expect(result.works[0].id).toBe('W1234567890'); + expect(result.works[0].doi).toBe('10.1234/test.2024.001'); + expect(result.works_id[0].openalex).toBe('W1234567890'); + expect(result.authors[0].id).toBe('A001'); + expect(result.works_concepts[0].concept_id).toBe('C100'); + }); + + it('produces correct counts for a single work with all entities', () => { + const result = transformDataModel([minimalWork]); + expect(result.works).toHaveLength(1); + expect(result.works_id).toHaveLength(1); + expect(result.works_biblio).toHaveLength(1); + expect(result.works_concepts).toHaveLength(1); + expect(result.works_topics).toHaveLength(1); + expect(result.works_mesh).toHaveLength(1); + expect(result.works_locations).toHaveLength(1); + expect(result.works_primary_locations).toHaveLength(1); + expect(result.works_best_oa_locations).toHaveLength(1); + expect(result.works_open_access).toHaveLength(1); + expect(result.works_referenced_works).toHaveLength(1); + expect(result.works_related_works).toHaveLength(1); + expect(result.authors).toHaveLength(1); + expect(result.authors_ids).toHaveLength(1); + expect(result.works_authorships).toHaveLength(1); + }); + + describe('edge case: null mesh qualifier/descriptor UIDs', () => { + it('passes null qualifier_ui through to the DB model (dedup layer handles filtering)', () => { + const result = transformDataModel([workWithNullMeshQualifiers]); + expect(result.works_mesh).toHaveLength(3); + + const nullQualifiers = result.works_mesh.filter(m => m.qualifier_ui == null); + expect(nullQualifiers).toHaveLength(2); + + const nullDescriptors = result.works_mesh.filter(m => m.descriptor_ui == null); + expect(nullDescriptors).toHaveLength(1); + }); + }); + + describe('edge case: no primary_location / best_oa_location', () => { + it('filters out null primary_locations and best_oa_locations', () => { + const result = transformDataModel([workWithNoPrimaryLocation]); + expect(result.works_primary_locations).toHaveLength(0); + expect(result.works_best_oa_locations).toHaveLength(0); + expect(result.works_locations).toHaveLength(0); + }); + }); + + describe('edge case: empty child arrays', () => { + it('produces empty arrays for all child tables without throwing', () => { + const result = transformDataModel([workWithEmptyArrays]); + expect(result.works).toHaveLength(1); + expect(result.works_authorships).toHaveLength(0); + expect(result.authors).toHaveLength(0); + expect(result.authors_ids).toHaveLength(0); + expect(result.works_concepts).toHaveLength(0); + expect(result.works_topics).toHaveLength(0); + expect(result.works_mesh).toHaveLength(0); + expect(result.works_locations).toHaveLength(0); + expect(result.works_referenced_works).toHaveLength(0); + expect(result.works_related_works).toHaveLength(0); + }); + }); + + describe('edge case: empty input', () => { + it('returns all keys with empty arrays', () => { + const result = transformDataModel([]); + for (const key of ALL_DATA_MODEL_KEYS) { + expect(result[key]).toEqual([]); + } + }); + }); + + describe('edge case: duplicate authorships (multi-affiliation)', () => { + it('deduplicates authors by ID', () => { + const result = transformDataModel([workWithDuplicateAuthorships]); + expect(result.authors).toHaveLength(1); + expect(result.authors_ids).toHaveLength(1); + expect(result.works_authorships).toHaveLength(2); + }); + }); + + describe('edge case: null concept and topic IDs', () => { + it('transforms null IDs without throwing (dedup layer filters)', () => { + const result = transformDataModel([workWithNullConceptAndTopicIds]); + expect(result.works_concepts).toHaveLength(2); + expect(result.works_topics).toHaveLength(2); + + const nullConcepts = result.works_concepts.filter(c => c.concept_id == null); + expect(nullConcepts).toHaveLength(1); + + const nullTopics = result.works_topics.filter(t => t.topic_id == null); + expect(nullTopics).toHaveLength(1); + }); + }); + + describe('edge case: null author IDs', () => { + it('transforms null author IDs without throwing', () => { + const result = transformDataModel([workWithNullAuthorIds]); + expect(result.works_authorships).toHaveLength(2); + const nullAuthorships = result.works_authorships.filter(a => a.author_id == null); + expect(nullAuthorships).toHaveLength(1); + }); + }); + + describe('batch of mixed works', () => { + it('handles a batch with both clean and problematic works', () => { + const batch = [ + minimalWork, + workWithNullMeshQualifiers, + workWithNoPrimaryLocation, + workWithEmptyArrays, + workWithNullConceptAndTopicIds, + ]; + + const result = transformDataModel(batch); + expect(result.works).toHaveLength(5); + expect(result.works_mesh.length).toBeGreaterThan(0); + expect(result.works_concepts.length).toBeGreaterThan(0); + }); + }); +}); diff --git a/sync-server/src/automerge-repo-storage-postgres/PostgresStorageAdapter.ts b/sync-server/src/automerge-repo-storage-postgres/PostgresStorageAdapter.ts index eb49b4222..421e83b65 100644 --- a/sync-server/src/automerge-repo-storage-postgres/PostgresStorageAdapter.ts +++ b/sync-server/src/automerge-repo-storage-postgres/PostgresStorageAdapter.ts @@ -32,16 +32,17 @@ export class PostgresStorageAdapter implements StorageAdapterInterface { async save(keyArray: StorageKey, binary: Uint8Array): Promise { const key = getKey(keyArray); - this.cache[key] = binary; try { await this.query( `INSERT INTO "${this.tableName}" (key, value) VALUES ($1, $2) ON CONFLICT(key) DO UPDATE SET value = $2 RETURNING key`, [key, Buffer.from(binary)], ); + this.cache[key] = binary; console.log('[saved]', { key }); } catch (e) { console.error({ e, key }, 'PostgresStorageAdapter::Save ==> Error saving document'); + throw e; } } @@ -73,25 +74,28 @@ export class PostgresStorageAdapter implements StorageAdapterInterface { } async removeRange(keyPrefix: StorageKey): Promise { - const key = getKey(keyPrefix); + const key = getKeyPrefix(keyPrefix); this.cachedKeys(keyPrefix).forEach((key) => delete this.cache[key]); try { - const result = await this.query(`DELETE FROM "${this.tableName}" WHERE key LIKE $1 RETURNING key`, [`${key}%`]); + await this.query(`DELETE FROM "${this.tableName}" WHERE key LIKE $1 RETURNING key`, [`${key}%`]); } catch (e) { console.error({ keyPrefix, key }, '[DELETE RANGE kEYS]'); + throw e; } } private cachedKeys(keyPrefix: string[]): string[] { - const cacheKeyPrefixString = getKey(keyPrefix); + const cacheKeyPrefixString = getKeyPrefix(keyPrefix); return Object.keys(this.cache).filter((key) => key.startsWith(cacheKeyPrefixString)); } private async loadRangeKeys(keyPrefix: string[]): Promise { - const response = await this.query(`SELECT key FROM "${this.tableName}" WHERE key LIKE $1`, [`${keyPrefix}%`]); + const key = getKeyPrefix(keyPrefix); + const response = await this.query(`SELECT key FROM "${this.tableName}" WHERE key LIKE $1`, [`${key}%`]); return response ? response.map((row) => row.key) : []; } } // HELPERS const getKey = (key: StorageKey): string => path.join(...key); +const getKeyPrefix = (key: StorageKey): string => (key.length ? getKey(key) : ''); diff --git a/sync-server/src/automerge-repo-storage-postgres/db.ts b/sync-server/src/automerge-repo-storage-postgres/db.ts index c4f736eef..d12ab3ad4 100644 --- a/sync-server/src/automerge-repo-storage-postgres/db.ts +++ b/sync-server/src/automerge-repo-storage-postgres/db.ts @@ -1,4 +1,4 @@ -import { Client, Pool } from 'pg'; +import { Pool } from 'pg'; import { err as serialiseErr } from 'pino-std-serializers'; export interface DbDriver { @@ -8,7 +8,7 @@ export interface DbDriver { export default { async init(connectionString: string) { console.log('[Hyperdrive] ✅', { connectionString }); - const pool = new Pool({ connectionString, connectionTimeoutMillis: 15000, query_timeout: 1000 }); + const pool = new Pool({ connectionString, connectionTimeoutMillis: 15000, query_timeout: 15000 }); pool.on('error', (err) => console.error('[Hyperdrive Error]::', { error: serialiseErr(err as Error), pool })); return { @@ -21,7 +21,7 @@ export default { return result.rows; } catch (err) { console.error('[Hyperdrive Error]::', { error: serialiseErr(err as Error), pool }); - return undefined; + throw err; } finally { // client.release(); } diff --git a/sync-server/src/index.ts b/sync-server/src/index.ts index 28a256e2a..7d9cd1d8a 100644 --- a/sync-server/src/index.ts +++ b/sync-server/src/index.ts @@ -1,6 +1,5 @@ import { Doc, - DocHandle, DocHandleChangePayload, // DocHandleEphemeralMessagePayload, DocHandleEvents, @@ -22,7 +21,7 @@ import { Env } from './types.js'; import { ensureUuidEndsWithDot } from './utils.js'; import { assert } from './automerge-repo-network-websocket/assert.js'; // import { actionsSchema } from './lib/schema.js'; -import { actionDispatcher, getAutomergeUrl, getDocumentUpdater } from './manifestRepo.js'; +import { actionDispatcher, getAutomergeUrl } from './manifestRepo.js'; import { ZodError } from 'zod'; interface ResearchObjectDocument { @@ -37,7 +36,6 @@ export class AutomergeServer extends PartyServer { // hibernate: true; // }; repo: Repo; - handle: DocHandle; constructor( private readonly ctx: DurableObjectState, @@ -172,8 +170,8 @@ export class AutomergeServer extends PartyServer { return new Response(JSON.stringify({ ok: false, message: 'Invalid body' }), { status: 400 }); } - if (!this.handle) this.handle = this.repo.find(getAutomergeUrl(documentId)); - const document: Doc | undefined = await this.handle.doc(); + const handle = this.repo.find(getAutomergeUrl(documentId)); + const document: Doc | undefined = await handle.doc(); return new Response(JSON.stringify({ document, ok: true }), { status: 200 }); } catch (err) { console.error('[getLatestDocument]', { err: errWithCause(err) }); @@ -192,13 +190,13 @@ export class AutomergeServer extends PartyServer { return new Response(JSON.stringify({ ok: false, message: 'No actions to dispatch' }), { status: 400 }); } - if (!this.handle) this.handle = this.repo.find(getAutomergeUrl(documentId)); + const handle = this.repo.find(getAutomergeUrl(documentId)); for (const action of actions) { - await actionDispatcher({ action, handle: this.handle, documentId }); + await actionDispatcher({ action, handle, documentId }); } - const document: Doc | undefined = await this.handle.doc(); + const document: Doc | undefined = await handle.doc(); if (!document) { console.error({ document }, 'Document not found'); return new Response(JSON.stringify({ ok: false, message: 'Document not found' }), { status: 400 }); @@ -247,6 +245,12 @@ async function handleCreateDocument(request: Request, env: Env) { await repo.flush(); let document = await handle.doc(); + assert(document, 'Document was not created'); + + const persistedChunks = await query('SELECT key FROM "DocumentStore" WHERE key LIKE $1 LIMIT 1', [ + `${handle.documentId}%`, + ]); + assert(persistedChunks && persistedChunks.length > 0, 'Document was not persisted'); console.log('[Request]::handleCreateDocument ', { uuid: body.uuid, created: !!document }); return new Response(JSON.stringify({ documentId: handle.documentId, document }), { status: 200 });