From e00a0cb02e736bba6b3b8cf012c7d0119ec1b9a5 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Tue, 11 Aug 2026 12:51:14 +0100 Subject: [PATCH 01/69] feat: add integration.sync_units control table Signed-off-by: Mouad BANI --- .../V1786442761__createSyncUnitsTable.sql | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 backend/src/database/migrations/V1786442761__createSyncUnitsTable.sql diff --git a/backend/src/database/migrations/V1786442761__createSyncUnitsTable.sql b/backend/src/database/migrations/V1786442761__createSyncUnitsTable.sql new file mode 100644 index 0000000000..893657e826 --- /dev/null +++ b/backend/src/database/migrations/V1786442761__createSyncUnitsTable.sql @@ -0,0 +1,25 @@ +CREATE TABLE IF NOT EXISTS integration.sync_units ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + "integrationId" UUID NOT NULL REFERENCES public.integrations(id), + platform TEXT NOT NULL, + "channelId" TEXT NOT NULL, + "channelName" TEXT NOT NULL, + "syncName" TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'active' + CHECK (status IN ('active','paused','dead_letter','decommissioned')), + "nextRunAt" TIMESTAMPTZ NOT NULL DEFAULT now(), + "lockedAt" TIMESTAMPTZ, + "lastRunAt" TIMESTAMPTZ, + "lastSuccessAt" TIMESTAMPTZ, + "consecutiveFailures" INT NOT NULL DEFAULT 0, + "lastErrorClass" TEXT, + watermark JSONB, + "emittedCount" INT, + "createdAt" TIMESTAMPTZ NOT NULL DEFAULT now(), + "updatedAt" TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE ("integrationId", "channelId", "syncName") +); + +CREATE INDEX IF NOT EXISTS "ix_sync_units_due" + ON integration.sync_units ("nextRunAt") + WHERE status = 'active'; From f5dafd74c31488d012e177abf28bbf3cb452019f Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Tue, 11 Aug 2026 12:52:04 +0100 Subject: [PATCH 02/69] feat: add sync units data access layer Signed-off-by: Mouad BANI --- .../src/integrationBuilder/syncUnits.ts | 115 ++++++++++++++++++ .../src/integrationBuilder/types.ts | 29 +++++ 2 files changed, 144 insertions(+) create mode 100644 services/libs/data-access-layer/src/integrationBuilder/syncUnits.ts create mode 100644 services/libs/data-access-layer/src/integrationBuilder/types.ts diff --git a/services/libs/data-access-layer/src/integrationBuilder/syncUnits.ts b/services/libs/data-access-layer/src/integrationBuilder/syncUnits.ts new file mode 100644 index 0000000000..9d399e8c27 --- /dev/null +++ b/services/libs/data-access-layer/src/integrationBuilder/syncUnits.ts @@ -0,0 +1,115 @@ +import type { QueryExecutor } from '../queryExecutor' + +import type { ISyncRunSuccess, ISyncUnit, SyncUnitUpsert } from './types' + +const MIN_INITIAL_DELAY_SECONDS = 10 +const MAX_INITIAL_DELAY_SECONDS = 900 +const CLAIM_LEASE_MINUTES = 5 + +export async function upsertSyncUnits(qx: QueryExecutor, units: SyncUnitUpsert[]): Promise { + if (units.length === 0) { + return 0 + } + + return qx.result( + `INSERT INTO integration.sync_units + ("integrationId", platform, "channelId", "channelName", "syncName", "nextRunAt") + SELECT u.*, now() + ($(minDelaySeconds) + random() * $(delaySpanSeconds)) * interval '1 second' + FROM unnest( + $(integrationIds)::uuid[], + $(platforms)::text[], + $(channelIds)::text[], + $(channelNames)::text[], + $(syncNames)::text[] + ) u + ON CONFLICT ("integrationId", "channelId", "syncName") + DO UPDATE SET "channelName" = EXCLUDED."channelName", "updatedAt" = now()`, + { + integrationIds: units.map((u) => u.integrationId), + platforms: units.map((u) => u.platform), + channelIds: units.map((u) => u.channelId), + channelNames: units.map((u) => u.channelName), + syncNames: units.map((u) => u.syncName), + minDelaySeconds: MIN_INITIAL_DELAY_SECONDS, + delaySpanSeconds: MAX_INITIAL_DELAY_SECONDS - MIN_INITIAL_DELAY_SECONDS, + }, + ) +} + +export async function claimDueUnits(qx: QueryExecutor, limit: number): Promise { + return qx.select( + `UPDATE integration.sync_units su + SET "lockedAt" = now(), "updatedAt" = now() + WHERE su.id IN ( + SELECT id + FROM integration.sync_units + WHERE status = 'active' + AND "nextRunAt" <= now() + AND ("lockedAt" IS NULL OR "lockedAt" < now() - $(leaseMinutes) * interval '1 minute') + ORDER BY "nextRunAt" + LIMIT $(limit) + FOR UPDATE SKIP LOCKED + ) + RETURNING su.*`, + { limit, leaseMinutes: CLAIM_LEASE_MINUTES }, + ) +} + +export async function rescheduleUnit( + qx: QueryExecutor, + id: string, + nextRunAt: Date, +): Promise { + await qx.result( + `UPDATE integration.sync_units + SET "nextRunAt" = $(nextRunAt), "lockedAt" = NULL, "updatedAt" = now() + WHERE id = $(id)`, + { id, nextRunAt }, + ) +} + +export async function recordRunSuccess( + qx: QueryExecutor, + id: string, + data: ISyncRunSuccess, +): Promise { + await qx.result( + `UPDATE integration.sync_units + SET watermark = $(watermark)::jsonb, + "emittedCount" = $(emittedCount), + "lastRunAt" = now(), + "lastSuccessAt" = now(), + "consecutiveFailures" = 0, + "updatedAt" = now() + WHERE id = $(id)`, + { id, watermark: JSON.stringify(data.watermark), emittedCount: data.emittedCount }, + ) +} + +export async function recordRunFailure( + qx: QueryExecutor, + id: string, + errorClass: string, + deadLetterAfter: number, +): Promise { + await qx.result( + `UPDATE integration.sync_units + SET "consecutiveFailures" = "consecutiveFailures" + 1, + "lastErrorClass" = $(errorClass), + "lastRunAt" = now(), + status = CASE WHEN "consecutiveFailures" + 1 >= $(deadLetterAfter) + THEN 'dead_letter' ELSE status END, + "updatedAt" = now() + WHERE id = $(id)`, + { id, errorClass, deadLetterAfter }, + ) +} + +export async function getUnitById(qx: QueryExecutor, id: string): Promise { + return qx.selectOneOrNone( + `SELECT * + FROM integration.sync_units + WHERE id = $(id)`, + { id }, + ) +} diff --git a/services/libs/data-access-layer/src/integrationBuilder/types.ts b/services/libs/data-access-layer/src/integrationBuilder/types.ts new file mode 100644 index 0000000000..925e2dba97 --- /dev/null +++ b/services/libs/data-access-layer/src/integrationBuilder/types.ts @@ -0,0 +1,29 @@ +export type SyncUnitStatus = 'active' | 'paused' | 'dead_letter' | 'decommissioned' + +export interface ISyncUnit { + id: string + integrationId: string + platform: string + channelId: string + channelName: string + syncName: string + status: SyncUnitStatus + nextRunAt: string + lockedAt: string | null + lastRunAt: string | null + lastSuccessAt: string | null + consecutiveFailures: number + lastErrorClass: string | null + watermark: Record | null + emittedCount: number | null +} + +export type SyncUnitUpsert = Pick< + ISyncUnit, + 'integrationId' | 'platform' | 'channelId' | 'channelName' | 'syncName' +> + +export interface ISyncRunSuccess { + watermark: Record + emittedCount: number +} From f41df42c29a553a7fc6e0afcbc3c2e8e669bf6bd Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Tue, 11 Aug 2026 13:00:55 +0100 Subject: [PATCH 03/69] feat: add integration-builder lib with connector registry Signed-off-by: Mouad BANI --- pnpm-lock.yaml | 47 +++++++++++++++---- .../libs/integration-builder/package.json | 21 +++++++++ .../libs/integration-builder/src/index.ts | 2 + .../libs/integration-builder/src/registry.ts | 23 +++++++++ .../libs/integration-builder/src/types.ts | 32 +++++++++++++ .../libs/integration-builder/tsconfig.json | 4 ++ 6 files changed, 119 insertions(+), 10 deletions(-) create mode 100644 services/libs/integration-builder/package.json create mode 100644 services/libs/integration-builder/src/index.ts create mode 100644 services/libs/integration-builder/src/registry.ts create mode 100644 services/libs/integration-builder/src/types.ts create mode 100644 services/libs/integration-builder/tsconfig.json diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 44e00af382..0329af257d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2328,6 +2328,28 @@ importers: specifier: ^5.6.3 version: 5.6.3 + services/libs/integration-builder: + dependencies: + '@crowd/common': + specifier: workspace:* + version: link:../common + '@crowd/data-access-layer': + specifier: workspace:* + version: link:../data-access-layer + '@crowd/logging': + specifier: workspace:* + version: link:../logging + zod: + specifier: ^3.22.0 + version: 3.25.76 + devDependencies: + '@types/node': + specifier: ^20.8.2 + version: 20.12.7 + typescript: + specifier: ^5.6.3 + version: 5.6.3 + services/libs/integrations: dependencies: '@crowd/common': @@ -10810,6 +10832,9 @@ packages: peerDependencies: zod: ^3.25.28 || ^4 + zod@3.25.76: + resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} + zod@4.3.6: resolution: {integrity: sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==} @@ -10969,8 +10994,8 @@ snapshots: dependencies: '@aws-crypto/sha256-browser': 3.0.0 '@aws-crypto/sha256-js': 3.0.0 - '@aws-sdk/client-sso-oidc': 3.572.0(@aws-sdk/client-sts@3.572.0) - '@aws-sdk/client-sts': 3.572.0 + '@aws-sdk/client-sso-oidc': 3.572.0 + '@aws-sdk/client-sts': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) '@aws-sdk/core': 3.572.0 '@aws-sdk/credential-provider-node': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0)(@aws-sdk/client-sts@3.572.0) '@aws-sdk/middleware-host-header': 3.567.0 @@ -11164,11 +11189,11 @@ snapshots: transitivePeerDependencies: - aws-crt - '@aws-sdk/client-sso-oidc@3.572.0(@aws-sdk/client-sts@3.572.0)': + '@aws-sdk/client-sso-oidc@3.572.0': dependencies: '@aws-crypto/sha256-browser': 3.0.0 '@aws-crypto/sha256-js': 3.0.0 - '@aws-sdk/client-sts': 3.572.0 + '@aws-sdk/client-sts': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) '@aws-sdk/core': 3.572.0 '@aws-sdk/credential-provider-node': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0)(@aws-sdk/client-sts@3.572.0) '@aws-sdk/middleware-host-header': 3.567.0 @@ -11207,7 +11232,6 @@ snapshots: '@smithy/util-utf8': 2.3.0 tslib: 2.6.2 transitivePeerDependencies: - - '@aws-sdk/client-sts' - aws-crt '@aws-sdk/client-sso@3.556.0': @@ -11383,11 +11407,11 @@ snapshots: transitivePeerDependencies: - aws-crt - '@aws-sdk/client-sts@3.572.0': + '@aws-sdk/client-sts@3.572.0(@aws-sdk/client-sso-oidc@3.572.0)': dependencies: '@aws-crypto/sha256-browser': 3.0.0 '@aws-crypto/sha256-js': 3.0.0 - '@aws-sdk/client-sso-oidc': 3.572.0(@aws-sdk/client-sts@3.572.0) + '@aws-sdk/client-sso-oidc': 3.572.0 '@aws-sdk/core': 3.572.0 '@aws-sdk/credential-provider-node': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0)(@aws-sdk/client-sts@3.572.0) '@aws-sdk/middleware-host-header': 3.567.0 @@ -11426,6 +11450,7 @@ snapshots: '@smithy/util-utf8': 2.3.0 tslib: 2.6.2 transitivePeerDependencies: + - '@aws-sdk/client-sso-oidc' - aws-crt '@aws-sdk/client-sts@3.985.0': @@ -11591,7 +11616,7 @@ snapshots: '@aws-sdk/credential-provider-ini@3.572.0(@aws-sdk/client-sso-oidc@3.572.0)(@aws-sdk/client-sts@3.572.0)': dependencies: - '@aws-sdk/client-sts': 3.572.0 + '@aws-sdk/client-sts': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) '@aws-sdk/credential-provider-env': 3.568.0 '@aws-sdk/credential-provider-process': 3.572.0 '@aws-sdk/credential-provider-sso': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) @@ -11768,7 +11793,7 @@ snapshots: '@aws-sdk/credential-provider-web-identity@3.568.0(@aws-sdk/client-sts@3.572.0)': dependencies: - '@aws-sdk/client-sts': 3.572.0 + '@aws-sdk/client-sts': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) '@aws-sdk/types': 3.567.0 '@smithy/property-provider': 2.2.0 '@smithy/types': 2.12.0 @@ -12080,7 +12105,7 @@ snapshots: '@aws-sdk/token-providers@3.572.0(@aws-sdk/client-sso-oidc@3.572.0)': dependencies: - '@aws-sdk/client-sso-oidc': 3.572.0(@aws-sdk/client-sts@3.572.0) + '@aws-sdk/client-sso-oidc': 3.572.0 '@aws-sdk/types': 3.567.0 '@smithy/property-provider': 2.2.0 '@smithy/shared-ini-file-loader': 2.4.0 @@ -21191,4 +21216,6 @@ snapshots: dependencies: zod: 4.3.6 + zod@3.25.76: {} + zod@4.3.6: {} diff --git a/services/libs/integration-builder/package.json b/services/libs/integration-builder/package.json new file mode 100644 index 0000000000..eb1298b5b9 --- /dev/null +++ b/services/libs/integration-builder/package.json @@ -0,0 +1,21 @@ +{ + "name": "@crowd/integration-builder", + "private": true, + "main": "src/index.ts", + "scripts": { + "lint": "npx eslint --ext .ts src --max-warnings=0", + "format": "npx prettier --write \"src/**/*.ts\"", + "format-check": "npx prettier --check .", + "tsc-check": "tsc --noEmit" + }, + "devDependencies": { + "@types/node": "^20.8.2", + "typescript": "^5.6.3" + }, + "dependencies": { + "@crowd/common": "workspace:*", + "@crowd/data-access-layer": "workspace:*", + "@crowd/logging": "workspace:*", + "zod": "^3.22.0" + } +} diff --git a/services/libs/integration-builder/src/index.ts b/services/libs/integration-builder/src/index.ts new file mode 100644 index 0000000000..636ed8d91f --- /dev/null +++ b/services/libs/integration-builder/src/index.ts @@ -0,0 +1,2 @@ +export * from './registry' +export * from './types' diff --git a/services/libs/integration-builder/src/registry.ts b/services/libs/integration-builder/src/registry.ts new file mode 100644 index 0000000000..081751934b --- /dev/null +++ b/services/libs/integration-builder/src/registry.ts @@ -0,0 +1,23 @@ +import type { Manifest, SyncDefinition } from './types' + +const manifests = new Map() + +export function registerConnector(manifest: Manifest): void { + manifests.set(manifest.platform, manifest) +} + +export function getManifest(platform: string): Manifest { + const manifest = manifests.get(platform) + if (!manifest) { + throw new Error(`unknown platform ${platform}`) + } + return manifest +} + +export function getSync(platform: string, syncName: string): SyncDefinition { + const sync = getManifest(platform).syncs.find((s) => s.name === syncName) + if (!sync) { + throw new Error(`unknown sync ${platform}/${syncName}`) + } + return sync +} diff --git a/services/libs/integration-builder/src/types.ts b/services/libs/integration-builder/src/types.ts new file mode 100644 index 0000000000..88e6fef35a --- /dev/null +++ b/services/libs/integration-builder/src/types.ts @@ -0,0 +1,32 @@ +import type { Logger } from '@crowd/logging' + +export interface Channel { + channelId: string + channelName: string +} + +export interface Credential { + platform: string + kind: 'github-app' | 'token' + data: Record +} + +export interface SyncContext { + channel: Channel + watermark: Record | null + emit: (records: unknown[]) => Promise + commitWatermark: (watermark: Record) => Promise + log: Logger +} + +export interface SyncDefinition { + name: string + cadenceMinutes: number + run: (ctx: SyncContext) => Promise +} + +export interface Manifest { + platform: string + syncs: SyncDefinition[] + discover: (credential: Credential) => Promise +} diff --git a/services/libs/integration-builder/tsconfig.json b/services/libs/integration-builder/tsconfig.json new file mode 100644 index 0000000000..bf7f183850 --- /dev/null +++ b/services/libs/integration-builder/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../base.tsconfig.json", + "include": ["src/**/*"] +} From 5062058dcc39311dddcfb7eb28e461016324db55 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Tue, 11 Aug 2026 13:05:46 +0100 Subject: [PATCH 04/69] fix: skip sync units of soft-deleted integrations when claiming Signed-off-by: Mouad BANI --- .../src/integrationBuilder/syncUnits.ts | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/services/libs/data-access-layer/src/integrationBuilder/syncUnits.ts b/services/libs/data-access-layer/src/integrationBuilder/syncUnits.ts index 9d399e8c27..3554b77935 100644 --- a/services/libs/data-access-layer/src/integrationBuilder/syncUnits.ts +++ b/services/libs/data-access-layer/src/integrationBuilder/syncUnits.ts @@ -41,12 +41,17 @@ export async function claimDueUnits(qx: QueryExecutor, limit: number): Promise Date: Tue, 11 Aug 2026 13:23:29 +0100 Subject: [PATCH 05/69] feat: add getCredential facade Signed-off-by: Mouad BANI --- .../integration-builder/src/credentials.ts | 51 +++++++++++++++++++ .../libs/integration-builder/src/index.ts | 1 + .../libs/integration-builder/src/types.ts | 11 +++- 3 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 services/libs/integration-builder/src/credentials.ts diff --git a/services/libs/integration-builder/src/credentials.ts b/services/libs/integration-builder/src/credentials.ts new file mode 100644 index 0000000000..552e214991 --- /dev/null +++ b/services/libs/integration-builder/src/credentials.ts @@ -0,0 +1,51 @@ +import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' + +import type { Credential } from './types' + +export async function getCredential( + qx: QueryExecutor, + integrationId: string, +): Promise { + const integration: { platform: string } | null = await qx.selectOneOrNone( + `SELECT platform + FROM integrations + WHERE id = $(integrationId) AND "deletedAt" IS NULL`, + { integrationId }, + ) + + if (!integration) { + throw new Error(`integration ${integrationId} not found`) + } + + // POC scope: GitHub only; each migrated connector adds its platform case here + switch (integration.platform) { + case 'github': + case 'github-nango': + return githubAppCredential() + default: + throw new Error(`unsupported platform ${integration.platform}`) + } +} + +// POC only: secrets come from env; secret-manager adoption (OCI Vault) swaps this +// body without touching callers — getCredential stays the single entry point +function githubAppCredential(): Credential { + const appId = process.env.CROWD_GITHUB_APP_ID + const rawPrivateKey = process.env.CROWD_GITHUB_PRIVATE_KEY + + if (!appId || !rawPrivateKey) { + throw new Error( + 'missing CROWD_GITHUB_APP_ID or CROWD_GITHUB_PRIVATE_KEY environment variables', + ) + } + + const privateKey = rawPrivateKey.startsWith('-----') + ? rawPrivateKey + : Buffer.from(rawPrivateKey, 'base64').toString('ascii') + + return { + platform: 'github', + kind: 'github-app', + data: { appId, privateKey }, + } +} diff --git a/services/libs/integration-builder/src/index.ts b/services/libs/integration-builder/src/index.ts index 636ed8d91f..6e932df937 100644 --- a/services/libs/integration-builder/src/index.ts +++ b/services/libs/integration-builder/src/index.ts @@ -1,2 +1,3 @@ +export * from './credentials' export * from './registry' export * from './types' diff --git a/services/libs/integration-builder/src/types.ts b/services/libs/integration-builder/src/types.ts index 88e6fef35a..9e60f1c91c 100644 --- a/services/libs/integration-builder/src/types.ts +++ b/services/libs/integration-builder/src/types.ts @@ -5,10 +5,17 @@ export interface Channel { channelName: string } +export interface GithubAppCredentialData { + appId: string + privateKey: string +} + +// POC only: single variant; becomes a discriminated union (token, oauth2, ...) +// as more connectors land export interface Credential { platform: string - kind: 'github-app' | 'token' - data: Record + kind: 'github-app' + data: GithubAppCredentialData } export interface SyncContext { From 4daecd3dcaae460f56050b16842cd688b039cdd8 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Tue, 11 Aug 2026 13:28:55 +0100 Subject: [PATCH 06/69] fix: export integration builder DAL from package root Signed-off-by: Mouad BANI --- services/libs/data-access-layer/src/index.ts | 1 + services/libs/data-access-layer/src/integrationBuilder/index.ts | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 services/libs/data-access-layer/src/integrationBuilder/index.ts diff --git a/services/libs/data-access-layer/src/index.ts b/services/libs/data-access-layer/src/index.ts index 9079622755..c96467b899 100644 --- a/services/libs/data-access-layer/src/index.ts +++ b/services/libs/data-access-layer/src/index.ts @@ -13,6 +13,7 @@ export * from './repositories' export * from './security_insights' export * from './segments' export * from './systemSettings' +export * from './integrationBuilder' export * from './integrations' export * from './auditLogs' export * from './maintainers' diff --git a/services/libs/data-access-layer/src/integrationBuilder/index.ts b/services/libs/data-access-layer/src/integrationBuilder/index.ts new file mode 100644 index 0000000000..ee1244f4c2 --- /dev/null +++ b/services/libs/data-access-layer/src/integrationBuilder/index.ts @@ -0,0 +1,2 @@ +export * from './syncUnits' +export * from './types' From 31d697f9e26bcb8ff48b4e0290286fe216ea2ed4 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Tue, 11 Aug 2026 13:41:09 +0100 Subject: [PATCH 07/69] style: fix prettier formatting in credentials facade Signed-off-by: Mouad BANI --- services/libs/integration-builder/src/credentials.ts | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/services/libs/integration-builder/src/credentials.ts b/services/libs/integration-builder/src/credentials.ts index 552e214991..d5daa68405 100644 --- a/services/libs/integration-builder/src/credentials.ts +++ b/services/libs/integration-builder/src/credentials.ts @@ -2,10 +2,7 @@ import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' import type { Credential } from './types' -export async function getCredential( - qx: QueryExecutor, - integrationId: string, -): Promise { +export async function getCredential(qx: QueryExecutor, integrationId: string): Promise { const integration: { platform: string } | null = await qx.selectOneOrNone( `SELECT platform FROM integrations @@ -34,9 +31,7 @@ function githubAppCredential(): Credential { const rawPrivateKey = process.env.CROWD_GITHUB_PRIVATE_KEY if (!appId || !rawPrivateKey) { - throw new Error( - 'missing CROWD_GITHUB_APP_ID or CROWD_GITHUB_PRIVATE_KEY environment variables', - ) + throw new Error('missing CROWD_GITHUB_APP_ID or CROWD_GITHUB_PRIVATE_KEY environment variables') } const privateKey = rawPrivateKey.startsWith('-----') From 94901f8231f30266442c3c5113db7a69716b6f60 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Tue, 11 Aug 2026 17:35:46 +0100 Subject: [PATCH 08/69] refactor: rename integration-builder to connectors Signed-off-by: Mouad BANI --- services/libs/{integration-builder => connectors}/package.json | 2 +- .../libs/{integration-builder => connectors}/src/credentials.ts | 0 services/libs/{integration-builder => connectors}/src/index.ts | 0 .../libs/{integration-builder => connectors}/src/registry.ts | 0 services/libs/{integration-builder => connectors}/src/types.ts | 0 services/libs/{integration-builder => connectors}/tsconfig.json | 0 .../src/{integrationBuilder => connectors}/index.ts | 0 .../src/{integrationBuilder => connectors}/syncUnits.ts | 0 .../src/{integrationBuilder => connectors}/types.ts | 0 services/libs/data-access-layer/src/index.ts | 2 +- 10 files changed, 2 insertions(+), 2 deletions(-) rename services/libs/{integration-builder => connectors}/package.json (92%) rename services/libs/{integration-builder => connectors}/src/credentials.ts (100%) rename services/libs/{integration-builder => connectors}/src/index.ts (100%) rename services/libs/{integration-builder => connectors}/src/registry.ts (100%) rename services/libs/{integration-builder => connectors}/src/types.ts (100%) rename services/libs/{integration-builder => connectors}/tsconfig.json (100%) rename services/libs/data-access-layer/src/{integrationBuilder => connectors}/index.ts (100%) rename services/libs/data-access-layer/src/{integrationBuilder => connectors}/syncUnits.ts (100%) rename services/libs/data-access-layer/src/{integrationBuilder => connectors}/types.ts (100%) diff --git a/services/libs/integration-builder/package.json b/services/libs/connectors/package.json similarity index 92% rename from services/libs/integration-builder/package.json rename to services/libs/connectors/package.json index eb1298b5b9..2e5d961cda 100644 --- a/services/libs/integration-builder/package.json +++ b/services/libs/connectors/package.json @@ -1,5 +1,5 @@ { - "name": "@crowd/integration-builder", + "name": "@crowd/connectors", "private": true, "main": "src/index.ts", "scripts": { diff --git a/services/libs/integration-builder/src/credentials.ts b/services/libs/connectors/src/credentials.ts similarity index 100% rename from services/libs/integration-builder/src/credentials.ts rename to services/libs/connectors/src/credentials.ts diff --git a/services/libs/integration-builder/src/index.ts b/services/libs/connectors/src/index.ts similarity index 100% rename from services/libs/integration-builder/src/index.ts rename to services/libs/connectors/src/index.ts diff --git a/services/libs/integration-builder/src/registry.ts b/services/libs/connectors/src/registry.ts similarity index 100% rename from services/libs/integration-builder/src/registry.ts rename to services/libs/connectors/src/registry.ts diff --git a/services/libs/integration-builder/src/types.ts b/services/libs/connectors/src/types.ts similarity index 100% rename from services/libs/integration-builder/src/types.ts rename to services/libs/connectors/src/types.ts diff --git a/services/libs/integration-builder/tsconfig.json b/services/libs/connectors/tsconfig.json similarity index 100% rename from services/libs/integration-builder/tsconfig.json rename to services/libs/connectors/tsconfig.json diff --git a/services/libs/data-access-layer/src/integrationBuilder/index.ts b/services/libs/data-access-layer/src/connectors/index.ts similarity index 100% rename from services/libs/data-access-layer/src/integrationBuilder/index.ts rename to services/libs/data-access-layer/src/connectors/index.ts diff --git a/services/libs/data-access-layer/src/integrationBuilder/syncUnits.ts b/services/libs/data-access-layer/src/connectors/syncUnits.ts similarity index 100% rename from services/libs/data-access-layer/src/integrationBuilder/syncUnits.ts rename to services/libs/data-access-layer/src/connectors/syncUnits.ts diff --git a/services/libs/data-access-layer/src/integrationBuilder/types.ts b/services/libs/data-access-layer/src/connectors/types.ts similarity index 100% rename from services/libs/data-access-layer/src/integrationBuilder/types.ts rename to services/libs/data-access-layer/src/connectors/types.ts diff --git a/services/libs/data-access-layer/src/index.ts b/services/libs/data-access-layer/src/index.ts index c96467b899..7424ddcdd4 100644 --- a/services/libs/data-access-layer/src/index.ts +++ b/services/libs/data-access-layer/src/index.ts @@ -13,7 +13,7 @@ export * from './repositories' export * from './security_insights' export * from './segments' export * from './systemSettings' -export * from './integrationBuilder' +export * from './connectors' export * from './integrations' export * from './auditLogs' export * from './maintainers' From d681c113506d2036ff3c5ba80391cefb39d90d11 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Tue, 11 Aug 2026 17:47:44 +0100 Subject: [PATCH 09/69] feat: add connectors worker with dispatcher workflow Signed-off-by: Mouad BANI --- pnpm-lock.yaml | 96 ++++++++++++++----- services/apps/connectors_worker/package.json | 35 +++++++ .../apps/connectors_worker/src/activities.ts | 3 + .../src/activities/dispatcherActivities.ts | 52 ++++++++++ services/apps/connectors_worker/src/main.ts | 36 +++++++ .../src/schedules/dispatcher.ts | 32 +++++++ services/apps/connectors_worker/src/types.ts | 1 + .../apps/connectors_worker/src/workflows.ts | 3 + .../src/workflows/dispatcher.ts | 21 ++++ services/apps/connectors_worker/tsconfig.json | 4 + 10 files changed, 261 insertions(+), 22 deletions(-) create mode 100644 services/apps/connectors_worker/package.json create mode 100644 services/apps/connectors_worker/src/activities.ts create mode 100644 services/apps/connectors_worker/src/activities/dispatcherActivities.ts create mode 100644 services/apps/connectors_worker/src/main.ts create mode 100644 services/apps/connectors_worker/src/schedules/dispatcher.ts create mode 100644 services/apps/connectors_worker/src/types.ts create mode 100644 services/apps/connectors_worker/src/workflows.ts create mode 100644 services/apps/connectors_worker/src/workflows/dispatcher.ts create mode 100644 services/apps/connectors_worker/tsconfig.json diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0329af257d..63c18dadf8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -608,6 +608,58 @@ importers: specifier: ^3.0.1 version: 3.1.0 + services/apps/connectors_worker: + dependencies: + '@crowd/archetype-standard': + specifier: workspace:* + version: link:../../archetypes/standard + '@crowd/archetype-worker': + specifier: workspace:* + version: link:../../archetypes/worker + '@crowd/common': + specifier: workspace:* + version: link:../../libs/common + '@crowd/connectors': + specifier: workspace:* + version: link:../../libs/connectors + '@crowd/data-access-layer': + specifier: workspace:* + version: link:../../libs/data-access-layer + '@crowd/logging': + specifier: workspace:* + version: link:../../libs/logging + '@crowd/redis': + specifier: workspace:* + version: link:../../libs/redis + '@crowd/temporal': + specifier: workspace:* + version: link:../../libs/temporal + '@crowd/types': + specifier: workspace:* + version: link:../../libs/types + '@temporalio/activity': + specifier: ~1.17.2 + version: 1.17.2 + '@temporalio/client': + specifier: ~1.17.2 + version: 1.17.2 + '@temporalio/workflow': + specifier: ~1.17.2 + version: 1.17.2 + tsx: + specifier: ^4.7.1 + version: 4.7.3 + typescript: + specifier: ^5.6.3 + version: 5.6.3 + devDependencies: + '@types/node': + specifier: ^20.8.2 + version: 20.12.7 + nodemon: + specifier: ^3.0.1 + version: 3.1.0 + services/apps/cron_service: dependencies: '@aws-sdk/client-s3': @@ -2227,6 +2279,28 @@ importers: specifier: ^5.6.3 version: 5.6.3 + services/libs/connectors: + dependencies: + '@crowd/common': + specifier: workspace:* + version: link:../common + '@crowd/data-access-layer': + specifier: workspace:* + version: link:../data-access-layer + '@crowd/logging': + specifier: workspace:* + version: link:../logging + zod: + specifier: ^3.22.0 + version: 3.25.76 + devDependencies: + '@types/node': + specifier: ^20.8.2 + version: 20.12.7 + typescript: + specifier: ^5.6.3 + version: 5.6.3 + services/libs/data-access-layer: dependencies: '@crowd/common': @@ -2328,28 +2402,6 @@ importers: specifier: ^5.6.3 version: 5.6.3 - services/libs/integration-builder: - dependencies: - '@crowd/common': - specifier: workspace:* - version: link:../common - '@crowd/data-access-layer': - specifier: workspace:* - version: link:../data-access-layer - '@crowd/logging': - specifier: workspace:* - version: link:../logging - zod: - specifier: ^3.22.0 - version: 3.25.76 - devDependencies: - '@types/node': - specifier: ^20.8.2 - version: 20.12.7 - typescript: - specifier: ^5.6.3 - version: 5.6.3 - services/libs/integrations: dependencies: '@crowd/common': diff --git a/services/apps/connectors_worker/package.json b/services/apps/connectors_worker/package.json new file mode 100644 index 0000000000..5cb359796b --- /dev/null +++ b/services/apps/connectors_worker/package.json @@ -0,0 +1,35 @@ +{ + "name": "@crowd/connectors-worker", + "private": true, + "scripts": { + "start": "CROWD_TEMPORAL_TASKQUEUE=connectors SERVICE=connectors-worker tsx src/main.ts", + "start:debug:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=connectors SERVICE=connectors-worker LOG_LEVEL=trace tsx --inspect=0.0.0.0:9243 src/main.ts", + "start:debug": "CROWD_TEMPORAL_TASKQUEUE=connectors SERVICE=connectors-worker LOG_LEVEL=trace tsx --inspect=0.0.0.0:9243 src/main.ts", + "dev:local": "nodemon --watch src --watch ../../libs --ext ts --exec pnpm run start:debug:local", + "dev": "nodemon --watch src --watch ../../libs --ext ts --exec pnpm run start:debug", + "lint": "npx eslint --ext .ts src --max-warnings=0", + "format": "npx prettier --write \"src/**/*.ts\"", + "format-check": "npx prettier --check .", + "tsc-check": "tsc --noEmit" + }, + "dependencies": { + "@crowd/archetype-standard": "workspace:*", + "@crowd/archetype-worker": "workspace:*", + "@crowd/common": "workspace:*", + "@crowd/data-access-layer": "workspace:*", + "@crowd/connectors": "workspace:*", + "@crowd/logging": "workspace:*", + "@crowd/redis": "workspace:*", + "@crowd/temporal": "workspace:*", + "@crowd/types": "workspace:*", + "@temporalio/activity": "~1.17.2", + "@temporalio/client": "~1.17.2", + "@temporalio/workflow": "~1.17.2", + "tsx": "^4.7.1", + "typescript": "^5.6.3" + }, + "devDependencies": { + "@types/node": "^20.8.2", + "nodemon": "^3.0.1" + } +} diff --git a/services/apps/connectors_worker/src/activities.ts b/services/apps/connectors_worker/src/activities.ts new file mode 100644 index 0000000000..c69b2985c9 --- /dev/null +++ b/services/apps/connectors_worker/src/activities.ts @@ -0,0 +1,3 @@ +import { claimDue, reschedule, startRun, touchHeartbeat } from './activities/dispatcherActivities' + +export { claimDue, reschedule, startRun, touchHeartbeat } diff --git a/services/apps/connectors_worker/src/activities/dispatcherActivities.ts b/services/apps/connectors_worker/src/activities/dispatcherActivities.ts new file mode 100644 index 0000000000..c5c47a644d --- /dev/null +++ b/services/apps/connectors_worker/src/activities/dispatcherActivities.ts @@ -0,0 +1,52 @@ +import { getSync } from '@crowd/connectors' +import { claimDueUnits, rescheduleUnit } from '@crowd/data-access-layer/src/connectors' +import type { ISyncUnit } from '@crowd/data-access-layer/src/connectors' +import { dbStoreQx } from '@crowd/data-access-layer/src/queryExecutor' +import { RedisCache } from '@crowd/redis' +import { WorkflowIdConflictPolicy, WorkflowIdReusePolicy } from '@crowd/temporal' + +import { svc } from '../main' +import type { StartRunResult } from '../types' + +const TASK_QUEUE = 'connectors' +const HEARTBEAT_TTL_SECONDS = 300 +const CADENCE_JITTER_RATIO = 0.1 + +export async function claimDue(limit: number): Promise { + return claimDueUnits(dbStoreQx(svc.postgres.writer), limit) +} + +export async function startRun(unit: ISyncUnit): Promise { + try { + await svc.temporal.workflow.start('syncRun', { + taskQueue: TASK_QUEUE, + workflowId: `sync-run/${unit.id}`, + workflowIdReusePolicy: WorkflowIdReusePolicy.ALLOW_DUPLICATE, + workflowIdConflictPolicy: WorkflowIdConflictPolicy.FAIL, + args: [unit.id], + }) + return 'started' + } catch (err) { + if (err instanceof Error && err.name === 'WorkflowExecutionAlreadyStartedError') { + return 'alreadyRunning' + } + throw err + } +} + +export async function reschedule( + unitId: string, + platform: string, + syncName: string, +): Promise { + const { cadenceMinutes } = getSync(platform, syncName) + const jitterMinutes = cadenceMinutes * CADENCE_JITTER_RATIO * (Math.random() * 2 - 1) + const nextRunAt = new Date(Date.now() + (cadenceMinutes + jitterMinutes) * 60_000) + + await rescheduleUnit(dbStoreQx(svc.postgres.writer), unitId, nextRunAt) +} + +export async function touchHeartbeat(): Promise { + const cache = new RedisCache('connectors', svc.redis, svc.log) + await cache.set('dispatcherHeartbeat', new Date().toISOString(), HEARTBEAT_TTL_SECONDS) +} diff --git a/services/apps/connectors_worker/src/main.ts b/services/apps/connectors_worker/src/main.ts new file mode 100644 index 0000000000..7ebd785514 --- /dev/null +++ b/services/apps/connectors_worker/src/main.ts @@ -0,0 +1,36 @@ +import { Config } from '@crowd/archetype-standard' +import { Options, ServiceWorker } from '@crowd/archetype-worker' + +import { scheduleDispatcher } from './schedules/dispatcher' + +const config: Config = { + envvars: [], + producer: { + enabled: false, + }, + temporal: { + enabled: true, + }, + redis: { + enabled: true, + }, +} + +const options: Options = { + postgres: { + enabled: true, + }, + opensearch: { + enabled: false, + }, +} + +export const svc = new ServiceWorker(config, options) + +setImmediate(async () => { + await svc.init() + + await scheduleDispatcher() + + await svc.start() +}) diff --git a/services/apps/connectors_worker/src/schedules/dispatcher.ts b/services/apps/connectors_worker/src/schedules/dispatcher.ts new file mode 100644 index 0000000000..63ed04d218 --- /dev/null +++ b/services/apps/connectors_worker/src/schedules/dispatcher.ts @@ -0,0 +1,32 @@ +import { ScheduleAlreadyRunning, ScheduleOverlapPolicy } from '@temporalio/client' + +import { svc } from '../main' +import { dispatcher } from '../workflows/dispatcher' + +export async function scheduleDispatcher(): Promise { + try { + await svc.temporal.schedule.create({ + scheduleId: 'connectors-dispatcher', + spec: { + intervals: [{ every: '30s' }], + }, + policies: { + overlap: ScheduleOverlapPolicy.SKIP, + catchupWindow: '1 minute', + }, + action: { + type: 'startWorkflow', + workflowType: dispatcher, + taskQueue: 'connectors', + args: [], + workflowExecutionTimeout: '5 minutes', + }, + }) + } catch (err) { + if (err instanceof ScheduleAlreadyRunning) { + svc.log.info('Dispatcher schedule already registered in Temporal.') + } else { + throw new Error(err) + } + } +} diff --git a/services/apps/connectors_worker/src/types.ts b/services/apps/connectors_worker/src/types.ts new file mode 100644 index 0000000000..ac915242b1 --- /dev/null +++ b/services/apps/connectors_worker/src/types.ts @@ -0,0 +1 @@ +export type StartRunResult = 'started' | 'alreadyRunning' diff --git a/services/apps/connectors_worker/src/workflows.ts b/services/apps/connectors_worker/src/workflows.ts new file mode 100644 index 0000000000..2044bd7bf3 --- /dev/null +++ b/services/apps/connectors_worker/src/workflows.ts @@ -0,0 +1,3 @@ +import { dispatcher } from './workflows/dispatcher' + +export { dispatcher } diff --git a/services/apps/connectors_worker/src/workflows/dispatcher.ts b/services/apps/connectors_worker/src/workflows/dispatcher.ts new file mode 100644 index 0000000000..44cbbae714 --- /dev/null +++ b/services/apps/connectors_worker/src/workflows/dispatcher.ts @@ -0,0 +1,21 @@ +import { proxyActivities } from '@temporalio/workflow' + +import type * as activities from '../activities/dispatcherActivities' + +const activity = proxyActivities({ + startToCloseTimeout: '1 minute', + retry: { maximumAttempts: 3, backoffCoefficient: 2 }, +}) + +const CLAIM_LIMIT = 100 + +export async function dispatcher(): Promise { + await activity.touchHeartbeat() + + const units = await activity.claimDue(CLAIM_LIMIT) + + for (const unit of units) { + await activity.startRun(unit) + await activity.reschedule(unit.id, unit.platform, unit.syncName) + } +} diff --git a/services/apps/connectors_worker/tsconfig.json b/services/apps/connectors_worker/tsconfig.json new file mode 100644 index 0000000000..bf7f183850 --- /dev/null +++ b/services/apps/connectors_worker/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../base.tsconfig.json", + "include": ["src/**/*"] +} From 8b22542601b1f9781ae99045fa7ed9829a1c5862 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Tue, 11 Aug 2026 18:20:22 +0100 Subject: [PATCH 10/69] feat: add sync-run workflow with dummy connector Signed-off-by: Mouad BANI --- .../apps/connectors_worker/src/activities.ts | 3 +- .../src/activities/syncRunActivities.ts | 70 +++++++++++++++++++ services/apps/connectors_worker/src/main.ts | 6 ++ .../apps/connectors_worker/src/workflows.ts | 3 +- .../src/workflows/dispatcher.ts | 10 ++- .../src/workflows/syncRun.ts | 13 ++++ .../connectors/src/testing/dummyConnector.ts | 18 +++++ 7 files changed, 118 insertions(+), 5 deletions(-) create mode 100644 services/apps/connectors_worker/src/activities/syncRunActivities.ts create mode 100644 services/apps/connectors_worker/src/workflows/syncRun.ts create mode 100644 services/libs/connectors/src/testing/dummyConnector.ts diff --git a/services/apps/connectors_worker/src/activities.ts b/services/apps/connectors_worker/src/activities.ts index c69b2985c9..585832d864 100644 --- a/services/apps/connectors_worker/src/activities.ts +++ b/services/apps/connectors_worker/src/activities.ts @@ -1,3 +1,4 @@ import { claimDue, reschedule, startRun, touchHeartbeat } from './activities/dispatcherActivities' +import { executeSync } from './activities/syncRunActivities' -export { claimDue, reschedule, startRun, touchHeartbeat } +export { claimDue, executeSync, reschedule, startRun, touchHeartbeat } diff --git a/services/apps/connectors_worker/src/activities/syncRunActivities.ts b/services/apps/connectors_worker/src/activities/syncRunActivities.ts new file mode 100644 index 0000000000..bab19f2844 --- /dev/null +++ b/services/apps/connectors_worker/src/activities/syncRunActivities.ts @@ -0,0 +1,70 @@ +import { Context } from '@temporalio/activity' + +import { getSync } from '@crowd/connectors' +import type { SyncContext } from '@crowd/connectors' +import { + getUnitById, + recordRunFailure, + recordRunSuccess, +} from '@crowd/data-access-layer/src/connectors' +import { dbStoreQx } from '@crowd/data-access-layer/src/queryExecutor' +import { getChildLogger } from '@crowd/logging' + +import { svc } from '../main' + +const DEAD_LETTER_AFTER = 5 +const HEARTBEAT_INTERVAL_MS = 10_000 + +export async function executeSync(unitId: string): Promise { + const qx = dbStoreQx(svc.postgres.writer) + + const unit = await getUnitById(qx, unitId) + if (!unit) { + throw new Error(`sync unit ${unitId} not found`) + } + + const activityContext = Context.current() + const log = getChildLogger('syncRun', svc.log, { + runId: activityContext.info.workflowExecution.runId, + unitId: unit.id, + platform: unit.platform, + syncName: unit.syncName, + channelName: unit.channelName, + }) + + let emittedCount = 0 + let committedWatermark = unit.watermark + + // POC only: emit collects counts in memory; the Kafka sink emitter arrives in M2 + const ctx: SyncContext = { + channel: { channelId: unit.channelId, channelName: unit.channelName }, + watermark: unit.watermark, + emit: async (records) => { + emittedCount += records.length + }, + commitWatermark: async (watermark) => { + committedWatermark = watermark + }, + log, + } + + const heartbeat = setInterval(() => activityContext.heartbeat(), HEARTBEAT_INTERVAL_MS) + + try { + const sync = getSync(unit.platform, unit.syncName) + await sync.run(ctx) + + await recordRunSuccess(qx, unitId, { + watermark: committedWatermark ?? {}, + emittedCount, + }) + log.info({ emittedCount }, 'sync run succeeded') + } catch (err) { + log.error(err, 'sync run failed') + // POC only: everything unclassified is framework.internal; the 7-class + // error taxonomy arrives with the M2 HTTP client + await recordRunFailure(qx, unitId, 'framework.internal', DEAD_LETTER_AFTER) + } finally { + clearInterval(heartbeat) + } +} diff --git a/services/apps/connectors_worker/src/main.ts b/services/apps/connectors_worker/src/main.ts index 7ebd785514..ec10cb39a6 100644 --- a/services/apps/connectors_worker/src/main.ts +++ b/services/apps/connectors_worker/src/main.ts @@ -1,5 +1,7 @@ import { Config } from '@crowd/archetype-standard' import { Options, ServiceWorker } from '@crowd/archetype-worker' +import { registerConnector } from '@crowd/connectors' +import { dummyConnector } from '@crowd/connectors/src/testing/dummyConnector' import { scheduleDispatcher } from './schedules/dispatcher' @@ -27,6 +29,10 @@ const options: Options = { export const svc = new ServiceWorker(config, options) +// POC only: dummy connector drives the control-plane end-to-end; real +// connectors register here starting with GitHub in M4 +registerConnector(dummyConnector) + setImmediate(async () => { await svc.init() diff --git a/services/apps/connectors_worker/src/workflows.ts b/services/apps/connectors_worker/src/workflows.ts index 2044bd7bf3..67cb17400b 100644 --- a/services/apps/connectors_worker/src/workflows.ts +++ b/services/apps/connectors_worker/src/workflows.ts @@ -1,3 +1,4 @@ import { dispatcher } from './workflows/dispatcher' +import { syncRun } from './workflows/syncRun' -export { dispatcher } +export { dispatcher, syncRun } diff --git a/services/apps/connectors_worker/src/workflows/dispatcher.ts b/services/apps/connectors_worker/src/workflows/dispatcher.ts index 44cbbae714..5908059622 100644 --- a/services/apps/connectors_worker/src/workflows/dispatcher.ts +++ b/services/apps/connectors_worker/src/workflows/dispatcher.ts @@ -1,4 +1,4 @@ -import { proxyActivities } from '@temporalio/workflow' +import { log, proxyActivities } from '@temporalio/workflow' import type * as activities from '../activities/dispatcherActivities' @@ -15,7 +15,11 @@ export async function dispatcher(): Promise { const units = await activity.claimDue(CLAIM_LIMIT) for (const unit of units) { - await activity.startRun(unit) - await activity.reschedule(unit.id, unit.platform, unit.syncName) + try { + await activity.startRun(unit) + await activity.reschedule(unit.id, unit.platform, unit.syncName) + } catch (err) { + log.error('failed to dispatch sync unit', { unitId: unit.id, err }) + } } } diff --git a/services/apps/connectors_worker/src/workflows/syncRun.ts b/services/apps/connectors_worker/src/workflows/syncRun.ts new file mode 100644 index 0000000000..40f2637e4e --- /dev/null +++ b/services/apps/connectors_worker/src/workflows/syncRun.ts @@ -0,0 +1,13 @@ +import { proxyActivities } from '@temporalio/workflow' + +import type * as activities from '../activities/syncRunActivities' + +const activity = proxyActivities({ + startToCloseTimeout: '30 minutes', + heartbeatTimeout: '1 minute', + retry: { maximumAttempts: 1 }, +}) + +export async function syncRun(unitId: string): Promise { + await activity.executeSync(unitId) +} diff --git a/services/libs/connectors/src/testing/dummyConnector.ts b/services/libs/connectors/src/testing/dummyConnector.ts new file mode 100644 index 0000000000..f63866084a --- /dev/null +++ b/services/libs/connectors/src/testing/dummyConnector.ts @@ -0,0 +1,18 @@ +import type { Manifest, SyncContext } from '../types' + +const TICK_COUNT = 3 + +export const dummyConnector: Manifest = { + platform: 'dummy', + syncs: [ + { + name: 'ticks', + cadenceMinutes: 60, + run: async (ctx: SyncContext) => { + await ctx.emit(Array.from({ length: TICK_COUNT }, (_, index) => ({ tick: index }))) + await ctx.commitWatermark({ since: new Date().toISOString() }) + }, + }, + ], + discover: async () => [{ channelId: 'dummy-channel', channelName: 'dummy/channel' }], +} From ff1248f3950cd198ed48e8b9296364db5d832d8b Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Wed, 12 Aug 2026 11:06:14 +0100 Subject: [PATCH 11/69] feat: add connectors worker docker setup Signed-off-by: Mouad BANI --- scripts/cli | 2 +- scripts/services/connectors-worker.yaml | 57 +++++++++++++++++++ .../docker/Dockerfile.connectors_worker | 23 ++++++++ 3 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 scripts/services/connectors-worker.yaml create mode 100644 scripts/services/docker/Dockerfile.connectors_worker diff --git a/scripts/cli b/scripts/cli index 0b226a614e..6917ed0bc0 100755 --- a/scripts/cli +++ b/scripts/cli @@ -1189,7 +1189,7 @@ while test $# -gt 0; do exit ;; clean-start-fe-dev) - IGNORED_SERVICES=("frontend" "python-worker" "job-generator" "webhook-api" "profiles-worker" "organizations-enrichment-worker" "merge-suggestions-worker" "members-enrichment-worker" "exports-worker" "entity-merging-worker" "cache-worker" "categorization-worker" "cron-service" "data-sink-worker" "git-integration" "mailing-list-integration" "integration-run-worker" "integration-stream-worker" "nango-webhook-api" "nango-worker" "script-executor-worker" "search-sync-api" "search-sync-worker" "security-best-practices-worker" "snowflake-connectors-worker" "automatic-projects-discovery-worker" "pcc-sync-worker" "projects-evaluation-worker" "bq-dataset-ingest" "cargo-worker" "dockerhub-sync" "github-repos-enricher" "go-worker" "maven-worker" "npm-worker" "nuget-worker" "osv-worker" "packagist-worker" "pypi-worker" "rubygems-worker" "security-contacts-worker") + IGNORED_SERVICES=("frontend" "python-worker" "job-generator" "webhook-api" "profiles-worker" "organizations-enrichment-worker" "merge-suggestions-worker" "members-enrichment-worker" "exports-worker" "entity-merging-worker" "cache-worker" "categorization-worker" "cron-service" "data-sink-worker" "git-integration" "mailing-list-integration" "integration-run-worker" "integration-stream-worker" "nango-webhook-api" "nango-worker" "connectors-worker" "script-executor-worker" "search-sync-api" "search-sync-worker" "security-best-practices-worker" "snowflake-connectors-worker" "automatic-projects-discovery-worker" "pcc-sync-worker" "projects-evaluation-worker" "bq-dataset-ingest" "cargo-worker" "dockerhub-sync" "github-repos-enricher" "go-worker" "maven-worker" "npm-worker" "nuget-worker" "osv-worker" "packagist-worker" "pypi-worker" "rubygems-worker" "security-contacts-worker") CLEAN_START=1 DEV=1 start diff --git a/scripts/services/connectors-worker.yaml b/scripts/services/connectors-worker.yaml new file mode 100644 index 0000000000..140b27639e --- /dev/null +++ b/scripts/services/connectors-worker.yaml @@ -0,0 +1,57 @@ +version: '3.1' + +x-env-args: &env-args + DOCKER_BUILDKIT: 1 + NODE_ENV: docker + SERVICE: connectors-worker + CROWD_TEMPORAL_TASKQUEUE: connectors + SHELL: /bin/sh + +services: + connectors-worker: + build: + context: ../../ + dockerfile: ./scripts/services/docker/Dockerfile.connectors_worker + command: 'pnpm run start' + working_dir: /usr/crowd/app/services/apps/connectors_worker + env_file: + - ../../backend/.env.dist.local + - ../../backend/.env.dist.composed + - ../../backend/.env.override.local + - ../../backend/.env.override.composed + environment: + <<: *env-args + restart: always + networks: + - crowd-bridge + + connectors-worker-dev: + build: + context: ../../ + dockerfile: ./scripts/services/docker/Dockerfile.connectors_worker + command: 'pnpm run dev' + working_dir: /usr/crowd/app/services/apps/connectors_worker + env_file: + - ../../backend/.env.dist.local + - ../../backend/.env.dist.composed + - ../../backend/.env.override.local + - ../../backend/.env.override.composed + environment: + <<: *env-args + hostname: connectors-worker + networks: + - crowd-bridge + volumes: + - ../../services/libs/common/src:/usr/crowd/app/services/libs/common/src + - ../../services/libs/connectors/src:/usr/crowd/app/services/libs/connectors/src + - ../../services/libs/data-access-layer/src:/usr/crowd/app/services/libs/data-access-layer/src + - ../../services/libs/database/src:/usr/crowd/app/services/libs/database/src + - ../../services/libs/logging/src:/usr/crowd/app/services/libs/logging/src + - ../../services/libs/redis/src:/usr/crowd/app/services/libs/redis/src + - ../../services/libs/temporal/src:/usr/crowd/app/services/libs/temporal/src + - ../../services/libs/types/src:/usr/crowd/app/services/libs/types/src + - ../../services/apps/connectors_worker/src:/usr/crowd/app/services/apps/connectors_worker/src + +networks: + crowd-bridge: + external: true diff --git a/scripts/services/docker/Dockerfile.connectors_worker b/scripts/services/docker/Dockerfile.connectors_worker new file mode 100644 index 0000000000..8fb33c1aff --- /dev/null +++ b/scripts/services/docker/Dockerfile.connectors_worker @@ -0,0 +1,23 @@ +FROM node:20-alpine as builder + +RUN apk add --no-cache python3 make g++ + +WORKDIR /usr/crowd/app +RUN npm install -g corepack@latest && corepack enable pnpm && corepack prepare pnpm@9.15.0 --activate + +COPY ./pnpm-workspace.yaml ./pnpm-lock.yaml ./ +RUN pnpm fetch + +COPY ./services ./services +RUN pnpm i --frozen-lockfile + +FROM node:20-bookworm-slim as runner + +WORKDIR /usr/crowd/app +RUN npm install -g corepack@latest && corepack enable pnpm && corepack prepare pnpm@9.15.0 --activate && apt update && apt install -y ca-certificates --no-install-recommends && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /usr/crowd/app/node_modules ./node_modules +COPY --from=builder /usr/crowd/app/services/base.tsconfig.json ./services/base.tsconfig.json +COPY --from=builder /usr/crowd/app/services/libs ./services/libs +COPY --from=builder /usr/crowd/app/services/archetypes/ ./services/archetypes +COPY --from=builder /usr/crowd/app/services/apps/connectors_worker/ ./services/apps/connectors_worker From ab10d0cc35786a044b7d819ebc919cff2128097b Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Wed, 12 Aug 2026 11:13:54 +0100 Subject: [PATCH 12/69] fix: rethrow sync run errors so temporal reflects failures Signed-off-by: Mouad BANI --- .../apps/connectors_worker/src/activities/syncRunActivities.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/services/apps/connectors_worker/src/activities/syncRunActivities.ts b/services/apps/connectors_worker/src/activities/syncRunActivities.ts index bab19f2844..391830360f 100644 --- a/services/apps/connectors_worker/src/activities/syncRunActivities.ts +++ b/services/apps/connectors_worker/src/activities/syncRunActivities.ts @@ -64,6 +64,7 @@ export async function executeSync(unitId: string): Promise { // POC only: everything unclassified is framework.internal; the 7-class // error taxonomy arrives with the M2 HTTP client await recordRunFailure(qx, unitId, 'framework.internal', DEAD_LETTER_AFTER) + throw err } finally { clearInterval(heartbeat) } From 9de976ffd31f32ca200197f60b0515f9ab0483c1 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Wed, 12 Aug 2026 12:37:18 +0100 Subject: [PATCH 13/69] feat: add connectors http core with error taxonomy Signed-off-by: Mouad BANI --- pnpm-lock.yaml | 65 +++---- services/libs/connectors/package.json | 1 + services/libs/connectors/src/http/client.ts | 184 ++++++++++++++++++++ services/libs/connectors/src/http/errors.ts | 84 +++++++++ services/libs/connectors/src/index.ts | 2 + 5 files changed, 296 insertions(+), 40 deletions(-) create mode 100644 services/libs/connectors/src/http/client.ts create mode 100644 services/libs/connectors/src/http/errors.ts diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 63c18dadf8..e7b06665b0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2290,6 +2290,9 @@ importers: '@crowd/logging': specifier: workspace:* version: link:../logging + axios: + specifier: ^1.6.8 + version: 1.16.1 zod: specifier: ^3.22.0 version: 3.25.76 @@ -5719,9 +5722,6 @@ packages: axios@1.13.1: resolution: {integrity: sha512-hU4EGxxt+j7TQijx1oYdAjw4xuIp1wRQSsbMFwSthCWeBQur1eF+qJ5iQ5sN3Tw8YRzQNKb8jszgBdMDVqwJcw==} - axios@1.13.5: - resolution: {integrity: sha512-cz4ur7Vb0xS4/KUN0tPWe44eqxrIu31me+fbang3ijiNscE129POzipJJA6zniq2C/Z6sJCjMimjS8Lc/GAs8Q==} - axios@1.16.1: resolution: {integrity: sha512-caYkukvroVPO8KrzuJEb50Hm07KwfBZPEC3VeFHTsqWHvKTsy54hjJz9BS/cdaypROE2rH6xvm9mHX4fgWkr3A==} @@ -7129,15 +7129,6 @@ packages: fn.name@1.1.0: resolution: {integrity: sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==} - follow-redirects@1.15.11: - resolution: {integrity: sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==} - engines: {node: '>=4.0'} - peerDependencies: - debug: '*' - peerDependenciesMeta: - debug: - optional: true - follow-redirects@1.15.6: resolution: {integrity: sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==} engines: {node: '>=4.0'} @@ -7350,11 +7341,11 @@ packages: glob@6.0.4: resolution: {integrity: sha512-MKZeRNyYZAVVVG1oZeLaWie1uweH40m9AZwIwxyPbTSX4hHrVYSzLg0Ro5Z5R7XKkIX+Cc6oD1rqeDJnwsB8/A==} - deprecated: Glob versions prior to v9 are no longer supported + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me glob@7.2.3: resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==} - deprecated: Glob versions prior to v9 are no longer supported + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me global-directory@4.0.1: resolution: {integrity: sha512-wHTUcDUoZ1H5/0iVqEudYW4/kAlN5cZ3j/bXn0Dpbizl9iaUVeWSHqiOjsgk6OW2bkLclbBjzewBz6weQ1zA2Q==} @@ -11046,8 +11037,8 @@ snapshots: dependencies: '@aws-crypto/sha256-browser': 3.0.0 '@aws-crypto/sha256-js': 3.0.0 - '@aws-sdk/client-sso-oidc': 3.572.0 - '@aws-sdk/client-sts': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) + '@aws-sdk/client-sso-oidc': 3.572.0(@aws-sdk/client-sts@3.572.0) + '@aws-sdk/client-sts': 3.572.0 '@aws-sdk/core': 3.572.0 '@aws-sdk/credential-provider-node': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0)(@aws-sdk/client-sts@3.572.0) '@aws-sdk/middleware-host-header': 3.567.0 @@ -11241,11 +11232,11 @@ snapshots: transitivePeerDependencies: - aws-crt - '@aws-sdk/client-sso-oidc@3.572.0': + '@aws-sdk/client-sso-oidc@3.572.0(@aws-sdk/client-sts@3.572.0)': dependencies: '@aws-crypto/sha256-browser': 3.0.0 '@aws-crypto/sha256-js': 3.0.0 - '@aws-sdk/client-sts': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) + '@aws-sdk/client-sts': 3.572.0 '@aws-sdk/core': 3.572.0 '@aws-sdk/credential-provider-node': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0)(@aws-sdk/client-sts@3.572.0) '@aws-sdk/middleware-host-header': 3.567.0 @@ -11284,6 +11275,7 @@ snapshots: '@smithy/util-utf8': 2.3.0 tslib: 2.6.2 transitivePeerDependencies: + - '@aws-sdk/client-sts' - aws-crt '@aws-sdk/client-sso@3.556.0': @@ -11459,11 +11451,11 @@ snapshots: transitivePeerDependencies: - aws-crt - '@aws-sdk/client-sts@3.572.0(@aws-sdk/client-sso-oidc@3.572.0)': + '@aws-sdk/client-sts@3.572.0': dependencies: '@aws-crypto/sha256-browser': 3.0.0 '@aws-crypto/sha256-js': 3.0.0 - '@aws-sdk/client-sso-oidc': 3.572.0 + '@aws-sdk/client-sso-oidc': 3.572.0(@aws-sdk/client-sts@3.572.0) '@aws-sdk/core': 3.572.0 '@aws-sdk/credential-provider-node': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0)(@aws-sdk/client-sts@3.572.0) '@aws-sdk/middleware-host-header': 3.567.0 @@ -11502,7 +11494,6 @@ snapshots: '@smithy/util-utf8': 2.3.0 tslib: 2.6.2 transitivePeerDependencies: - - '@aws-sdk/client-sso-oidc' - aws-crt '@aws-sdk/client-sts@3.985.0': @@ -11668,7 +11659,7 @@ snapshots: '@aws-sdk/credential-provider-ini@3.572.0(@aws-sdk/client-sso-oidc@3.572.0)(@aws-sdk/client-sts@3.572.0)': dependencies: - '@aws-sdk/client-sts': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) + '@aws-sdk/client-sts': 3.572.0 '@aws-sdk/credential-provider-env': 3.568.0 '@aws-sdk/credential-provider-process': 3.572.0 '@aws-sdk/credential-provider-sso': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) @@ -11845,7 +11836,7 @@ snapshots: '@aws-sdk/credential-provider-web-identity@3.568.0(@aws-sdk/client-sts@3.572.0)': dependencies: - '@aws-sdk/client-sts': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) + '@aws-sdk/client-sts': 3.572.0 '@aws-sdk/types': 3.567.0 '@smithy/property-provider': 2.2.0 '@smithy/types': 2.12.0 @@ -12157,7 +12148,7 @@ snapshots: '@aws-sdk/token-providers@3.572.0(@aws-sdk/client-sso-oidc@3.572.0)': dependencies: - '@aws-sdk/client-sso-oidc': 3.572.0 + '@aws-sdk/client-sso-oidc': 3.572.0(@aws-sdk/client-sts@3.572.0) '@aws-sdk/types': 3.567.0 '@smithy/property-provider': 2.2.0 '@smithy/shared-ini-file-loader': 2.4.0 @@ -13660,9 +13651,10 @@ snapshots: '@sendgrid/client@8.1.3': dependencies: '@sendgrid/helpers': 8.0.0 - axios: 1.13.5 + axios: 1.16.1 transitivePeerDependencies: - debug + - supports-color '@sendgrid/helpers@8.0.0': dependencies: @@ -13674,6 +13666,7 @@ snapshots: '@sendgrid/helpers': 8.0.0 transitivePeerDependencies: - debug + - supports-color '@sindresorhus/is@0.14.0': {} @@ -13691,7 +13684,7 @@ snapshots: '@slack/types': 2.11.0 '@types/is-stream': 1.1.0 '@types/node': 20.12.7 - axios: 1.13.5 + axios: 1.16.1 eventemitter3: 3.1.2 form-data: 2.5.1 is-electron: 2.2.2 @@ -13700,6 +13693,7 @@ snapshots: p-retry: 4.6.2 transitivePeerDependencies: - debug + - supports-color '@slack/webhook@6.1.0': dependencies: @@ -15352,7 +15346,7 @@ snapshots: axios@0.21.4: dependencies: - follow-redirects: 1.15.11 + follow-redirects: 1.16.0 transitivePeerDependencies: - debug @@ -15373,7 +15367,7 @@ snapshots: axios@1.12.0: dependencies: - follow-redirects: 1.15.11 + follow-redirects: 1.16.0 form-data: 4.0.5 proxy-from-env: 1.1.0 transitivePeerDependencies: @@ -15387,14 +15381,6 @@ snapshots: transitivePeerDependencies: - debug - axios@1.13.5: - dependencies: - follow-redirects: 1.15.11 - form-data: 4.0.5 - proxy-from-env: 1.1.0 - transitivePeerDependencies: - - debug - axios@1.16.1: dependencies: follow-redirects: 1.16.0 @@ -17170,8 +17156,6 @@ snapshots: fn.name@1.1.0: {} - follow-redirects@1.15.11: {} - follow-redirects@1.15.6: {} follow-redirects@1.16.0: {} @@ -19177,10 +19161,11 @@ snapshots: peopledatalabs@6.1.5: dependencies: - axios: 1.13.5 + axios: 1.16.1 copy-anything: 3.0.5 transitivePeerDependencies: - debug + - supports-color pg-cloudflare@1.1.1: optional: true @@ -20085,7 +20070,7 @@ snapshots: asn1.js: 5.4.1 asn1.js-rfc2560: 5.0.1(asn1.js@5.4.1) asn1.js-rfc5280: 3.0.0 - axios: 1.13.5 + axios: 1.16.1 big-integer: 1.6.52 bignumber.js: 9.1.2 bn.js: 5.2.1 diff --git a/services/libs/connectors/package.json b/services/libs/connectors/package.json index 2e5d961cda..6346f02d6c 100644 --- a/services/libs/connectors/package.json +++ b/services/libs/connectors/package.json @@ -16,6 +16,7 @@ "@crowd/common": "workspace:*", "@crowd/data-access-layer": "workspace:*", "@crowd/logging": "workspace:*", + "axios": "^1.6.8", "zod": "^3.22.0" } } diff --git a/services/libs/connectors/src/http/client.ts b/services/libs/connectors/src/http/client.ts new file mode 100644 index 0000000000..6712c0b631 --- /dev/null +++ b/services/libs/connectors/src/http/client.ts @@ -0,0 +1,184 @@ +import axios, { AxiosHeaders, AxiosRequestConfig, AxiosResponse } from 'axios' + +import { timeout } from '@crowd/common' +import type { Logger } from '@crowd/logging' + +import { + ConnectorError, + ProviderUnavailableError, + RateLimitError, + errorFromHttpStatus, +} from './errors' + +export interface IPooledToken { + id: string + value: string +} + +export interface HttpResponse { + status: number + headers: Record + data: unknown +} + +export type TokenApplier = (config: AxiosRequestConfig, token: IPooledToken) => AxiosRequestConfig + +export type ResponseInterpreter = (response: HttpResponse) => ConnectorError | null + +export interface HttpClientDeps { + acquireToken: () => Promise + parkToken: (tokenId: string, resumeAt: Date) => Promise + quarantineToken: (tokenId: string) => Promise + correctBudget: (headers: Record) => Promise + log: Logger + applyToken?: TokenApplier + interpretResponse?: ResponseInterpreter +} + +export interface ConnectorHttp { + request(config: AxiosRequestConfig): Promise +} + +const MAX_ATTEMPTS = 3 +const BACKOFF_BASE_MS = 1000 +const RATE_LIMIT_FALLBACK_MS = 60_000 + +export function createHttpClient(deps: HttpClientDeps): ConnectorHttp { + return { + request: (config: AxiosRequestConfig) => requestWithRetry(deps, config), + } +} + +async function requestWithRetry(deps: HttpClientDeps, config: AxiosRequestConfig): Promise { + let lastError: ConnectorError = new ProviderUnavailableError() + for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { + try { + return await attemptRequest(deps, config, true) + } catch (err) { + if (!(err instanceof ConnectorError) || err.errorClass !== 'provider.unavailable') { + throw err + } + lastError = err + if (attempt < MAX_ATTEMPTS) { + const delay = BACKOFF_BASE_MS * 2 ** (attempt - 1) + deps.log.warn({ attempt, delay, reason: err.message }, 'provider unavailable, backing off') + await timeout(delay) + } + } + } + throw lastError +} + +async function attemptRequest( + deps: HttpClientDeps, + config: AxiosRequestConfig, + allowTokenRotation: boolean, +): Promise { + const token = await deps.acquireToken() + const response = await send(deps, config, token) + const headers = normalizeHeaders(response.headers) + const error = classifyResponse(deps, response.status, headers, response.data) + + if (!error) { + await deps.correctBudget(headers) + return response.data + } + + if (error.errorClass === 'provider.rate_limit') { + const resumeAt = error.options?.resumeAt ?? computeResumeAt(headers) + await deps.parkToken(token.id, resumeAt) + if (allowTokenRotation) { + deps.log.info( + { tokenId: token.id, resumeAt }, + 'token rate limited, retrying with fresh token', + ) + return attemptRequest(deps, config, false) + } + throw new RateLimitError(error.message, { ...error.options, resumeAt }) + } + + if (error.errorClass === 'provider.auth') { + await deps.quarantineToken(token.id) + deps.log.warn( + { tokenId: token.id, status: response.status }, + 'token quarantined on auth failure', + ) + } + + throw error +} + +async function send( + deps: HttpClientDeps, + config: AxiosRequestConfig, + token: IPooledToken, +): Promise> { + const applyToken = deps.applyToken ?? applyBearerToken + try { + return await axios.request({ ...applyToken(config, token), validateStatus: () => true }) + } catch (err) { + throw new ProviderUnavailableError('no response from provider', { cause: err }) + } +} + +function applyBearerToken(config: AxiosRequestConfig, token: IPooledToken): AxiosRequestConfig { + const headers = AxiosHeaders.from(config.headers) + if (!headers.has('Authorization')) { + headers.set('Authorization', `Bearer ${token.value}`) + } + return { ...config, headers } +} + +function classifyResponse( + deps: HttpClientDeps, + status: number, + headers: Record, + data: unknown, +): ConnectorError | null { + const custom = deps.interpretResponse?.({ status, headers, data }) + if (custom) { + return custom + } + if (isRateLimited(status, headers)) { + return new RateLimitError(`provider rate limited (status ${status})`, { + status, + resumeAt: computeResumeAt(headers), + }) + } + if (status >= 400) { + return errorFromHttpStatus(status) + } + return null +} + +function isRateLimited(status: number, headers: Record): boolean { + if (status === 429) { + return true + } + if (status === 403 && headers['x-ratelimit-remaining'] === '0') { + return true + } + return 'retry-after' in headers +} + +function computeResumeAt(headers: Record): Date { + const retryAfterSeconds = Number(headers['retry-after']) + if (Number.isFinite(retryAfterSeconds) && retryAfterSeconds > 0) { + return new Date(Date.now() + retryAfterSeconds * 1000) + } + const resetEpochSeconds = Number(headers['x-ratelimit-reset']) + if (Number.isFinite(resetEpochSeconds) && resetEpochSeconds > 0) { + return new Date(resetEpochSeconds * 1000) + } + return new Date(Date.now() + RATE_LIMIT_FALLBACK_MS) +} + +function normalizeHeaders(raw: AxiosResponse['headers']): Record { + const headers: Record = {} + for (const [key, value] of Object.entries(raw)) { + if (value !== undefined && value !== null) { + headers[key.toLowerCase()] = Array.isArray(value) ? value.join(', ') : String(value) + } + } + return headers +} diff --git a/services/libs/connectors/src/http/errors.ts b/services/libs/connectors/src/http/errors.ts new file mode 100644 index 0000000000..7318cd12a6 --- /dev/null +++ b/services/libs/connectors/src/http/errors.ts @@ -0,0 +1,84 @@ +export type ErrorClass = + | 'provider.unavailable' + | 'provider.rate_limit' + | 'provider.auth' + | 'provider.contract' + | 'connector.code' + | 'sink.rejected' + | 'unknown' + +export interface ConnectorErrorOptions { + status?: number + resumeAt?: Date + cause?: unknown +} + +export class ConnectorError extends Error { + constructor( + readonly errorClass: ErrorClass, + message: string, + readonly options?: ConnectorErrorOptions, + ) { + super(message) + this.name = 'ConnectorError' + } +} + +export class ProviderUnavailableError extends ConnectorError { + constructor(message = 'provider unavailable', options?: ConnectorErrorOptions) { + super('provider.unavailable', message, options) + this.name = 'ProviderUnavailableError' + } +} + +export class RateLimitError extends ConnectorError { + constructor(message = 'rate limited by provider', options?: ConnectorErrorOptions) { + super('provider.rate_limit', message, options) + this.name = 'RateLimitError' + } +} + +export class ProviderAuthError extends ConnectorError { + constructor(message = 'provider authentication failed', options?: ConnectorErrorOptions) { + super('provider.auth', message, options) + this.name = 'ProviderAuthError' + } +} + +export class ProviderContractError extends ConnectorError { + constructor(message = 'unexpected provider response', options?: ConnectorErrorOptions) { + super('provider.contract', message, options) + this.name = 'ProviderContractError' + } +} + +export class ConnectorCodeError extends ConnectorError { + constructor(message = 'connector code error', options?: ConnectorErrorOptions) { + super('connector.code', message, options) + this.name = 'ConnectorCodeError' + } +} + +export function errorFromHttpStatus( + status: number | undefined, + message?: string, + options?: ConnectorErrorOptions, +): ConnectorError { + const opts = { ...options, status } + if (status === undefined) { + return new ProviderUnavailableError(message ?? 'no response from provider', opts) + } + if (status === 401 || status === 403) { + return new ProviderAuthError(message ?? `provider returned status ${status}`, opts) + } + if (status === 429) { + return new RateLimitError(message ?? `provider returned status ${status}`, opts) + } + if (status >= 500) { + return new ProviderUnavailableError(message ?? `provider returned status ${status}`, opts) + } + if (status >= 400) { + return new ProviderContractError(message ?? `provider returned status ${status}`, opts) + } + return new ConnectorError('unknown', message ?? `unexpected status ${status}`, opts) +} diff --git a/services/libs/connectors/src/index.ts b/services/libs/connectors/src/index.ts index 6e932df937..4e2d55400e 100644 --- a/services/libs/connectors/src/index.ts +++ b/services/libs/connectors/src/index.ts @@ -1,3 +1,5 @@ export * from './credentials' +export * from './http/client' +export * from './http/errors' export * from './registry' export * from './types' From f76efcc07c03236b5b0daf96fb351880d0bba17c Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Wed, 12 Aug 2026 12:40:01 +0100 Subject: [PATCH 14/69] fix: guard heartbeat callback and add connectors worker dockerignore Signed-off-by: Mouad BANI --- .../Dockerfile.connectors_worker.dockerignore | 18 ++++++++++++++++++ .../src/activities/syncRunActivities.ts | 8 +++++++- 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 scripts/services/docker/Dockerfile.connectors_worker.dockerignore diff --git a/scripts/services/docker/Dockerfile.connectors_worker.dockerignore b/scripts/services/docker/Dockerfile.connectors_worker.dockerignore new file mode 100644 index 0000000000..4b74fc87af --- /dev/null +++ b/scripts/services/docker/Dockerfile.connectors_worker.dockerignore @@ -0,0 +1,18 @@ +**/.git +**/node_modules +**/venv* +**/.webpack +**/.serverless +**/.env +**/.env.* +**/.idea +**/.vscode +**/dist +.vscode/ +.github/ +frontend/ +scripts/ +.flake8 +*.md +Makefile +backend/ diff --git a/services/apps/connectors_worker/src/activities/syncRunActivities.ts b/services/apps/connectors_worker/src/activities/syncRunActivities.ts index 391830360f..fd8390a381 100644 --- a/services/apps/connectors_worker/src/activities/syncRunActivities.ts +++ b/services/apps/connectors_worker/src/activities/syncRunActivities.ts @@ -48,7 +48,13 @@ export async function executeSync(unitId: string): Promise { log, } - const heartbeat = setInterval(() => activityContext.heartbeat(), HEARTBEAT_INTERVAL_MS) + const heartbeat = setInterval(() => { + try { + activityContext.heartbeat() + } catch (err) { + log.warn({ errMsg: (err as Error).message }, 'heartbeat failed') + } + }, HEARTBEAT_INTERVAL_MS) try { const sync = getSync(unit.platform, unit.syncName) From 8a1391b5079d1fa08fe85624542d64aeec75a0a0 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Wed, 12 Aug 2026 12:47:09 +0100 Subject: [PATCH 15/69] fix: restrict rate limit detection to unambiguous signals Signed-off-by: Mouad BANI --- services/libs/connectors/src/http/client.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/services/libs/connectors/src/http/client.ts b/services/libs/connectors/src/http/client.ts index 6712c0b631..cc07cce58a 100644 --- a/services/libs/connectors/src/http/client.ts +++ b/services/libs/connectors/src/http/client.ts @@ -155,10 +155,7 @@ function isRateLimited(status: number, headers: Record): boolean if (status === 429) { return true } - if (status === 403 && headers['x-ratelimit-remaining'] === '0') { - return true - } - return 'retry-after' in headers + return status === 403 && headers['x-ratelimit-remaining'] === '0' } function computeResumeAt(headers: Record): Date { From 5e4117a308e07119a94c604ed732ba9f7ba0762f Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Wed, 12 Aug 2026 17:51:33 +0100 Subject: [PATCH 16/69] fix: always apply pooled token in http client auth Signed-off-by: Mouad BANI --- services/libs/connectors/src/http/client.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/services/libs/connectors/src/http/client.ts b/services/libs/connectors/src/http/client.ts index cc07cce58a..deb60509e4 100644 --- a/services/libs/connectors/src/http/client.ts +++ b/services/libs/connectors/src/http/client.ts @@ -123,9 +123,7 @@ async function send( function applyBearerToken(config: AxiosRequestConfig, token: IPooledToken): AxiosRequestConfig { const headers = AxiosHeaders.from(config.headers) - if (!headers.has('Authorization')) { - headers.set('Authorization', `Bearer ${token.value}`) - } + headers.set('Authorization', `Bearer ${token.value}`) return { ...config, headers } } From b5828760d9c62860670a88e6f813fad869382508 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Wed, 12 Aug 2026 18:14:16 +0100 Subject: [PATCH 17/69] fix: add default http timeout and narrow claimed unit payload Signed-off-by: Mouad BANI --- .../src/activities/dispatcherActivities.ts | 6 +++--- services/libs/connectors/src/http/client.ts | 7 ++++++- .../libs/data-access-layer/src/connectors/syncUnits.ts | 6 +++--- services/libs/data-access-layer/src/connectors/types.ts | 2 ++ 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/services/apps/connectors_worker/src/activities/dispatcherActivities.ts b/services/apps/connectors_worker/src/activities/dispatcherActivities.ts index c5c47a644d..d1f21b1d61 100644 --- a/services/apps/connectors_worker/src/activities/dispatcherActivities.ts +++ b/services/apps/connectors_worker/src/activities/dispatcherActivities.ts @@ -1,6 +1,6 @@ import { getSync } from '@crowd/connectors' import { claimDueUnits, rescheduleUnit } from '@crowd/data-access-layer/src/connectors' -import type { ISyncUnit } from '@crowd/data-access-layer/src/connectors' +import type { IClaimedUnit } from '@crowd/data-access-layer/src/connectors' import { dbStoreQx } from '@crowd/data-access-layer/src/queryExecutor' import { RedisCache } from '@crowd/redis' import { WorkflowIdConflictPolicy, WorkflowIdReusePolicy } from '@crowd/temporal' @@ -12,11 +12,11 @@ const TASK_QUEUE = 'connectors' const HEARTBEAT_TTL_SECONDS = 300 const CADENCE_JITTER_RATIO = 0.1 -export async function claimDue(limit: number): Promise { +export async function claimDue(limit: number): Promise { return claimDueUnits(dbStoreQx(svc.postgres.writer), limit) } -export async function startRun(unit: ISyncUnit): Promise { +export async function startRun(unit: IClaimedUnit): Promise { try { await svc.temporal.workflow.start('syncRun', { taskQueue: TASK_QUEUE, diff --git a/services/libs/connectors/src/http/client.ts b/services/libs/connectors/src/http/client.ts index deb60509e4..4daa5b7196 100644 --- a/services/libs/connectors/src/http/client.ts +++ b/services/libs/connectors/src/http/client.ts @@ -42,6 +42,7 @@ export interface ConnectorHttp { const MAX_ATTEMPTS = 3 const BACKOFF_BASE_MS = 1000 const RATE_LIMIT_FALLBACK_MS = 60_000 +const DEFAULT_TIMEOUT_MS = 60_000 export function createHttpClient(deps: HttpClientDeps): ConnectorHttp { return { @@ -115,7 +116,11 @@ async function send( ): Promise> { const applyToken = deps.applyToken ?? applyBearerToken try { - return await axios.request({ ...applyToken(config, token), validateStatus: () => true }) + return await axios.request({ + timeout: DEFAULT_TIMEOUT_MS, + ...applyToken(config, token), + validateStatus: () => true, + }) } catch (err) { throw new ProviderUnavailableError('no response from provider', { cause: err }) } diff --git a/services/libs/data-access-layer/src/connectors/syncUnits.ts b/services/libs/data-access-layer/src/connectors/syncUnits.ts index 3554b77935..df20b7e5a5 100644 --- a/services/libs/data-access-layer/src/connectors/syncUnits.ts +++ b/services/libs/data-access-layer/src/connectors/syncUnits.ts @@ -1,6 +1,6 @@ import type { QueryExecutor } from '../queryExecutor' -import type { ISyncRunSuccess, ISyncUnit, SyncUnitUpsert } from './types' +import type { IClaimedUnit, ISyncRunSuccess, ISyncUnit, SyncUnitUpsert } from './types' const MIN_INITIAL_DELAY_SECONDS = 10 const MAX_INITIAL_DELAY_SECONDS = 900 @@ -36,7 +36,7 @@ export async function upsertSyncUnits(qx: QueryExecutor, units: SyncUnitUpsert[] ) } -export async function claimDueUnits(qx: QueryExecutor, limit: number): Promise { +export async function claimDueUnits(qx: QueryExecutor, limit: number): Promise { return qx.select( `UPDATE integration.sync_units su SET "lockedAt" = now(), "updatedAt" = now() @@ -55,7 +55,7 @@ export async function claimDueUnits(qx: QueryExecutor, limit: number): Promise +export type IClaimedUnit = Pick + export interface ISyncRunSuccess { watermark: Record emittedCount: number From fac2c6289b2fd1b434dbce533303cb528847225b Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Thu, 13 Aug 2026 12:55:41 +0100 Subject: [PATCH 18/69] feat: add redis token pool for connectors Signed-off-by: Mouad BANI --- pnpm-lock.yaml | 23 ++-- services/libs/connectors/package.json | 1 + services/libs/connectors/src/index.ts | 1 + .../libs/connectors/src/pool/tokenPool.ts | 115 ++++++++++++++++++ 4 files changed, 130 insertions(+), 10 deletions(-) create mode 100644 services/libs/connectors/src/pool/tokenPool.ts diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e7b06665b0..b0707d7c7c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2290,6 +2290,9 @@ importers: '@crowd/logging': specifier: workspace:* version: link:../logging + '@crowd/redis': + specifier: workspace:* + version: link:../redis axios: specifier: ^1.6.8 version: 1.16.1 @@ -11037,8 +11040,8 @@ snapshots: dependencies: '@aws-crypto/sha256-browser': 3.0.0 '@aws-crypto/sha256-js': 3.0.0 - '@aws-sdk/client-sso-oidc': 3.572.0(@aws-sdk/client-sts@3.572.0) - '@aws-sdk/client-sts': 3.572.0 + '@aws-sdk/client-sso-oidc': 3.572.0 + '@aws-sdk/client-sts': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) '@aws-sdk/core': 3.572.0 '@aws-sdk/credential-provider-node': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0)(@aws-sdk/client-sts@3.572.0) '@aws-sdk/middleware-host-header': 3.567.0 @@ -11232,11 +11235,11 @@ snapshots: transitivePeerDependencies: - aws-crt - '@aws-sdk/client-sso-oidc@3.572.0(@aws-sdk/client-sts@3.572.0)': + '@aws-sdk/client-sso-oidc@3.572.0': dependencies: '@aws-crypto/sha256-browser': 3.0.0 '@aws-crypto/sha256-js': 3.0.0 - '@aws-sdk/client-sts': 3.572.0 + '@aws-sdk/client-sts': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) '@aws-sdk/core': 3.572.0 '@aws-sdk/credential-provider-node': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0)(@aws-sdk/client-sts@3.572.0) '@aws-sdk/middleware-host-header': 3.567.0 @@ -11275,7 +11278,6 @@ snapshots: '@smithy/util-utf8': 2.3.0 tslib: 2.6.2 transitivePeerDependencies: - - '@aws-sdk/client-sts' - aws-crt '@aws-sdk/client-sso@3.556.0': @@ -11451,11 +11453,11 @@ snapshots: transitivePeerDependencies: - aws-crt - '@aws-sdk/client-sts@3.572.0': + '@aws-sdk/client-sts@3.572.0(@aws-sdk/client-sso-oidc@3.572.0)': dependencies: '@aws-crypto/sha256-browser': 3.0.0 '@aws-crypto/sha256-js': 3.0.0 - '@aws-sdk/client-sso-oidc': 3.572.0(@aws-sdk/client-sts@3.572.0) + '@aws-sdk/client-sso-oidc': 3.572.0 '@aws-sdk/core': 3.572.0 '@aws-sdk/credential-provider-node': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0)(@aws-sdk/client-sts@3.572.0) '@aws-sdk/middleware-host-header': 3.567.0 @@ -11494,6 +11496,7 @@ snapshots: '@smithy/util-utf8': 2.3.0 tslib: 2.6.2 transitivePeerDependencies: + - '@aws-sdk/client-sso-oidc' - aws-crt '@aws-sdk/client-sts@3.985.0': @@ -11659,7 +11662,7 @@ snapshots: '@aws-sdk/credential-provider-ini@3.572.0(@aws-sdk/client-sso-oidc@3.572.0)(@aws-sdk/client-sts@3.572.0)': dependencies: - '@aws-sdk/client-sts': 3.572.0 + '@aws-sdk/client-sts': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) '@aws-sdk/credential-provider-env': 3.568.0 '@aws-sdk/credential-provider-process': 3.572.0 '@aws-sdk/credential-provider-sso': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) @@ -11836,7 +11839,7 @@ snapshots: '@aws-sdk/credential-provider-web-identity@3.568.0(@aws-sdk/client-sts@3.572.0)': dependencies: - '@aws-sdk/client-sts': 3.572.0 + '@aws-sdk/client-sts': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) '@aws-sdk/types': 3.567.0 '@smithy/property-provider': 2.2.0 '@smithy/types': 2.12.0 @@ -12148,7 +12151,7 @@ snapshots: '@aws-sdk/token-providers@3.572.0(@aws-sdk/client-sso-oidc@3.572.0)': dependencies: - '@aws-sdk/client-sso-oidc': 3.572.0(@aws-sdk/client-sts@3.572.0) + '@aws-sdk/client-sso-oidc': 3.572.0 '@aws-sdk/types': 3.567.0 '@smithy/property-provider': 2.2.0 '@smithy/shared-ini-file-loader': 2.4.0 diff --git a/services/libs/connectors/package.json b/services/libs/connectors/package.json index 6346f02d6c..d6d839b833 100644 --- a/services/libs/connectors/package.json +++ b/services/libs/connectors/package.json @@ -16,6 +16,7 @@ "@crowd/common": "workspace:*", "@crowd/data-access-layer": "workspace:*", "@crowd/logging": "workspace:*", + "@crowd/redis": "workspace:*", "axios": "^1.6.8", "zod": "^3.22.0" } diff --git a/services/libs/connectors/src/index.ts b/services/libs/connectors/src/index.ts index 4e2d55400e..d5b10139c0 100644 --- a/services/libs/connectors/src/index.ts +++ b/services/libs/connectors/src/index.ts @@ -1,5 +1,6 @@ export * from './credentials' export * from './http/client' export * from './http/errors' +export * from './pool/tokenPool' export * from './registry' export * from './types' diff --git a/services/libs/connectors/src/pool/tokenPool.ts b/services/libs/connectors/src/pool/tokenPool.ts new file mode 100644 index 0000000000..b62685d0fa --- /dev/null +++ b/services/libs/connectors/src/pool/tokenPool.ts @@ -0,0 +1,115 @@ +import type { RedisClient } from '@crowd/redis' + +import type { IPooledToken } from '../http/client' +import { ProviderAuthError, RateLimitError } from '../http/errors' + +interface ITokenState { + value: string + parkedUntil?: string + quarantined?: boolean +} + +export interface TokenPool { + acquire(): Promise + park(tokenId: string, resumeAt: Date): Promise + quarantine(tokenId: string): Promise + seed(tokenId: string, value: string): Promise + earliestResumeAt(): Promise +} + +export function createTokenPool( + redis: RedisClient, + platform: string, + connectionId: string, +): TokenPool { + const tokensKey = `connectors:pool:${platform}:${connectionId}:tokens` + const lruKey = `connectors:pool:${platform}:${connectionId}:lru` + + async function readStates(): Promise> { + const raw = await redis.hGetAll(tokensKey) + const states = new Map() + for (const [id, json] of Object.entries(raw)) { + states.set(id, JSON.parse(json) as ITokenState) + } + return states + } + + function isHealthy(state: ITokenState, nowMs: number): boolean { + if (state.quarantined) { + return false + } + if (state.parkedUntil && new Date(state.parkedUntil).getTime() > nowMs) { + return false + } + return true + } + + function earliestParkedUntil(states: Map, nowMs: number): Date | null { + let earliest: Date | null = null + for (const state of states.values()) { + if (state.quarantined || !state.parkedUntil) { + continue + } + const parkedUntil = new Date(state.parkedUntil) + if (parkedUntil.getTime() <= nowMs) { + continue + } + if (!earliest || parkedUntil < earliest) { + earliest = parkedUntil + } + } + return earliest + } + + // POC only: read-modify-write can lose a concurrent park/quarantine on the same token + // within a ~ms window; accepted tradeoff — fix with atomic writes when productizing. + async function updateState(tokenId: string, update: Partial): Promise { + const json = await redis.hGet(tokensKey, tokenId) + if (!json) { + return + } + const state = JSON.parse(json) as ITokenState + await redis.hSet(tokensKey, tokenId, JSON.stringify({ ...state, ...update })) + } + + return { + async acquire(): Promise { + const nowMs = Date.now() + const states = await readStates() + const ordered = await redis.zRange(lruKey, 0, -1) + for (const id of ordered) { + const state = states.get(id) + if (state && isHealthy(state, nowMs)) { + await redis.zAdd(lruKey, { score: nowMs, value: id }) + return { id, value: state.value } + } + } + const resumeAt = earliestParkedUntil(states, nowMs) + if (resumeAt) { + throw new RateLimitError('token pool exhausted', { resumeAt }) + } + throw new ProviderAuthError('token pool empty') + }, + + async park(tokenId: string, resumeAt: Date): Promise { + await updateState(tokenId, { parkedUntil: resumeAt.toISOString() }) + }, + + // POC only: quarantined tokens are kept for inspection and never revived automatically + async quarantine(tokenId: string): Promise { + await updateState(tokenId, { quarantined: true }) + }, + + async seed(tokenId: string, value: string): Promise { + const json = await redis.hGet(tokensKey, tokenId) + const state = json ? (JSON.parse(json) as ITokenState) : {} + await redis.hSet(tokensKey, tokenId, JSON.stringify({ ...state, value })) + await redis.zAdd(lruKey, { score: 0, value: tokenId }, { NX: true }) + }, + + async earliestResumeAt(): Promise { + const states = await readStates() + return earliestParkedUntil(states, Date.now()) + }, + } +} From 032d4cdde29f22c4fdd0598ba0342bc62f6fc6c2 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Thu, 13 Aug 2026 17:26:09 +0100 Subject: [PATCH 19/69] feat: add token budgets with dispatcher admission Signed-off-by: Mouad BANI --- .../apps/connectors_worker/src/activities.ts | 11 +- .../src/activities/dispatcherActivities.ts | 26 +++- services/apps/connectors_worker/src/types.ts | 7 + .../src/workflows/dispatcher.ts | 8 +- .../libs/connectors/src/pool/tokenPool.ts | 139 +++++++++++++++++- 5 files changed, 178 insertions(+), 13 deletions(-) diff --git a/services/apps/connectors_worker/src/activities.ts b/services/apps/connectors_worker/src/activities.ts index 585832d864..054ba53dff 100644 --- a/services/apps/connectors_worker/src/activities.ts +++ b/services/apps/connectors_worker/src/activities.ts @@ -1,4 +1,11 @@ -import { claimDue, reschedule, startRun, touchHeartbeat } from './activities/dispatcherActivities' +import { + admitByBudget, + claimDue, + deferUnit, + reschedule, + startRun, + touchHeartbeat, +} from './activities/dispatcherActivities' import { executeSync } from './activities/syncRunActivities' -export { claimDue, executeSync, reschedule, startRun, touchHeartbeat } +export { admitByBudget, claimDue, deferUnit, executeSync, reschedule, startRun, touchHeartbeat } diff --git a/services/apps/connectors_worker/src/activities/dispatcherActivities.ts b/services/apps/connectors_worker/src/activities/dispatcherActivities.ts index d1f21b1d61..e64e2ed876 100644 --- a/services/apps/connectors_worker/src/activities/dispatcherActivities.ts +++ b/services/apps/connectors_worker/src/activities/dispatcherActivities.ts @@ -1,4 +1,4 @@ -import { getSync } from '@crowd/connectors' +import { createTokenPool, getSync } from '@crowd/connectors' import { claimDueUnits, rescheduleUnit } from '@crowd/data-access-layer/src/connectors' import type { IClaimedUnit } from '@crowd/data-access-layer/src/connectors' import { dbStoreQx } from '@crowd/data-access-layer/src/queryExecutor' @@ -6,16 +6,38 @@ import { RedisCache } from '@crowd/redis' import { WorkflowIdConflictPolicy, WorkflowIdReusePolicy } from '@crowd/temporal' import { svc } from '../main' -import type { StartRunResult } from '../types' +import type { IAdmissionResult, StartRunResult } from '../types' const TASK_QUEUE = 'connectors' const HEARTBEAT_TTL_SECONDS = 300 const CADENCE_JITTER_RATIO = 0.1 +const DEFAULT_RUN_ESTIMATE = 50 +const DEFER_MIN_MS = 30_000 +const DEFER_JITTER_MS = 60_000 export async function claimDue(limit: number): Promise { return claimDueUnits(dbStoreQx(svc.postgres.writer), limit) } +export async function admitByBudget(units: IClaimedUnit[]): Promise { + const admitted: IClaimedUnit[] = [] + const deferred: IClaimedUnit[] = [] + for (const unit of units) { + const pool = createTokenPool(svc.redis, unit.platform, unit.integrationId) + if (await pool.hasHeadroom(DEFAULT_RUN_ESTIMATE)) { + admitted.push(unit) + } else { + deferred.push(unit) + } + } + return { admitted, deferred } +} + +export async function deferUnit(unitId: string): Promise { + const delayMs = DEFER_MIN_MS + Math.random() * DEFER_JITTER_MS + await rescheduleUnit(dbStoreQx(svc.postgres.writer), unitId, new Date(Date.now() + delayMs)) +} + export async function startRun(unit: IClaimedUnit): Promise { try { await svc.temporal.workflow.start('syncRun', { diff --git a/services/apps/connectors_worker/src/types.ts b/services/apps/connectors_worker/src/types.ts index ac915242b1..31f1c992c0 100644 --- a/services/apps/connectors_worker/src/types.ts +++ b/services/apps/connectors_worker/src/types.ts @@ -1 +1,8 @@ +import type { IClaimedUnit } from '@crowd/data-access-layer/src/connectors' + export type StartRunResult = 'started' | 'alreadyRunning' + +export interface IAdmissionResult { + admitted: IClaimedUnit[] + deferred: IClaimedUnit[] +} diff --git a/services/apps/connectors_worker/src/workflows/dispatcher.ts b/services/apps/connectors_worker/src/workflows/dispatcher.ts index 5908059622..f3d800e79c 100644 --- a/services/apps/connectors_worker/src/workflows/dispatcher.ts +++ b/services/apps/connectors_worker/src/workflows/dispatcher.ts @@ -14,7 +14,9 @@ export async function dispatcher(): Promise { const units = await activity.claimDue(CLAIM_LIMIT) - for (const unit of units) { + const { admitted, deferred } = await activity.admitByBudget(units) + + for (const unit of admitted) { try { await activity.startRun(unit) await activity.reschedule(unit.id, unit.platform, unit.syncName) @@ -22,4 +24,8 @@ export async function dispatcher(): Promise { log.error('failed to dispatch sync unit', { unitId: unit.id, err }) } } + + for (const unit of deferred) { + await activity.deferUnit(unit.id) + } } diff --git a/services/libs/connectors/src/pool/tokenPool.ts b/services/libs/connectors/src/pool/tokenPool.ts index b62685d0fa..53d74ce529 100644 --- a/services/libs/connectors/src/pool/tokenPool.ts +++ b/services/libs/connectors/src/pool/tokenPool.ts @@ -3,27 +3,58 @@ import type { RedisClient } from '@crowd/redis' import type { IPooledToken } from '../http/client' import { ProviderAuthError, RateLimitError } from '../http/errors' -interface ITokenState { - value: string - parkedUntil?: string - quarantined?: boolean +const PROBE_STALENESS_MS = 90_000 + +export interface BudgetSnapshot { + limit: number + remaining: number + resetAt: Date +} + +export type BudgetProbe = ( + platform: string, + connectionId: string, + tokenId: string, +) => Promise + +// POC only: the probe is the single source of truth for budgets (github /rate_limit is free and +// limits are per installation token); budgets for other platforms are a later decision. +export interface TokenPoolOptions { + probeBudget?: BudgetProbe } export interface TokenPool { acquire(): Promise + hasHeadroom(estimate: number): Promise park(tokenId: string, resumeAt: Date): Promise quarantine(tokenId: string): Promise seed(tokenId: string, value: string): Promise earliestResumeAt(): Promise } +interface ITokenState { + value: string + parkedUntil?: string + quarantined?: boolean +} + +interface IBucket { + limit: number + remaining: number + resetAtMs: number + probedAtMs: number +} + export function createTokenPool( redis: RedisClient, platform: string, connectionId: string, + options?: TokenPoolOptions, ): TokenPool { const tokensKey = `connectors:pool:${platform}:${connectionId}:tokens` const lruKey = `connectors:pool:${platform}:${connectionId}:lru` + const bucketKey = (tokenId: string) => + `connectors:pool:${platform}:${connectionId}:budget:${tokenId}` async function readStates(): Promise> { const raw = await redis.hGetAll(tokensKey) @@ -72,25 +103,117 @@ export function createTokenPool( await redis.hSet(tokensKey, tokenId, JSON.stringify({ ...state, ...update })) } + async function readBucket(tokenId: string): Promise { + const raw = await redis.hGetAll(bucketKey(tokenId)) + if (!raw.probedAt) { + return null + } + return { + limit: Number(raw.limit), + remaining: Number(raw.remaining), + resetAtMs: Number(raw.resetAt), + probedAtMs: Number(raw.probedAt), + } + } + + function needsProbe(bucket: IBucket | null, nowMs: number): boolean { + return !bucket || nowMs - bucket.probedAtMs > PROBE_STALENESS_MS || nowMs >= bucket.resetAtMs + } + + async function loadBucket( + probe: BudgetProbe, + tokenId: string, + nowMs: number, + ): Promise { + const bucket = await readBucket(tokenId) + if (!needsProbe(bucket, nowMs)) { + return bucket + } + const snapshot = await probe(platform, connectionId, tokenId) + if (!snapshot) { + return null + } + const probed = { + limit: snapshot.limit, + remaining: snapshot.remaining, + resetAtMs: snapshot.resetAt.getTime(), + probedAtMs: nowMs, + } + await redis.hSet(bucketKey(tokenId), { + limit: String(probed.limit), + remaining: String(probed.remaining), + resetAt: String(probed.resetAtMs), + probedAt: String(probed.probedAtMs), + }) + return probed + } + return { async acquire(): Promise { const nowMs = Date.now() const states = await readStates() const ordered = await redis.zRange(lruKey, 0, -1) + const probe = options?.probeBudget + let earliestBudgetResetAt: Date | null = null for (const id of ordered) { const state = states.get(id) - if (state && isHealthy(state, nowMs)) { - await redis.zAdd(lruKey, { score: nowMs, value: id }) - return { id, value: state.value } + if (!state || !isHealthy(state, nowMs)) { + continue } + if (probe) { + const bucket = await loadBucket(probe, id, nowMs) + if (bucket && bucket.remaining <= 0) { + const resetAt = new Date(bucket.resetAtMs) + if (!earliestBudgetResetAt || resetAt < earliestBudgetResetAt) { + earliestBudgetResetAt = resetAt + } + continue + } + if (bucket) { + await redis.hIncrBy(bucketKey(id), 'remaining', -1) + } + } + await redis.zAdd(lruKey, { score: nowMs, value: id }) + return { id, value: state.value } } - const resumeAt = earliestParkedUntil(states, nowMs) + const parkedResumeAt = earliestParkedUntil(states, nowMs) + const resumeAt = + parkedResumeAt && earliestBudgetResetAt + ? new Date(Math.min(parkedResumeAt.getTime(), earliestBudgetResetAt.getTime())) + : (parkedResumeAt ?? earliestBudgetResetAt) if (resumeAt) { throw new RateLimitError('token pool exhausted', { resumeAt }) } throw new ProviderAuthError('token pool empty') }, + async hasHeadroom(estimate: number): Promise { + const probe = options?.probeBudget + if (!probe) { + return true + } + const nowMs = Date.now() + const states = await readStates() + if (states.size === 0) { + return true + } + let pooledRemaining = 0 + for (const [id, state] of states.entries()) { + if (!isHealthy(state, nowMs)) { + continue + } + const bucket = await loadBucket(probe, id, nowMs) + if (!bucket) { + return true + } + pooledRemaining += bucket.remaining + if (pooledRemaining >= estimate) { + return true + } + } + return false + }, + async park(tokenId: string, resumeAt: Date): Promise { await updateState(tokenId, { parkedUntil: resumeAt.toISOString() }) }, From 1f5324b7713183c78a02c1312392198b075830f5 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Thu, 27 Aug 2026 11:15:01 +0100 Subject: [PATCH 20/69] feat: add emit with zod validation reusing legacy results publishing Signed-off-by: Mouad BANI --- pnpm-lock.yaml | 6 ++ services/libs/connectors/package.json | 2 + services/libs/connectors/src/emit.ts | 56 +++++++++++++++++++ services/libs/connectors/src/index.ts | 1 + .../connectors/src/testing/dummyConnector.ts | 3 + services/libs/connectors/src/types.ts | 3 + 6 files changed, 71 insertions(+) create mode 100644 services/libs/connectors/src/emit.ts diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b0707d7c7c..11050f044d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2284,6 +2284,9 @@ importers: '@crowd/common': specifier: workspace:* version: link:../common + '@crowd/common_services': + specifier: workspace:* + version: link:../common_services '@crowd/data-access-layer': specifier: workspace:* version: link:../data-access-layer @@ -2293,6 +2296,9 @@ importers: '@crowd/redis': specifier: workspace:* version: link:../redis + '@crowd/types': + specifier: workspace:* + version: link:../types axios: specifier: ^1.6.8 version: 1.16.1 diff --git a/services/libs/connectors/package.json b/services/libs/connectors/package.json index d6d839b833..5d697977c2 100644 --- a/services/libs/connectors/package.json +++ b/services/libs/connectors/package.json @@ -14,9 +14,11 @@ }, "dependencies": { "@crowd/common": "workspace:*", + "@crowd/common_services": "workspace:*", "@crowd/data-access-layer": "workspace:*", "@crowd/logging": "workspace:*", "@crowd/redis": "workspace:*", + "@crowd/types": "workspace:*", "axios": "^1.6.8", "zod": "^3.22.0" } diff --git a/services/libs/connectors/src/emit.ts b/services/libs/connectors/src/emit.ts new file mode 100644 index 0000000000..e724ccbf44 --- /dev/null +++ b/services/libs/connectors/src/emit.ts @@ -0,0 +1,56 @@ +import { ZodError, ZodType } from 'zod' + +import type { DataSinkWorkerEmitter } from '@crowd/common_services' +import type { ISyncUnit } from '@crowd/data-access-layer/src/connectors' +import type { Logger } from '@crowd/logging' +import { IIntegrationResult, IntegrationResultType } from '@crowd/types' + +import { ConnectorError } from './http/errors' + +export interface EmitterDeps { + publishResult: (integrationId: string, result: IIntegrationResult) => Promise + sinkEmitter: DataSinkWorkerEmitter + unit: ISyncUnit + segmentId: string + schema: ZodType + log: Logger +} + +export interface Emitter { + emit: (records: unknown[]) => Promise + emittedCount: () => number +} + +export function createEmit(deps: EmitterDeps): Emitter { + let emitted = 0 + + const emit = async (records: unknown[]): Promise => { + for (const record of records) { + try { + deps.schema.parse(record) + } catch (err) { + if (err instanceof ZodError) { + throw new ConnectorError('connector.code', 'record failed schema validation', { + cause: err, + }) + } + throw err + } + + const payload = { ...(record as Record), channel: deps.unit.channelName } + + const resultId = await deps.publishResult(deps.unit.integrationId, { + type: IntegrationResultType.ACTIVITY, + segmentId: deps.segmentId, + data: payload, + }) + await deps.sinkEmitter.triggerResultProcessing(resultId, resultId, false) + + emitted += 1 + } + + deps.log.debug({ count: records.length, total: emitted }, 'emitted records') + } + + return { emit, emittedCount: () => emitted } +} diff --git a/services/libs/connectors/src/index.ts b/services/libs/connectors/src/index.ts index d5b10139c0..0e212766d4 100644 --- a/services/libs/connectors/src/index.ts +++ b/services/libs/connectors/src/index.ts @@ -1,4 +1,5 @@ export * from './credentials' +export * from './emit' export * from './http/client' export * from './http/errors' export * from './pool/tokenPool' diff --git a/services/libs/connectors/src/testing/dummyConnector.ts b/services/libs/connectors/src/testing/dummyConnector.ts index f63866084a..d67d26a676 100644 --- a/services/libs/connectors/src/testing/dummyConnector.ts +++ b/services/libs/connectors/src/testing/dummyConnector.ts @@ -1,3 +1,5 @@ +import { z } from 'zod' + import type { Manifest, SyncContext } from '../types' const TICK_COUNT = 3 @@ -8,6 +10,7 @@ export const dummyConnector: Manifest = { { name: 'ticks', cadenceMinutes: 60, + schema: z.record(z.unknown()), run: async (ctx: SyncContext) => { await ctx.emit(Array.from({ length: TICK_COUNT }, (_, index) => ({ tick: index }))) await ctx.commitWatermark({ since: new Date().toISOString() }) diff --git a/services/libs/connectors/src/types.ts b/services/libs/connectors/src/types.ts index 9e60f1c91c..a1a6e4f562 100644 --- a/services/libs/connectors/src/types.ts +++ b/services/libs/connectors/src/types.ts @@ -1,3 +1,5 @@ +import type { ZodType } from 'zod' + import type { Logger } from '@crowd/logging' export interface Channel { @@ -29,6 +31,7 @@ export interface SyncContext { export interface SyncDefinition { name: string cadenceMinutes: number + schema: ZodType run: (ctx: SyncContext) => Promise } From 5a88aedb0a924ddc37e6a3dd68a69a695eee29e6 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Thu, 27 Aug 2026 11:20:54 +0100 Subject: [PATCH 21/69] chore: sync pnpm lockfile with main Signed-off-by: Mouad BANI --- pnpm-lock.yaml | 2495 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 1694 insertions(+), 801 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 11050f044d..5483fd1a81 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -10,7 +10,7 @@ importers: devDependencies: '@commitlint/cli': specifier: ^19.8.0 - version: 19.8.0(@types/node@22.19.10)(typescript@5.6.3) + version: 19.8.0(@types/node@24.13.3)(typescript@5.6.3) '@commitlint/config-conventional': specifier: ^19.8.0 version: 19.8.0 @@ -19,10 +19,10 @@ importers: version: 9.1.7 vite: specifier: 6.3.5 - version: 6.3.5(@types/node@22.19.10)(jiti@2.4.2)(terser@5.43.1)(yaml@2.7.0) + version: 6.3.5(@types/node@24.13.3)(jiti@2.4.2)(terser@5.43.1)(tsx@4.23.12)(yaml@2.7.0) vitest: specifier: 4.1.7 - version: 4.1.7(@types/node@22.19.10)(vite@6.3.5(@types/node@22.19.10)(jiti@2.4.2)(terser@5.43.1)(yaml@2.7.0)) + version: 4.1.7(@types/node@24.13.3)(vite@6.3.5(@types/node@24.13.3)(jiti@2.4.2)(terser@5.43.1)(tsx@4.23.12)(yaml@2.7.0)) .github/actions/node: dependencies: @@ -41,7 +41,7 @@ importers: devDependencies: '@types/node': specifier: ^20.10.6 - version: 20.12.7 + version: 20.19.43 '@typescript-eslint/eslint-plugin': specifier: ^6.7.4 version: 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.6.3))(eslint@8.57.0)(typescript@5.6.3) @@ -65,7 +65,7 @@ importers: version: 3.2.5 ts-node: specifier: ^10.9.1 - version: 10.9.2(@swc/core@1.4.17)(@types/node@20.12.7)(typescript@5.6.3) + version: 10.9.2(@swc/core@1.4.17)(@types/node@20.19.43)(typescript@5.6.3) typescript: specifier: ^5.6.3 version: 5.6.3 @@ -158,7 +158,7 @@ importers: version: 2.11.0 '@slack/web-api': specifier: ^6.7.2 - version: 6.12.0 + version: 6.13.0 analytics-node: specifier: ^6.2.0 version: 6.2.0 @@ -200,7 +200,7 @@ importers: version: 6.1.3 config: specifier: ^3.3.8 - version: 3.3.11 + version: 3.3.12 cors: specifier: 2.8.5 version: 2.8.5 @@ -369,7 +369,7 @@ importers: version: 8.1.1 '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 '@types/sanitize-html': specifier: ^2.6.2 version: 2.11.0 @@ -423,7 +423,7 @@ importers: version: 7.5.0(encoding@0.1.13)(openapi-types@12.1.3) tsx: specifier: ^4.7.1 - version: 4.7.3 + version: 4.23.12 typescript: specifier: ^5.6.3 version: 5.6.3 @@ -495,14 +495,14 @@ importers: version: 5.5.6 tsx: specifier: ^4.7.1 - version: 4.7.3 + version: 4.23.12 typescript: specifier: ^5.6.3 version: 5.6.3 devDependencies: '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 nodemon: specifier: ^3.0.1 version: 3.1.0 @@ -541,14 +541,14 @@ importers: version: 2.29.4 tsx: specifier: ^4.7.1 - version: 4.7.3 + version: 4.23.12 typescript: specifier: ^5.6.3 version: 5.6.3 devDependencies: '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 nodemon: specifier: ^3.0.1 version: 3.1.0 @@ -596,14 +596,14 @@ importers: version: 9.0.5 tsx: specifier: ^4.7.1 - version: 4.7.3 + version: 4.23.12 typescript: specifier: ^5.6.3 version: 5.6.3 devDependencies: '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 nodemon: specifier: ^3.0.1 version: 3.1.0 @@ -648,14 +648,14 @@ importers: version: 1.17.2 tsx: specifier: ^4.7.1 - version: 4.7.3 + version: 4.23.12 typescript: specifier: ^5.6.3 version: 5.6.3 devDependencies: '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 nodemon: specifier: ^3.0.1 version: 3.1.0 @@ -721,7 +721,7 @@ importers: version: 3.0.2 tsx: specifier: ^4.7.1 - version: 4.7.3 + version: 4.23.12 typescript: specifier: ^5.6.3 version: 5.6.3 @@ -731,7 +731,7 @@ importers: version: 4.0.9 '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 nodemon: specifier: ^3.0.1 version: 3.1.0 @@ -776,10 +776,10 @@ importers: version: 3.3.4 '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 config: specifier: ^3.3.9 - version: 3.3.11 + version: 3.3.12 crowd-sentiment: specifier: ^1.1.7 version: 1.1.7 @@ -797,7 +797,7 @@ importers: version: 0.5.45 tsx: specifier: ^4.7.1 - version: 4.7.3 + version: 4.23.12 typescript: specifier: ^5.6.3 version: 5.6.3 @@ -843,14 +843,14 @@ importers: version: 1.17.2 tsx: specifier: ^4.7.1 - version: 4.7.3 + version: 4.23.12 typescript: specifier: ^5.6.3 version: 5.6.3 devDependencies: '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 nodemon: specifier: ^3.0.1 version: 3.1.0 @@ -913,14 +913,14 @@ importers: version: 2.30.1 tsx: specifier: ^4.7.1 - version: 4.7.3 + version: 4.23.12 typescript: specifier: ^5.6.3 version: 5.6.3 devDependencies: '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 '@types/uuid': specifier: ~9.0.6 version: 9.0.8 @@ -959,13 +959,13 @@ importers: version: link:../../libs/types axios: specifier: ^1.6.8 - version: 1.6.8 + version: 1.19.0 config: specifier: ^3.3.9 - version: 3.3.11 + version: 3.3.12 tsx: specifier: ^4.7.1 - version: 4.7.3 + version: 4.23.12 typescript: specifier: ^5.6.3 version: 5.6.3 @@ -975,7 +975,7 @@ importers: version: 3.3.4 '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 nodemon: specifier: ^2.0.22 version: 2.0.22 @@ -1008,10 +1008,10 @@ importers: version: link:../../libs/types config: specifier: ^3.3.9 - version: 3.3.11 + version: 3.3.12 tsx: specifier: ^4.7.1 - version: 4.7.3 + version: 4.23.12 typescript: specifier: ^5.6.3 version: 5.6.3 @@ -1021,7 +1021,7 @@ importers: version: 3.3.4 '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 nodemon: specifier: ^2.0.22 version: 2.0.22 @@ -1036,7 +1036,7 @@ importers: version: 4.17.21 '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 bunyan-middleware: specifier: ^1.0.2 version: 1.0.2 @@ -1048,7 +1048,7 @@ importers: version: 15.1.3 tsx: specifier: ^4.7.1 - version: 4.7.3 + version: 4.23.12 typescript: specifier: ^5.6.3 version: 5.6.3 @@ -1109,14 +1109,14 @@ importers: version: 2.29.4 tsx: specifier: ^4.7.1 - version: 4.7.3 + version: 4.23.12 typescript: specifier: ^5.6.3 version: 5.6.3 devDependencies: '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 '@types/uuid': specifier: ~9.0.6 version: 9.0.8 @@ -1161,7 +1161,7 @@ importers: version: 1.17.2 axios: specifier: ^1.6.8 - version: 1.6.8 + version: 1.19.0 fast-levenshtein: specifier: ^3.0.0 version: 3.0.0 @@ -1170,14 +1170,14 @@ importers: version: 4.7.0 tsx: specifier: ^4.7.1 - version: 4.7.3 + version: 4.23.12 typescript: specifier: ^5.6.3 version: 5.6.3 devDependencies: '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 nodemon: specifier: ^3.0.1 version: 3.1.0 @@ -1216,14 +1216,14 @@ importers: version: 4.21.2 tsx: specifier: ^4.7.1 - version: 4.7.3 + version: 4.23.12 typescript: specifier: ^5.6.3 version: 5.6.3 devDependencies: '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 nodemon: specifier: ^3.0.1 version: 3.1.0 @@ -1274,14 +1274,14 @@ importers: version: 9.0.5 tsx: specifier: ^4.7.1 - version: 4.7.3 + version: 4.23.12 typescript: specifier: ^5.6.3 version: 5.6.3 devDependencies: '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 nodemon: specifier: ^3.0.1 version: 3.1.0 @@ -1332,14 +1332,14 @@ importers: version: 6.1.5 tsx: specifier: ^4.7.1 - version: 4.7.3 + version: 4.23.12 typescript: specifier: ^5.6.3 version: 5.6.3 devDependencies: '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 '@types/uuid': specifier: ~9.0.6 version: 9.0.8 @@ -1402,7 +1402,7 @@ importers: version: 1.17.2 axios: specifier: ^1.16.1 - version: 1.16.1 + version: 1.19.0 fast-xml-parser: specifier: ^5.8.0 version: 5.8.0 @@ -1423,7 +1423,7 @@ importers: version: 7.5.21 tsx: specifier: ^4.7.1 - version: 4.7.3 + version: 4.23.12 typescript: specifier: ^5.6.3 version: 5.6.3 @@ -1442,7 +1442,7 @@ importers: version: 9.0.6 '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 '@types/semver': specifier: ^7.5.8 version: 7.5.8 @@ -1454,7 +1454,7 @@ importers: version: 3.1.0 vitest: specifier: ^3.2.4 - version: 3.2.4(@types/debug@4.1.12)(@types/node@20.12.7)(jiti@2.4.2)(terser@5.43.1)(tsx@4.7.3)(yaml@2.7.0) + version: 3.2.4(@types/debug@4.1.12)(@types/node@20.19.43)(jiti@2.4.2)(terser@5.43.1)(tsx@4.23.12)(yaml@2.7.0) services/apps/pcc_sync_worker: dependencies: @@ -1493,7 +1493,7 @@ importers: version: 1.17.2 tsx: specifier: ^4.7.1 - version: 4.7.3 + version: 4.23.12 typescript: specifier: ^5.6.3 version: 5.6.3 @@ -1533,20 +1533,20 @@ importers: version: 1.17.2 axios: specifier: ^1.6.8 - version: 1.8.4 + version: 1.19.0 lodash.mergewith: specifier: ^4.6.2 version: 4.6.2 tsx: specifier: ^4.7.1 - version: 4.7.3 + version: 4.23.12 typescript: specifier: ^5.6.3 version: 5.6.3 devDependencies: '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 '@types/uuid': specifier: ~9.0.6 version: 9.0.8 @@ -1579,14 +1579,14 @@ importers: version: 1.17.2 tsx: specifier: ^4.7.1 - version: 4.7.3 + version: 4.23.12 typescript: specifier: ^5.6.3 version: 5.6.3 devDependencies: '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 nodemon: specifier: ^3.0.1 version: 3.1.0 @@ -1634,7 +1634,7 @@ importers: version: 1.17.2 axios: specifier: ^1.6.8 - version: 1.6.8 + version: 1.19.0 csv-parse: specifier: ^5.5.6 version: 5.5.6 @@ -1643,14 +1643,14 @@ importers: version: 2.29.4 tsx: specifier: ^4.7.1 - version: 4.7.3 + version: 4.23.12 typescript: specifier: ^5.6.3 version: 5.6.3 devDependencies: '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 nodemon: specifier: ^3.0.1 version: 3.1.0 @@ -1680,7 +1680,7 @@ importers: version: 1.0.2 config: specifier: ^3.3.9 - version: 3.3.11 + version: 3.3.12 cors: specifier: ^2.8.5 version: 2.8.5 @@ -1689,7 +1689,7 @@ importers: version: 4.18.3 tsx: specifier: ^4.7.1 - version: 4.7.3 + version: 4.23.12 typescript: specifier: ^5.6.3 version: 5.6.3 @@ -1702,7 +1702,7 @@ importers: version: 4.17.21 '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 nodemon: specifier: ^2.0.22 version: 2.0.22 @@ -1732,10 +1732,10 @@ importers: version: link:../../libs/types config: specifier: ^3.3.9 - version: 3.3.11 + version: 3.3.12 tsx: specifier: ^4.7.1 - version: 4.7.3 + version: 4.23.12 typescript: specifier: ^5.6.3 version: 5.6.3 @@ -1745,7 +1745,7 @@ importers: version: 3.3.4 '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 nodemon: specifier: ^2.0.22 version: 2.0.22 @@ -1784,7 +1784,7 @@ importers: version: 1.17.2 axios: specifier: ^1.6.8 - version: 1.8.1 + version: 1.19.0 js-yaml: specifier: ^4.1.1 version: 4.1.1 @@ -1793,14 +1793,14 @@ importers: version: 2.29.4 tsx: specifier: ^4.7.1 - version: 4.7.3 + version: 4.23.12 typescript: specifier: ^5.6.3 version: 5.6.3 devDependencies: '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 nodemon: specifier: ^3.0.1 version: 3.1.0 @@ -1854,7 +1854,7 @@ importers: version: 1.17.2 tsx: specifier: ^4.7.1 - version: 4.7.3 + version: 4.23.12 typescript: specifier: ^5.6.3 version: 5.6.3 @@ -1873,14 +1873,14 @@ importers: version: link:../../archetypes/standard tsx: specifier: ^4.7.1 - version: 4.7.3 + version: 4.23.12 typescript: specifier: ^5.6.3 version: 5.6.3 devDependencies: '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 nodemon: specifier: ^3.0.1 version: 3.1.0 @@ -1898,14 +1898,14 @@ importers: version: 1.17.2 tsx: specifier: ^4.7.1 - version: 4.7.3 + version: 4.23.12 typescript: specifier: ^5.6.3 version: 5.6.3 devDependencies: '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 nodemon: specifier: ^3.0.1 version: 3.1.0 @@ -1929,14 +1929,14 @@ importers: version: 4.18.3 tsx: specifier: ^4.7.1 - version: 4.7.3 + version: 4.23.12 typescript: specifier: ^5.6.3 version: 5.6.3 devDependencies: '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 nodemon: specifier: ^3.0.1 version: 3.1.0 @@ -1975,13 +1975,13 @@ importers: version: 4.17.21 '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 bunyan-middleware: specifier: ^1.0.2 version: 1.0.2 config: specifier: ^3.3.9 - version: 3.3.11 + version: 3.3.12 cors: specifier: ^2.8.5 version: 2.8.5 @@ -1990,7 +1990,7 @@ importers: version: 4.18.3 tsx: specifier: ^4.7.1 - version: 4.7.3 + version: 4.23.12 typescript: specifier: ^5.6.3 version: 5.6.3 @@ -2016,7 +2016,7 @@ importers: version: 3.3.4 '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 typescript: specifier: ^5.6.3 version: 5.6.3 @@ -2059,7 +2059,7 @@ importers: version: 3.3.4 '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 typescript: specifier: ^5.6.3 version: 5.6.3 @@ -2108,7 +2108,7 @@ importers: version: 3.3.4 '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 typescript: specifier: ^5.6.3 version: 5.6.3 @@ -2124,7 +2124,7 @@ importers: devDependencies: '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 typescript: specifier: ^5.6.3 version: 5.6.3 @@ -2140,7 +2140,7 @@ importers: devDependencies: '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 typescript: specifier: ^5.6.3 version: 5.6.3 @@ -2152,7 +2152,7 @@ importers: version: link:../types config: specifier: ^3.3.9 - version: 3.3.11 + version: 3.3.12 i18n-iso-countries: specifier: ^7.14.0 version: 7.14.0 @@ -2207,7 +2207,7 @@ importers: version: 3.3.4 '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 typescript: specifier: ^5.6.3 version: 5.6.3 @@ -2255,7 +2255,7 @@ importers: version: 22.0.0 axios: specifier: ^1.13.1 - version: 1.13.1 + version: 1.19.0 jsonwebtoken: specifier: ^9.0.0 version: 9.0.3 @@ -2274,7 +2274,7 @@ importers: devDependencies: '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 typescript: specifier: ^5.6.3 version: 5.6.3 @@ -2301,14 +2301,14 @@ importers: version: link:../types axios: specifier: ^1.6.8 - version: 1.16.1 + version: 1.19.0 zod: specifier: ^3.22.0 version: 3.25.76 devDependencies: '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 typescript: specifier: ^5.6.3 version: 5.6.3 @@ -2387,7 +2387,7 @@ importers: version: link:../test-kit '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 typescript: specifier: ^5.6.3 version: 5.6.3 @@ -2402,14 +2402,14 @@ importers: version: link:../logging axios: specifier: ^1.11.0 - version: 1.11.0 + version: 1.19.0 pg-promise: specifier: ^11.4.3 version: 11.6.0 devDependencies: '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 typescript: specifier: ^5.6.3 version: 5.6.3 @@ -2445,7 +2445,7 @@ importers: version: 5.0.6(encoding@0.1.13) axios: specifier: ^1.4.0 - version: 1.6.8 + version: 1.19.0 he: specifier: ^1.2.0 version: 1.2.0 @@ -2467,7 +2467,7 @@ importers: version: 1.2.3 '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 '@types/sanitize-html': specifier: ^2.9.0 version: 2.11.0 @@ -2495,7 +2495,7 @@ importers: version: 0.2.9 '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 typescript: specifier: ^5.6.3 version: 5.6.3 @@ -2522,11 +2522,11 @@ importers: version: 0.69.22 axios: specifier: ^1.8.4 - version: 1.8.4 + version: 1.19.0 devDependencies: '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 typescript: specifier: ^5.6.3 version: 5.6.3 @@ -2562,14 +2562,14 @@ importers: version: 2.11.0 axios: specifier: ^1.6.0 - version: 1.6.8 + version: 1.19.0 lodash.merge: specifier: ^4.6.2 version: 4.6.2 devDependencies: '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 typescript: specifier: ^5.6.3 version: 5.6.3 @@ -2594,7 +2594,7 @@ importers: devDependencies: '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 typescript: specifier: ^5.6.3 version: 5.6.3 @@ -2616,7 +2616,7 @@ importers: devDependencies: '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 typescript: specifier: ^5.6.3 version: 5.6.3 @@ -2635,7 +2635,7 @@ importers: devDependencies: '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 typescript: specifier: ^5.6.3 version: 5.6.3 @@ -2656,14 +2656,14 @@ importers: version: 1.8.7(bufferutil@4.0.8)(utf-8-validate@5.0.10) snowflake-sdk: specifier: ^2.3.3 - version: 2.3.4(asn1.js@5.4.1) + version: 2.4.0(asn1.js@5.4.1) devDependencies: '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 tsx: specifier: ^4.7.1 - version: 4.7.3 + version: 4.23.12 typescript: specifier: ^5.6.3 version: 5.6.3 @@ -2675,11 +2675,11 @@ importers: version: link:../logging dd-trace: specifier: ^4.38.0 - version: 4.38.0 + version: 4.55.0 devDependencies: '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 typescript: specifier: ^5.6.3 version: 5.6.3 @@ -2704,7 +2704,7 @@ importers: devDependencies: '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 typescript: specifier: ^5.6.3 version: 5.6.3 @@ -2735,19 +2735,19 @@ importers: devDependencies: '@types/node': specifier: ^20.0.0 - version: 20.12.7 + version: 20.19.43 typescript: specifier: ^5.6.3 version: 5.6.3 vitest: specifier: 4.1.7 - version: 4.1.7(@types/node@20.12.7)(vite@6.3.5(@types/node@20.12.7)(jiti@2.4.2)(terser@5.43.1)(yaml@2.7.0)) + version: 4.1.7(@types/node@20.19.43)(vite@6.3.5(@types/node@20.19.43)(jiti@2.4.2)(terser@5.43.1)(tsx@4.23.12)(yaml@2.7.0)) services/libs/types: devDependencies: '@types/node': specifier: ^20.8.2 - version: 20.12.7 + version: 20.19.43 typescript: specifier: ^5.6.3 version: 5.6.3 @@ -2879,6 +2879,10 @@ packages: resolution: {integrity: sha512-P5d7hSgM2PMkYU4yomh6PE23eRj2sXQjMWh/gSSq2hyroJHPXFvx17C8zNfTTTrJ4xvc3fdYm6KS2iwXimwQZQ==} engines: {node: '>=14.0.0'} + '@aws-sdk/client-s3@3.1033.0': + resolution: {integrity: sha512-c8iDFppzyhQUTTPsUWDy43mSKzQsTIi+RkY9u9fHPDiu1bUJWO/2xhuFx9j6l0+29HKqlQx8yJGe8lRF3xSw3w==} + engines: {node: '>=20.0.0'} + '@aws-sdk/client-s3@3.985.0': resolution: {integrity: sha512-S9TqjzzZEEIKBnC7yFpvqM7CG9ALpY5qhQ5BnDBJtdG20NoGpjKLGUUfD2wmZItuhbrcM4Z8c6m6Fg0XYIOVvw==} engines: {node: '>=20.0.0'} @@ -2905,6 +2909,10 @@ packages: resolution: {integrity: sha512-81J8iE8MuXhdbMfIz4sWFj64Pe41bFi/uqqmqOC5SlGv+kwoyLsyKS/rH2tW2t5buih4vTUxskRjxlqikTD4oQ==} engines: {node: '>=20.0.0'} + '@aws-sdk/client-sts@3.1033.0': + resolution: {integrity: sha512-adkXryXCIgrfktuN0ZYkfsVFy9eo4OvlULL7euWPYipRKl/31MH/t6nq/uU99Kuz5PZSmfRqmiA4f9CMfaZZ2g==} + engines: {node: '>=20.0.0'} + '@aws-sdk/client-sts@3.556.0': resolution: {integrity: sha512-TsK3js7Suh9xEmC886aY+bv0KdLLYtzrcmVt6sJ/W6EnDXYQhBuKYFhp03NrN2+vSvMGpqJwR62DyfKe1G0QzQ==} engines: {node: '>=14.0.0'} @@ -2915,10 +2923,6 @@ packages: resolution: {integrity: sha512-jCQuH2qkbWoSY4wckLSfzf3OPh7zc7ZckEbIGGVUQar/JVff6EIbpQ+uNG29DDEOpdPPd8rrJsVuUlA/nvJdXA==} engines: {node: '>=16.0.0'} - '@aws-sdk/client-sts@3.985.0': - resolution: {integrity: sha512-YKj2f0FZAXQ/s1460PoVJOGijDwMycknSbDO7EZQnR0TZh5I12AUZzAiNEOXltKJ0TSHex2YDuNwGmlzvXoeuw==} - engines: {node: '>=20.0.0'} - '@aws-sdk/core@3.556.0': resolution: {integrity: sha512-vJaSaHw2kPQlo11j/Rzuz0gk1tEaKdz+2ser0f0qZ5vwFlANjt08m/frU17ctnVKC1s58bxpctO/1P894fHLrA==} engines: {node: '>=14.0.0'} @@ -2931,10 +2935,18 @@ packages: resolution: {integrity: sha512-wNZZQQNlJ+hzD49cKdo+PY6rsTDElO8yDImnrI69p2PLBa7QomeUKAJWYp9xnaR38nlHqWhMHZuYLCQ3oSX+xg==} engines: {node: '>=20.0.0'} + '@aws-sdk/core@3.974.2': + resolution: {integrity: sha512-oav5AOAz+1XkwUfp6SrEm42UPDpUP5D4jNYXkDwFR1VfWqYX62+jpytdfzURmJ9McSoJIQwi0OJlC4oCi6t0VQ==} + engines: {node: '>=20.0.0'} + '@aws-sdk/crc64-nvme@3.972.0': resolution: {integrity: sha512-ThlLhTqX68jvoIVv+pryOdb5coP1cX1/MaTbB9xkGDCbWbsqQcLqzPxuSoW1DCnAAIacmXCWpzUNOB9pv+xXQw==} engines: {node: '>=20.0.0'} + '@aws-sdk/crc64-nvme@3.972.7': + resolution: {integrity: sha512-QUagVVBbC8gODCF6e1aV0mE2TXWB9Opz4k8EJFdNrujUVQm5R4AjJa1mpOqzwOuROBzqJU9zawzig7M96L8Ejg==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-env@3.535.0': resolution: {integrity: sha512-XppwO8c0GCGSAvdzyJOhbtktSEaShg14VJKg8mpMa1XcgqzmcqqHQjtDWbx5rZheY1VdpXZhpEzJkB6LpQejpA==} engines: {node: '>=14.0.0'} @@ -2943,6 +2955,10 @@ packages: resolution: {integrity: sha512-MVTQoZwPnP1Ev5A7LG+KzeU6sCB8BcGkZeDT1z1V5Wt7GPq0MgFQTSSjhImnB9jqRSZkl1079Bt3PbO6lfIS8g==} engines: {node: '>=16.0.0'} + '@aws-sdk/credential-provider-env@3.972.28': + resolution: {integrity: sha512-87GdRJ2OR0qR4VkMjXN/SZi66DZsunW2qQCbtw9rKw3Y7JurFi6tQWYKOSLY/gOADrU6OxGqFmdw3hKzZqDZOQ==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-env@3.972.5': resolution: {integrity: sha512-LxJ9PEO4gKPXzkufvIESUysykPIdrV7+Ocb9yAhbhJLE4TiAYqbCVUE+VuKP1leGR1bBfjWjYgSV5MxprlX3mQ==} engines: {node: '>=20.0.0'} @@ -2955,6 +2971,10 @@ packages: resolution: {integrity: sha512-gL0NlyI2eW17hnCrh45hZV+qjtBquB+Bckiip9R6DIVRKqYcoILyiFhuOgf2bXeF23gVh6j18pvUvIoTaFWs5w==} engines: {node: '>=16.0.0'} + '@aws-sdk/credential-provider-http@3.972.30': + resolution: {integrity: sha512-6quozmW2PKwBJTUQLb+lk1q8w5Pm45qaqhx4Tld9EIqYYQOVGj+MT0a8NRVS7QgWJj7rzGlB7rQu3KYBFHemJw==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-http@3.972.7': resolution: {integrity: sha512-L2uOGtvp2x3bTcxFTpSM+GkwFIPd8pHfGWO1764icMbo7e5xJh0nfhx1UwkXLnwvocTNEf8A7jISZLYjUSNaTg==} engines: {node: '>=20.0.0'} @@ -2969,10 +2989,18 @@ packages: peerDependencies: '@aws-sdk/client-sts': 3.572.0 + '@aws-sdk/credential-provider-ini@3.972.32': + resolution: {integrity: sha512-Nkr+UKtczZlocUjc6g96WzQadZSIZO/HVXPki4qbfaVOZYSbfLQKWKfADtJ0kGYsCvSYOZrO66tSc9dkboUt/w==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-ini@3.972.5': resolution: {integrity: sha512-SdDTYE6jkARzOeL7+kudMIM4DaFnP5dZVeatzw849k4bSXDdErDS188bgeNzc/RA2WGrlEpsqHUKP6G7sVXhZg==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-login@3.972.32': + resolution: {integrity: sha512-UxgwT1HmZz1QPXuBy5ZUPJNFXOSlhwdQL61eGhWRthF0xRrT02BCOVJ1p5Ejg5AXfnESTWoKPJ7v/sCkNUtB9g==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-login@3.972.5': resolution: {integrity: sha512-uYq1ILyTSI6ZDCMY5+vUsRM0SOCVI7kaW4wBrehVVkhAxC6y+e9rvGtnoZqCOWL1gKjTMouvsf4Ilhc5NCg1Aw==} engines: {node: '>=20.0.0'} @@ -2985,6 +3013,10 @@ packages: resolution: {integrity: sha512-anlYZnpmVkfp9Gan+LcEkQvmRf/m0KcbR11th8sBEyI5lxMaHKXhnAtC/hEGT7e3L6rgNOrFYTPuSvllITD/Pg==} engines: {node: '>=16.0.0'} + '@aws-sdk/credential-provider-node@3.972.33': + resolution: {integrity: sha512-6pGQnEdSeRvBViTQh/FwaRKB38a3Th+W2mVxuvqAd2Z1Ayo3e6eJ5QqJoZwEMwR6xoxkl3wz3qAfiB1xRhMC+w==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-node@3.972.6': resolution: {integrity: sha512-DZ3CnAAtSVtVz+G+ogqecaErMLgzph4JH5nYbHoBMgBkwTUV+SUcjsjOJwdBJTHu3Dm6l5LBYekZoU2nDqQk2A==} engines: {node: '>=20.0.0'} @@ -2997,6 +3029,10 @@ packages: resolution: {integrity: sha512-hXcOytf0BadSm/MMy7MV8mmY0+Jv3mkavsHNBx0R82hw5ollD0I3JyOAaCtdUpztF0I72F8K+q8SpJQZ+EwArw==} engines: {node: '>=16.0.0'} + '@aws-sdk/credential-provider-process@3.972.28': + resolution: {integrity: sha512-CRAlD8u6oNBhjnX/3ekVGocarD+lFmEn/qeDzytgIdmwrmwMJGFPqS9lGwEfhOTihZKrQ0xSp3z6paX+iXJJhA==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-process@3.972.5': resolution: {integrity: sha512-HDKF3mVbLnuqGg6dMnzBf1VUOywE12/N286msI9YaK9mEIzdsGCtLTvrDhe3Up0R9/hGFbB+9l21/TwF5L1C6g==} engines: {node: '>=20.0.0'} @@ -3009,6 +3045,10 @@ packages: resolution: {integrity: sha512-iIlnpJiDXFp3XC4hJNSiNurnU24mr3iLB3HoNa9efr944bo6XBl9FQdk3NssIkqzSmgyoB2CEUx/daBHz4XSow==} engines: {node: '>=16.0.0'} + '@aws-sdk/credential-provider-sso@3.972.32': + resolution: {integrity: sha512-whhmQghRYOt9mJxFyVMhX7eB8n0oA25OCvqoR7dzFAZjmioCkf7WVB22Bc6llM5cFpBXFX7s4Jv+xVq32VPGWg==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-sso@3.972.5': resolution: {integrity: sha512-8urj3AoeNeQisjMmMBhFeiY2gxt6/7wQQbEGun0YV/OaOOiXrIudTIEYF8ZfD+NQI6X1FY5AkRsx6O/CaGiybA==} engines: {node: '>=20.0.0'} @@ -3023,12 +3063,16 @@ packages: peerDependencies: '@aws-sdk/client-sts': ^3.568.0 + '@aws-sdk/credential-provider-web-identity@3.972.32': + resolution: {integrity: sha512-Z0Y0LDaqyQDznlmr9gv6n4+eWKKWNgmi9j5L6RENr6wyOCguhO8FRPmqDbVLSw0DPdMqICKnA3PurJiS8bD6Cw==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-web-identity@3.972.5': resolution: {integrity: sha512-OK3cULuJl6c+RcDZfPpaK5o3deTOnKZbxm7pzhFNGA3fI2hF9yDih17fGRazJzGGWaDVlR9ejZrpDef4DJCEsw==} engines: {node: '>=20.0.0'} - '@aws-sdk/ec2-metadata-service@3.985.0': - resolution: {integrity: sha512-oHwfreqIg54tg55VjOZph6s9BeX7OqncRIHa4yQQ3V7KymZ2pPK4BEC/zo8PFkrBRr3NQrBk52nZI8xF/ojxqg==} + '@aws-sdk/ec2-metadata-service@3.1033.0': + resolution: {integrity: sha512-WefveJCo0ofJ0BM9+XSrAEbGwQ5SEm3vosjnZ5S4eEGyUzib6Rx+4qBBuQ3bSN3wowmd+9rmjKGxIlOvdClSYg==} engines: {node: '>=20.0.0'} '@aws-sdk/hash-node@3.374.0': @@ -3036,10 +3080,18 @@ packages: engines: {node: '>=14.0.0'} deprecated: This package has moved to @smithy/hash-node + '@aws-sdk/middleware-bucket-endpoint@3.972.10': + resolution: {integrity: sha512-Vbc2frZH7wXlMNd+ZZSXUEs/l1Sv8Jj4zUnIfwrYF5lwaLdXHZ9xx4U3rjUcaye3HRhFVc+E5DbBxpRAbB16BA==} + engines: {node: '>=20.0.0'} + '@aws-sdk/middleware-bucket-endpoint@3.972.3': resolution: {integrity: sha512-fmbgWYirF67YF1GfD7cg5N6HHQ96EyRNx/rDIrTF277/zTWVuPI2qS/ZHgofwR1NZPe/NWvoppflQY01LrbVLg==} engines: {node: '>=20.0.0'} + '@aws-sdk/middleware-expect-continue@3.972.10': + resolution: {integrity: sha512-2Yn0f1Qiq/DjxYR3wfI3LokXnjOhFM7Ssn4LTdFDIxRMCE6I32MAsVnhPX1cUZsuVA9tiZtwwhlSLAtFGxAZlQ==} + engines: {node: '>=20.0.0'} + '@aws-sdk/middleware-expect-continue@3.972.3': resolution: {integrity: sha512-4msC33RZsXQpUKR5QR4HnvBSNCPLGHmB55oDiROqqgyOc+TOfVu2xgi5goA7ms6MdZLeEh2905UfWMnMMF4mRg==} engines: {node: '>=20.0.0'} @@ -3048,6 +3100,10 @@ packages: resolution: {integrity: sha512-SF/1MYWx67OyCrLA4icIpWUfCkdlOi8Y1KecQ9xYxkL10GMjVdPTGPnYhAg0dw5U43Y9PVUWhAV2ezOaG+0BLg==} engines: {node: '>=20.0.0'} + '@aws-sdk/middleware-flexible-checksums@3.974.10': + resolution: {integrity: sha512-R9oqyD1hR7aF2UQaYBo90/ILNn8Sq7gl/2Y4WkDDvsaqklqPomso++sFbgYgNmN/Kfx6gqvJwcjSkxJHEBK1tQ==} + engines: {node: '>=20.0.0'} + '@aws-sdk/middleware-host-header@3.535.0': resolution: {integrity: sha512-0h6TWjBWtDaYwHMQJI9ulafeS4lLaw1vIxRjbpH0svFRt6Eve+Sy8NlVhECfTU2hNz/fLubvrUxsXoThaLBIew==} engines: {node: '>=14.0.0'} @@ -3056,10 +3112,18 @@ packages: resolution: {integrity: sha512-zQHHj2N3in9duKghH7AuRNrOMLnKhW6lnmb7dznou068DJtDr76w475sHp2TF0XELsOGENbbBsOlN/S5QBFBVQ==} engines: {node: '>=16.0.0'} + '@aws-sdk/middleware-host-header@3.972.10': + resolution: {integrity: sha512-IJSsIMeVQ8MMCPbuh1AbltkFhLBLXn7aejzfX5YKT/VLDHn++Dcz8886tXckE+wQssyPUhaXrJhdakO2VilRhg==} + engines: {node: '>=20.0.0'} + '@aws-sdk/middleware-host-header@3.972.3': resolution: {integrity: sha512-aknPTb2M+G3s+0qLCx4Li/qGZH8IIYjugHMv15JTYMe6mgZO8VBpYgeGYsNMGCqCZOcWzuf900jFBG5bopfzmA==} engines: {node: '>=20.0.0'} + '@aws-sdk/middleware-location-constraint@3.972.10': + resolution: {integrity: sha512-rI3NZvJcEvjoD0+0PI0iUAwlPw2IlSlhyvgBK/3WkKJQE/YiKFedd9dMN2lVacdNxPNhxL/jzQaKQdrGtQagjQ==} + engines: {node: '>=20.0.0'} + '@aws-sdk/middleware-location-constraint@3.972.3': resolution: {integrity: sha512-nIg64CVrsXp67vbK0U1/Is8rik3huS3QkRHn2DRDx4NldrEFMgdkZGI/+cZMKD9k4YOS110Dfu21KZLHrFA/1g==} engines: {node: '>=20.0.0'} @@ -3072,6 +3136,10 @@ packages: resolution: {integrity: sha512-BinH72RG7K3DHHC1/tCulocFv+ZlQ9SrPF9zYT0T1OT95JXuHhB7fH8gEABrc6DAtOdJJh2fgxQjPy5tzPtsrA==} engines: {node: '>=16.0.0'} + '@aws-sdk/middleware-logger@3.972.10': + resolution: {integrity: sha512-OOuGvvz1Dm20SjZo5oEBePFqxt5nf8AwkNDSyUHvD9/bfNASmstcYxFAHUowy4n6Io7mWUZ04JURZwSBvyQanQ==} + engines: {node: '>=20.0.0'} + '@aws-sdk/middleware-logger@3.972.3': resolution: {integrity: sha512-Ftg09xNNRqaz9QNzlfdQWfpqMCJbsQdnZVJP55jfhbKi1+FTWxGuvfPoBhDHIovqWKjqbuiew3HuhxbJ0+OjgA==} engines: {node: '>=20.0.0'} @@ -3084,6 +3152,10 @@ packages: resolution: {integrity: sha512-rFk3QhdT4IL6O/UWHmNdjJiURutBCy+ogGqaNHf/RELxgXH3KmYorLwCe0eFb5hq8f6vr3zl4/iH7YtsUOuo1w==} engines: {node: '>=16.0.0'} + '@aws-sdk/middleware-recursion-detection@3.972.11': + resolution: {integrity: sha512-+zz6f79Kj9V5qFK2P+D8Ehjnw4AhphAlCAsPjUqEcInA9umtSSKMrHbSagEeOIsDNuvVrH98bjRHcyQukTrhaQ==} + engines: {node: '>=20.0.0'} + '@aws-sdk/middleware-recursion-detection@3.972.3': resolution: {integrity: sha512-PY57QhzNuXHnwbJgbWYTrqIDHYSeOlhfYERTAuc16LKZpTZRJUjzBFokp9hF7u1fuGeE3D70ERXzdbMBOqQz7Q==} engines: {node: '>=20.0.0'} @@ -3092,10 +3164,18 @@ packages: resolution: {integrity: sha512-4W/dnxqj1B6/uS/5Z+3UHaqDDGjNPgEVlqf5d3ToOFZ31ZfpANwhcCmyX39JklC4aolCEi9renQ5wHnTCC8K8g==} engines: {node: '>=14.0.0'} + '@aws-sdk/middleware-sdk-s3@3.972.31': + resolution: {integrity: sha512-5hS08Fp0Rm+59uGCmkWhZmveXiA7OUV7Wa+IARejdzf9JTZ1qAVeIOE9JoBpsLPvUgEjmsGNHBuFbtGmYyqiqQ==} + engines: {node: '>=20.0.0'} + '@aws-sdk/middleware-sdk-s3@3.972.7': resolution: {integrity: sha512-VtZ7tMIw18VzjG+I6D6rh2eLkJfTtByiFoCIauGDtTTPBEUMQUiGaJ/zZrPlCY6BsvLLeFKz3+E5mntgiOWmIg==} engines: {node: '>=20.0.0'} + '@aws-sdk/middleware-ssec@3.972.10': + resolution: {integrity: sha512-Gli9A0u8EVVb+5bFDGS/QbSVg28w/wpEidg1ggVcSj65BDTdGR6punsOcVjqdiu1i42WHWo51MCvARPIIz9juw==} + engines: {node: '>=20.0.0'} + '@aws-sdk/middleware-ssec@3.972.3': resolution: {integrity: sha512-dU6kDuULN3o3jEHcjm0c4zWJlY1zWVkjG9NPe9qxYLLpcbdj5kRYBS2DdWYD+1B9f910DezRuws7xDEqKkHQIg==} engines: {node: '>=20.0.0'} @@ -3108,6 +3188,10 @@ packages: resolution: {integrity: sha512-R4bBbLp1ywtF1kJoOX1juDMztKPWeQHNj6XuTvtruFDn1RdfnBlbM3+9rguRfH5s4V+xfl8SSWchnyo2cI00xg==} engines: {node: '>=16.0.0'} + '@aws-sdk/middleware-user-agent@3.972.32': + resolution: {integrity: sha512-HQ0x9DDKqLZOGhDiL2eicYXXkYT5dogE4mw0lAfHCpJ6t7MM0PNIsJl2TZzWKU9SpBzOMXHRa7K6ZLKUJu1y0w==} + engines: {node: '>=20.0.0'} + '@aws-sdk/middleware-user-agent@3.972.7': resolution: {integrity: sha512-HUD+geASjXSCyL/DHPQc/Ua7JhldTcIglVAoCV8kiVm99IaFSlAbTvEnyhZwdE6bdFyTL+uIaWLaCFSRsglZBQ==} engines: {node: '>=20.0.0'} @@ -3116,6 +3200,10 @@ packages: resolution: {integrity: sha512-TsWwKzb/2WHafAY0CE7uXgLj0FmnkBTgfioG9HO+7z/zCPcl1+YU+i7dW4o0y+aFxFgxTMG+ExBQpqT/k2ao8g==} engines: {node: '>=20.0.0'} + '@aws-sdk/nested-clients@3.997.0': + resolution: {integrity: sha512-4bI5GHjUiY5R8N6PtchpG6tW2Dl8I2IcZNg3JwqwxHRXjfvQlPoo4VMknG4qkd5W0t3Y20rQ6C7pSR561YG5JQ==} + engines: {node: '>=20.0.0'} + '@aws-sdk/protocol-http@3.374.0': resolution: {integrity: sha512-9WpRUbINdGroV3HiZZIBoJvL2ndoWk39OfwxWs2otxByppJZNN14bg/lvCx5e8ggHUti7IBk5rb0nqQZ4m05pg==} engines: {node: '>=14.0.0'} @@ -3129,6 +3217,10 @@ packages: resolution: {integrity: sha512-xkZMIxek44F4YW5r9otD1O5Y/kDkgAb6JNJePkP1qPVojrkCmin3OFYAOZgGm+T4DZAQ5rWhpaqTAWmnRumYfw==} engines: {node: '>=16.0.0'} + '@aws-sdk/region-config-resolver@3.972.12': + resolution: {integrity: sha512-QQI43Mxd53nBij0pm8HXC+t4IOC6gnhhZfzxE0OATQyO6QfPV4e+aTIRRuAJKA6Nig/cR8eLwPryqYTX9ZrjAQ==} + engines: {node: '>=20.0.0'} + '@aws-sdk/region-config-resolver@3.972.3': resolution: {integrity: sha512-v4J8qYAWfOMcZ4MJUyatntOicTzEMaU7j3OpkRCGGFSL2NgXQ5VbxauIyORA+pxdKZ0qQG2tCQjQjZDlXEC3Ow==} engines: {node: '>=20.0.0'} @@ -3145,6 +3237,14 @@ packages: resolution: {integrity: sha512-W6hTSOPiSbh4IdTYVxN7xHjpCh0qvfQU1GKGBzGQm0ZEIOaMmWqiDEvFfyGYKmfBvumT8vHKxQRTX0av9omtIg==} engines: {node: '>=20.0.0'} + '@aws-sdk/signature-v4-multi-region@3.996.19': + resolution: {integrity: sha512-7Sy8+GhfwUi06NQNLplxuJuXMKJURDsNQfK8yTW6E9wN2J1B+8S5dWZG7vg3InvPPhaXqkcYTr8pzeE+dLjMbQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/token-providers@3.1033.0': + resolution: {integrity: sha512-/TsXhqjyRAFb0xVgmbFAha3cJfZdWjnyn6ohJ3AB4E3peLgxNcmKfYr45hruHymyJAydiHoXC3N1a8qgl41cog==} + engines: {node: '>=20.0.0'} + '@aws-sdk/token-providers@3.564.0': resolution: {integrity: sha512-Kk5ixcl9HjqwzfBJZGQAtsqwKa7Z8P7Mdug837BG8zCJbhf7wwNsmItzXTiAlpVrDZyT8P1yWIxsLOS1YUtmow==} engines: {node: '>=14.0.0'} @@ -3171,6 +3271,10 @@ packages: resolution: {integrity: sha512-DwHBiMNOB468JiX6+i34c+THsKHErYUdNQ3HexeXZvVn4zouLjgaS4FejiGSi2HyBuzuyHg7SuOPmjSvoU9NRg==} engines: {node: '>=20.0.0'} + '@aws-sdk/types@3.973.8': + resolution: {integrity: sha512-gjlAdtHMbtR9X5iIhVUvbVcy55KnznpC6bkDUWW9z915bi0ckdUr5cjf16Kp6xq0bP5HBD2xzgbL9F9Quv5vUw==} + engines: {node: '>=20.0.0'} + '@aws-sdk/url-parser@3.374.0': resolution: {integrity: sha512-RC3yEj4iqw5vbCmR4IQ3rhmFQilwHtWO1mZ9kRTUxfJCge3TVlrZzj9PRW3hxlYKdu3xZjSvCgX3ip8SFKXtbw==} deprecated: This package has moved to @smithy/url-parser @@ -3183,6 +3287,10 @@ packages: resolution: {integrity: sha512-VkykWbqMjlSgBFDyrY3nOSqupMc6ivXuGmvci6Q3NnLq5kC+mKQe2QBZ4nrWRE/jqOxeFP2uYzLtwncYYcvQDg==} engines: {node: '>=20.0.0'} + '@aws-sdk/util-arn-parser@3.972.3': + resolution: {integrity: sha512-HzSD8PMFrvgi2Kserxuff5VitNq2sgf3w9qxmskKDiDTThWfVteJxuCS9JXiPIPtmCrp+7N9asfIaVhBFORllA==} + engines: {node: '>=20.0.0'} + '@aws-sdk/util-endpoints@3.540.0': resolution: {integrity: sha512-1kMyQFAWx6f8alaI6UT65/5YW/7pDWAKAdNwL6vuJLea03KrZRX3PMoONOSJpAS5m3Ot7HlWZvf3wZDNTLELZw==} engines: {node: '>=14.0.0'} @@ -3195,6 +3303,10 @@ packages: resolution: {integrity: sha512-vth7UfGSUR3ljvaq8V4Rc62FsM7GUTH/myxPWkaEgOrprz1/Pc72EgTXxj+cPPPDAfHFIpjhkB7T7Td0RJx+BA==} engines: {node: '>=20.0.0'} + '@aws-sdk/util-endpoints@3.996.7': + resolution: {integrity: sha512-ty4LQxN1QC+YhUP28NfEgZDEGXkyqOQy+BDriBozqHsrYO4JMgiPhfizqOGF7P+euBTZ5Ez6SKlLAMCLo8tzmw==} + engines: {node: '>=20.0.0'} + '@aws-sdk/util-format-url@3.535.0': resolution: {integrity: sha512-ElbNkm0bddu53CuW44Iuux1ZbTV50fydbSh/4ypW3LrmUvHx193ogj0HXQ7X26kmmo9rXcsrLdM92yIeTjidVg==} engines: {node: '>=14.0.0'} @@ -3209,6 +3321,9 @@ packages: '@aws-sdk/util-user-agent-browser@3.567.0': resolution: {integrity: sha512-cqP0uXtZ7m7hRysf3fRyJwcY1jCgQTpJy7BHB5VpsE7DXlXHD5+Ur5L42CY7UrRPrB6lc6YGFqaAOs5ghMcLyA==} + '@aws-sdk/util-user-agent-browser@3.972.10': + resolution: {integrity: sha512-FAzqXvfEssGdSIz8ejatan0bOdx1qefBWKF/gWmVBXIP1HkS7v/wjjaqrAGGKvyihrXTXW00/2/1nTJtxpXz7g==} + '@aws-sdk/util-user-agent-browser@3.972.3': resolution: {integrity: sha512-JurOwkRUcXD/5MTDBcqdyQ9eVedtAsZgw5rBwktsPTN7QtPiS2Ld1jkJepNgYoCufz1Wcut9iup7GJDoIHp8Fw==} @@ -3239,9 +3354,22 @@ packages: aws-crt: optional: true + '@aws-sdk/util-user-agent-node@3.973.18': + resolution: {integrity: sha512-Nh4YvAL0Mzv5jBvzXLFL0tLf7WPrRMnYZQ5jlFuyS0xiVJQsObMUKAkbYjmt/e04wpQqUaa+Is7k+mBr89A9yA==} + engines: {node: '>=20.0.0'} + peerDependencies: + aws-crt: '>=1.0.0' + peerDependenciesMeta: + aws-crt: + optional: true + '@aws-sdk/util-utf8-browser@3.259.0': resolution: {integrity: sha512-UvFa/vR+e19XookZF8RzFZBrw2EUkQWxiBW0yYQAhvk3C+QVGl0H3ouca8LDBlBfQKXwmW3huo/59H8rwb1wJw==} + '@aws-sdk/xml-builder@3.972.18': + resolution: {integrity: sha512-BMDNVG1ETXRhl1tnisQiYBef3RShJ1kfZA7x7afivTFMLirfHNTb6U71K569HNXhSXbQZsweHvSDZ6euBw8hPA==} + engines: {node: '>=20.0.0'} + '@aws-sdk/xml-builder@3.972.4': resolution: {integrity: sha512-0zJ05ANfYqI6+rGqj8samZBFod0dPPousBjLEqg8WdxSgbMAkRgLyn81lP215Do0rFJ/17LIXwr7q0yK24mP6Q==} engines: {node: '>=20.0.0'} @@ -3258,10 +3386,6 @@ packages: resolution: {integrity: sha512-ykRMW8PjVAn+RS6ww5cmK9U2CyH9p4Q88YJwvUslfuMmN98w/2rdGRLPqJYObapBCdzBVeDgYWdJnFPFb7qzpg==} engines: {node: '>=20.0.0'} - '@azure/core-auth@1.9.0': - resolution: {integrity: sha512-FPwHpZywuyasDSLMqJ6fhbOK3TqUdviZNF8OqRGA4W5Ewib2lEEZ+pBsYcBa88B2NGO/SEnYPGhyBqNlE8ilSw==} - engines: {node: '>=18.0.0'} - '@azure/core-client@1.10.1': resolution: {integrity: sha512-Nh5PhEOeY6PrnxNPsEHRr9eimxLwgLlpmguQaHKBinFYA/RU9+kOYVOQqOrTsCL+KSxrLLl1gD8Dk5BFW/7l/w==} engines: {node: '>=20.0.0'} @@ -3289,10 +3413,6 @@ packages: resolution: {integrity: sha512-9MWKevR7Hz8kNzzPLfX4EAtGM2b8mr50HPDBvio96bURP/9C+HjdH3sBlLSNNrvRAr5/k/svoH457gB5IKpmwQ==} engines: {node: '>=20.0.0'} - '@azure/core-util@1.11.0': - resolution: {integrity: sha512-DxOSLua+NdpWoSqULhjDyAZTXFdP/LKkqtYuxxz1SCN289zk3OG8UOpnCQAz/tygyACBtWp/BoO72ptK7msY8g==} - engines: {node: '>=18.0.0'} - '@azure/core-util@1.13.1': resolution: {integrity: sha512-XPArKLzsvl0Hf0CaGyKHUyVgF7oDnhKoP85Xv6M4StF/1AhfORhZudHtOyf2s+FcbuQ9dPRAjB8J2KvRRMUK2A==} engines: {node: '>=20.0.0'} @@ -3305,10 +3425,6 @@ packages: resolution: {integrity: sha512-uWC0fssc+hs1TGGVkkghiaFkkS7NkTxfnCH+Hdg+yTehTpMcehpok4PgUKKdyCH+9ldu6FhiHRv84Ntqj1vVcw==} engines: {node: '>=20.0.0'} - '@azure/logger@1.1.4': - resolution: {integrity: sha512-4IXXzcCdLdlXuCG+8UKEwLA1T1NHqUfanhXYHiQTn+6sfWCZXduqbtXDGceg3Ce5QxTGo7EqmbV6Bi+aqKuClQ==} - engines: {node: '>=18.0.0'} - '@azure/logger@1.3.0': resolution: {integrity: sha512-fCqPIfOcLE+CGqGPd66c8bZpwAji98tZ4JI9i/mlTNTlsIWslCfpg48s/ypyLxZTump5sypjrKn2/kY7q8oAbA==} engines: {node: '>=20.0.0'} @@ -3482,25 +3598,28 @@ packages: '@dabh/diagnostics@2.0.3': resolution: {integrity: sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==} - '@datadog/native-appsec@7.1.1': - resolution: {integrity: sha512-1XVrCY4g1ArN79SQANMtiIkaxKSPfgdAGv0VAM4Pz+NQuxKfl+2xQPXjQPm87LI1KQIO6MU6qzv3sUUSesb9lA==} - engines: {node: '>=14'} + '@datadog/libdatadog@0.3.0': + resolution: {integrity: sha512-TbP8+WyXfh285T17FnLeLUOPl4SbkRYMqKgcmknID2mXHNrbt5XJgW9bnDgsrrtu31Q7FjWWw2WolgRLWyzLRA==} + + '@datadog/native-appsec@8.4.0': + resolution: {integrity: sha512-LC47AnpVLpQFEUOP/nIIs+i0wLb8XYO+et3ACaJlHa2YJM3asR4KZTqQjDQNy08PTAUbVvYWKwfSR1qVsU/BeA==} + engines: {node: '>=16'} - '@datadog/native-iast-rewriter@2.3.1': - resolution: {integrity: sha512-3pmt5G1Ai/+MPyxP7wBerIu/zH/BnAHxEu/EAMr+77IMpK5m7THPDUoWrPRCWcgFBfn0pK5DR7gRItG0wX3e0g==} + '@datadog/native-iast-rewriter@2.6.1': + resolution: {integrity: sha512-zv7cr/MzHg560jhAnHcO7f9pLi4qaYrBEcB+Gla0xkVouYSDsp8cGXIGG4fiGdAMHdt7SpDNS6+NcEAqD/v8Ig==} engines: {node: '>= 10'} deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. - '@datadog/native-iast-taint-tracking@2.1.0': - resolution: {integrity: sha512-DjZ6itJcjLrTdKk2vP96hak2xS0ABd0NIB8poZG3OBQU5efkzu8JOQoxbIKMklG/0P2zh7EquvGP88PdVXT9aA==} + '@datadog/native-iast-taint-tracking@3.2.0': + resolution: {integrity: sha512-Mc6FzCoyvU5yXLMsMS9yKnEqJMWoImAukJXolNWCTm+JQYCMf2yMsJ8pBAm7KyZKliamM9rCn7h7Tr2H3lXwjA==} - '@datadog/native-metrics@2.0.0': - resolution: {integrity: sha512-YklGVwUtmKGYqFf1MNZuOHvTYdKuR4+Af1XkWcMD8BwOAjxmd9Z+97328rCOY8TFUJzlGUPaXzB8j2qgG/BMwA==} - engines: {node: '>=12'} + '@datadog/native-metrics@3.1.1': + resolution: {integrity: sha512-MU1gHrolwryrU4X9g+fylA1KPH3S46oqJPEtVyrO+3Kh29z80fegmtyrU22bNt8LigPUK/EdPCnSbMe88QbnxQ==} + engines: {node: '>=16'} - '@datadog/pprof@5.3.0': - resolution: {integrity: sha512-53z2Q3K92T6Pf4vz4Ezh8kfkVEvLzbnVqacZGgcbkP//q0joFzO8q00Etw1S6NdnCX0XmX08ULaF4rUI5r14mw==} - engines: {node: '>=14'} + '@datadog/pprof@5.4.1': + resolution: {integrity: sha512-IvpL96e/cuh8ugP5O8Czdup7XQOLHeIDgM5pac5W7Lc1YzGe5zTtebKFpitvb1CPw1YY+1qFx0pWGgKP2kOfHg==} + engines: {node: '>=16'} '@datadog/sketches-js@2.1.1': resolution: {integrity: sha512-d5RjycE+MObE/hU+8OM5Zp4VjTwiPLRa8299fj7muOmR16fb942z8byoMbCErnGh0lBevvgkGrLclQDvINbIyg==} @@ -3509,23 +3628,17 @@ packages: resolution: {integrity: sha512-3DkkN3FKJ++BQy8PE3LONoVSVBONG7hld9D+VqBp1Dcfw2PGSnWyg20NET5cJkiygWK2y2H1ra8GWk3yixzuuA==} engines: {node: '>=18.18.2'} - '@esbuild/aix-ppc64@0.19.12': - resolution: {integrity: sha512-bmoCYyWdEL3wDQIVbcyzRyeKLgk2WtWLTWz1ZIAZF/EGbNOwSA6ew3PftJ1PqMiOOGu0OyFMzG53L0zqIpPeNA==} - engines: {node: '>=12'} - cpu: [ppc64] - os: [aix] - '@esbuild/aix-ppc64@0.25.12': resolution: {integrity: sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==} engines: {node: '>=18'} cpu: [ppc64] os: [aix] - '@esbuild/android-arm64@0.19.12': - resolution: {integrity: sha512-P0UVNGIienjZv3f5zq0DP3Nt2IE/3plFzuaS96vihvD0Hd6H/q4WXUGpCxD/E8YrSXfNyRPbpTq+T8ZQioSuPA==} - engines: {node: '>=12'} - cpu: [arm64] - os: [android] + '@esbuild/aix-ppc64@0.28.0': + resolution: {integrity: sha512-lhRUCeuOyJQURhTxl4WkpFTjIsbDayJHih5kZC1giwE+MhIzAb7mEsQMqMf18rHLsrb5qI1tafG20mLxEWcWlA==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] '@esbuild/android-arm64@0.25.12': resolution: {integrity: sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==} @@ -3533,10 +3646,10 @@ packages: cpu: [arm64] os: [android] - '@esbuild/android-arm@0.19.12': - resolution: {integrity: sha512-qg/Lj1mu3CdQlDEEiWrlC4eaPZ1KztwGJ9B6J+/6G+/4ewxJg7gqj8eVYWvao1bXrqGiW2rsBZFSX3q2lcW05w==} - engines: {node: '>=12'} - cpu: [arm] + '@esbuild/android-arm64@0.28.0': + resolution: {integrity: sha512-+WzIXQOSaGs33tLEgYPYe/yQHf0WTU0X42Jca3y8NWMbUVhp7rUnw+vAsRC/QiDrdD31IszMrZy+qwPOPjd+rw==} + engines: {node: '>=18'} + cpu: [arm64] os: [android] '@esbuild/android-arm@0.25.12': @@ -3545,10 +3658,10 @@ packages: cpu: [arm] os: [android] - '@esbuild/android-x64@0.19.12': - resolution: {integrity: sha512-3k7ZoUW6Q6YqhdhIaq/WZ7HwBpnFBlW905Fa4s4qWJyiNOgT1dOqDiVAQFwBH7gBRZr17gLrlFCRzF6jFh7Kew==} - engines: {node: '>=12'} - cpu: [x64] + '@esbuild/android-arm@0.28.0': + resolution: {integrity: sha512-wqh0ByljabXLKHeWXYLqoJ5jKC4XBaw6Hk08OfMrCRd2nP2ZQ5eleDZC41XHyCNgktBGYMbqnrJKq/K/lzPMSQ==} + engines: {node: '>=18'} + cpu: [arm] os: [android] '@esbuild/android-x64@0.25.12': @@ -3557,11 +3670,11 @@ packages: cpu: [x64] os: [android] - '@esbuild/darwin-arm64@0.19.12': - resolution: {integrity: sha512-B6IeSgZgtEzGC42jsI+YYu9Z3HKRxp8ZT3cqhvliEHovq8HSX2YX8lNocDn79gCKJXOSaEot9MVYky7AKjCs8g==} - engines: {node: '>=12'} - cpu: [arm64] - os: [darwin] + '@esbuild/android-x64@0.28.0': + resolution: {integrity: sha512-+VJggoaKhk2VNNqVL7f6S189UzShHC/mR9EE8rDdSkdpN0KflSwWY/gWjDrNxxisg8Fp1ZCD9jLMo4m0OUfeUA==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] '@esbuild/darwin-arm64@0.25.12': resolution: {integrity: sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==} @@ -3569,10 +3682,10 @@ packages: cpu: [arm64] os: [darwin] - '@esbuild/darwin-x64@0.19.12': - resolution: {integrity: sha512-hKoVkKzFiToTgn+41qGhsUJXFlIjxI/jSYeZf3ugemDYZldIXIxhvwN6erJGlX4t5h417iFuheZ7l+YVn05N3A==} - engines: {node: '>=12'} - cpu: [x64] + '@esbuild/darwin-arm64@0.28.0': + resolution: {integrity: sha512-0T+A9WZm+bZ84nZBtk1ckYsOvyA3x7e2Acj1KdVfV4/2tdG4fzUp91YHx+GArWLtwqp77pBXVCPn2We7Letr0Q==} + engines: {node: '>=18'} + cpu: [arm64] os: [darwin] '@esbuild/darwin-x64@0.25.12': @@ -3581,11 +3694,11 @@ packages: cpu: [x64] os: [darwin] - '@esbuild/freebsd-arm64@0.19.12': - resolution: {integrity: sha512-4aRvFIXmwAcDBw9AueDQ2YnGmz5L6obe5kmPT8Vd+/+x/JMVKCgdcRwH6APrbpNXsPz+K653Qg8HB/oXvXVukA==} - engines: {node: '>=12'} - cpu: [arm64] - os: [freebsd] + '@esbuild/darwin-x64@0.28.0': + resolution: {integrity: sha512-fyzLm/DLDl/84OCfp2f/XQ4flmORsjU7VKt8HLjvIXChJoFFOIL6pLJPH4Yhd1n1gGFF9mPwtlN5Wf82DZs+LQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] '@esbuild/freebsd-arm64@0.25.12': resolution: {integrity: sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==} @@ -3593,10 +3706,10 @@ packages: cpu: [arm64] os: [freebsd] - '@esbuild/freebsd-x64@0.19.12': - resolution: {integrity: sha512-EYoXZ4d8xtBoVN7CEwWY2IN4ho76xjYXqSXMNccFSx2lgqOG/1TBPW0yPx1bJZk94qu3tX0fycJeeQsKovA8gg==} - engines: {node: '>=12'} - cpu: [x64] + '@esbuild/freebsd-arm64@0.28.0': + resolution: {integrity: sha512-l9GeW5UZBT9k9brBYI+0WDffcRxgHQD8ShN2Ur4xWq/NFzUKm3k5lsH4PdaRgb2w7mI9u61nr2gI2mLI27Nh3Q==} + engines: {node: '>=18'} + cpu: [arm64] os: [freebsd] '@esbuild/freebsd-x64@0.25.12': @@ -3605,11 +3718,11 @@ packages: cpu: [x64] os: [freebsd] - '@esbuild/linux-arm64@0.19.12': - resolution: {integrity: sha512-EoTjyYyLuVPfdPLsGVVVC8a0p1BFFvtpQDB/YLEhaXyf/5bczaGeN15QkR+O4S5LeJ92Tqotve7i1jn35qwvdA==} - engines: {node: '>=12'} - cpu: [arm64] - os: [linux] + '@esbuild/freebsd-x64@0.28.0': + resolution: {integrity: sha512-BXoQai/A0wPO6Es3yFJ7APCiKGc1tdAEOgeTNy3SsB491S3aHn4S4r3e976eUnPdU+NbdtmBuLncYir2tMU9Nw==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] '@esbuild/linux-arm64@0.25.12': resolution: {integrity: sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==} @@ -3617,10 +3730,10 @@ packages: cpu: [arm64] os: [linux] - '@esbuild/linux-arm@0.19.12': - resolution: {integrity: sha512-J5jPms//KhSNv+LO1S1TX1UWp1ucM6N6XuL6ITdKWElCu8wXP72l9MM0zDTzzeikVyqFE6U8YAV9/tFyj0ti+w==} - engines: {node: '>=12'} - cpu: [arm] + '@esbuild/linux-arm64@0.28.0': + resolution: {integrity: sha512-RVyzfb3FWsGA55n6WY0MEIEPURL1FcbhFE6BffZEMEekfCzCIMtB5yyDcFnVbTnwk+CLAgTujmV/Lgvih56W+A==} + engines: {node: '>=18'} + cpu: [arm64] os: [linux] '@esbuild/linux-arm@0.25.12': @@ -3629,10 +3742,10 @@ packages: cpu: [arm] os: [linux] - '@esbuild/linux-ia32@0.19.12': - resolution: {integrity: sha512-Thsa42rrP1+UIGaWz47uydHSBOgTUnwBwNq59khgIwktK6x60Hivfbux9iNR0eHCHzOLjLMLfUMLCypBkZXMHA==} - engines: {node: '>=12'} - cpu: [ia32] + '@esbuild/linux-arm@0.28.0': + resolution: {integrity: sha512-CjaaREJagqJp7iTaNQjjidaNbCKYcd4IDkzbwwxtSvjI7NZm79qiHc8HqciMddQ6CKvJT6aBd8lO9kN/ZudLlw==} + engines: {node: '>=18'} + cpu: [arm] os: [linux] '@esbuild/linux-ia32@0.25.12': @@ -3641,10 +3754,10 @@ packages: cpu: [ia32] os: [linux] - '@esbuild/linux-loong64@0.19.12': - resolution: {integrity: sha512-LiXdXA0s3IqRRjm6rV6XaWATScKAXjI4R4LoDlvO7+yQqFdlr1Bax62sRwkVvRIrwXxvtYEHHI4dm50jAXkuAA==} - engines: {node: '>=12'} - cpu: [loong64] + '@esbuild/linux-ia32@0.28.0': + resolution: {integrity: sha512-KBnSTt1kxl9x70q+ydterVdl+Cn0H18ngRMRCEQfrbqdUuntQQ0LoMZv47uB97NljZFzY6HcfqEZ2SAyIUTQBQ==} + engines: {node: '>=18'} + cpu: [ia32] os: [linux] '@esbuild/linux-loong64@0.25.12': @@ -3653,10 +3766,10 @@ packages: cpu: [loong64] os: [linux] - '@esbuild/linux-mips64el@0.19.12': - resolution: {integrity: sha512-fEnAuj5VGTanfJ07ff0gOA6IPsvrVHLVb6Lyd1g2/ed67oU1eFzL0r9WL7ZzscD+/N6i3dWumGE1Un4f7Amf+w==} - engines: {node: '>=12'} - cpu: [mips64el] + '@esbuild/linux-loong64@0.28.0': + resolution: {integrity: sha512-zpSlUce1mnxzgBADvxKXX5sl8aYQHo2ezvMNI8I0lbblJtp8V4odlm3Yzlj7gPyt3T8ReksE6bK+pT3WD+aJRg==} + engines: {node: '>=18'} + cpu: [loong64] os: [linux] '@esbuild/linux-mips64el@0.25.12': @@ -3665,10 +3778,10 @@ packages: cpu: [mips64el] os: [linux] - '@esbuild/linux-ppc64@0.19.12': - resolution: {integrity: sha512-nYJA2/QPimDQOh1rKWedNOe3Gfc8PabU7HT3iXWtNUbRzXS9+vgB0Fjaqr//XNbd82mCxHzik2qotuI89cfixg==} - engines: {node: '>=12'} - cpu: [ppc64] + '@esbuild/linux-mips64el@0.28.0': + resolution: {integrity: sha512-2jIfP6mmjkdmeTlsX/9vmdmhBmKADrWqN7zcdtHIeNSCH1SqIoNI63cYsjQR8J+wGa4Y5izRcSHSm8K3QWmk3w==} + engines: {node: '>=18'} + cpu: [mips64el] os: [linux] '@esbuild/linux-ppc64@0.25.12': @@ -3677,10 +3790,10 @@ packages: cpu: [ppc64] os: [linux] - '@esbuild/linux-riscv64@0.19.12': - resolution: {integrity: sha512-2MueBrlPQCw5dVJJpQdUYgeqIzDQgw3QtiAHUC4RBz9FXPrskyyU3VI1hw7C0BSKB9OduwSJ79FTCqtGMWqJHg==} - engines: {node: '>=12'} - cpu: [riscv64] + '@esbuild/linux-ppc64@0.28.0': + resolution: {integrity: sha512-bc0FE9wWeC0WBm49IQMPSPILRocGTQt3j5KPCA8os6VprfuJ7KD+5PzESSrJ6GmPIPJK965ZJHTUlSA6GNYEhg==} + engines: {node: '>=18'} + cpu: [ppc64] os: [linux] '@esbuild/linux-riscv64@0.25.12': @@ -3689,10 +3802,10 @@ packages: cpu: [riscv64] os: [linux] - '@esbuild/linux-s390x@0.19.12': - resolution: {integrity: sha512-+Pil1Nv3Umes4m3AZKqA2anfhJiVmNCYkPchwFJNEJN5QxmTs1uzyy4TvmDrCRNT2ApwSari7ZIgrPeUx4UZDg==} - engines: {node: '>=12'} - cpu: [s390x] + '@esbuild/linux-riscv64@0.28.0': + resolution: {integrity: sha512-SQPZOwoTTT/HXFXQJG/vBX8sOFagGqvZyXcgLA3NhIqcBv1BJU1d46c0rGcrij2B56Z2rNiSLaZOYW5cUk7yLQ==} + engines: {node: '>=18'} + cpu: [riscv64] os: [linux] '@esbuild/linux-s390x@0.25.12': @@ -3701,10 +3814,10 @@ packages: cpu: [s390x] os: [linux] - '@esbuild/linux-x64@0.19.12': - resolution: {integrity: sha512-B71g1QpxfwBvNrfyJdVDexenDIt1CiDN1TIXLbhOw0KhJzE78KIFGX6OJ9MrtC0oOqMWf+0xop4qEU8JrJTwCg==} - engines: {node: '>=12'} - cpu: [x64] + '@esbuild/linux-s390x@0.28.0': + resolution: {integrity: sha512-SCfR0HN8CEEjnYnySJTd2cw0k9OHB/YFzt5zgJEwa+wL/T/raGWYMBqwDNAC6dqFKmJYZoQBRfHjgwLHGSrn3Q==} + engines: {node: '>=18'} + cpu: [s390x] os: [linux] '@esbuild/linux-x64@0.25.12': @@ -3713,16 +3826,22 @@ packages: cpu: [x64] os: [linux] + '@esbuild/linux-x64@0.28.0': + resolution: {integrity: sha512-us0dSb9iFxIi8srnpl931Nvs65it/Jd2a2K3qs7fz2WfGPHqzfzZTfec7oxZJRNPXPnNYZtanmRc4AL/JwVzHQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + '@esbuild/netbsd-arm64@0.25.12': resolution: {integrity: sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==} engines: {node: '>=18'} cpu: [arm64] os: [netbsd] - '@esbuild/netbsd-x64@0.19.12': - resolution: {integrity: sha512-3ltjQ7n1owJgFbuC61Oj++XhtzmymoCihNFgT84UAmJnxJfm4sYCiSLTXZtE00VWYpPMYc+ZQmB6xbSdVh0JWA==} - engines: {node: '>=12'} - cpu: [x64] + '@esbuild/netbsd-arm64@0.28.0': + resolution: {integrity: sha512-CR/RYotgtCKwtftMwJlUU7xCVNg3lMYZ0RzTmAHSfLCXw3NtZtNpswLEj/Kkf6kEL3Gw+BpOekRX0BYCtklhUw==} + engines: {node: '>=18'} + cpu: [arm64] os: [netbsd] '@esbuild/netbsd-x64@0.25.12': @@ -3731,16 +3850,22 @@ packages: cpu: [x64] os: [netbsd] + '@esbuild/netbsd-x64@0.28.0': + resolution: {integrity: sha512-nU1yhmYutL+fQ71Kxnhg8uEOdC0pwEW9entHykTgEbna2pw2dkbFSMeqjjyHZoCmt8SBkOSvV+yNmm94aUrrqw==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + '@esbuild/openbsd-arm64@0.25.12': resolution: {integrity: sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==} engines: {node: '>=18'} cpu: [arm64] os: [openbsd] - '@esbuild/openbsd-x64@0.19.12': - resolution: {integrity: sha512-RbrfTB9SWsr0kWmb9srfF+L933uMDdu9BIzdA7os2t0TXhCRjrQyCeOt6wVxr79CKD4c+p+YhCj31HBkYcXebw==} - engines: {node: '>=12'} - cpu: [x64] + '@esbuild/openbsd-arm64@0.28.0': + resolution: {integrity: sha512-cXb5vApOsRsxsEl4mcZ1XY3D4DzcoMxR/nnc4IyqYs0rTI8ZKmW6kyyg+11Z8yvgMfAEldKzP7AdP64HnSC/6g==} + engines: {node: '>=18'} + cpu: [arm64] os: [openbsd] '@esbuild/openbsd-x64@0.25.12': @@ -3749,17 +3874,23 @@ packages: cpu: [x64] os: [openbsd] + '@esbuild/openbsd-x64@0.28.0': + resolution: {integrity: sha512-8wZM2qqtv9UP3mzy7HiGYNH/zjTA355mpeuA+859TyR+e+Tc08IHYpLJuMsfpDJwoLo1ikIJI8jC3GFjnRClzA==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + '@esbuild/openharmony-arm64@0.25.12': resolution: {integrity: sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==} engines: {node: '>=18'} cpu: [arm64] os: [openharmony] - '@esbuild/sunos-x64@0.19.12': - resolution: {integrity: sha512-HKjJwRrW8uWtCQnQOz9qcU3mUZhTUQvi56Q8DPTLLB+DawoiQdjsYq+j+D3s9I8VFtDr+F9CjgXKKC4ss89IeA==} - engines: {node: '>=12'} - cpu: [x64] - os: [sunos] + '@esbuild/openharmony-arm64@0.28.0': + resolution: {integrity: sha512-FLGfyizszcef5C3YtoyQDACyg95+dndv79i2EekILBofh5wpCa1KuBqOWKrEHZg3zrL3t5ouE5jgr94vA+Wb2w==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] '@esbuild/sunos-x64@0.25.12': resolution: {integrity: sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==} @@ -3767,11 +3898,11 @@ packages: cpu: [x64] os: [sunos] - '@esbuild/win32-arm64@0.19.12': - resolution: {integrity: sha512-URgtR1dJnmGvX864pn1B2YUYNzjmXkuJOIqG2HdU62MVS4EHpU2946OZoTMnRUHklGtJdJZ33QfzdjGACXhn1A==} - engines: {node: '>=12'} - cpu: [arm64] - os: [win32] + '@esbuild/sunos-x64@0.28.0': + resolution: {integrity: sha512-1ZgjUoEdHZZl/YlV76TSCz9Hqj9h9YmMGAgAPYd+q4SicWNX3G5GCyx9uhQWSLcbvPW8Ni7lj4gDa1T40akdlw==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] '@esbuild/win32-arm64@0.25.12': resolution: {integrity: sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==} @@ -3779,10 +3910,10 @@ packages: cpu: [arm64] os: [win32] - '@esbuild/win32-ia32@0.19.12': - resolution: {integrity: sha512-+ZOE6pUkMOJfmxmBZElNOx72NKpIa/HFOMGzu8fqzQJ5kgf6aTGrcJaFsNiVMH4JKpMipyK+7k0n2UXN7a8YKQ==} - engines: {node: '>=12'} - cpu: [ia32] + '@esbuild/win32-arm64@0.28.0': + resolution: {integrity: sha512-Q9StnDmQ/enxnpxCCLSg0oo4+34B9TdXpuyPeTedN/6+iXBJ4J+zwfQI28u/Jl40nOYAxGoNi7mFP40RUtkmUA==} + engines: {node: '>=18'} + cpu: [arm64] os: [win32] '@esbuild/win32-ia32@0.25.12': @@ -3791,10 +3922,10 @@ packages: cpu: [ia32] os: [win32] - '@esbuild/win32-x64@0.19.12': - resolution: {integrity: sha512-T1QyPSDCyMXaO3pzBkF96E8xMkiRYbUEZADd29SyPGabqxMViNoii+NcK7eWJAEoU6RZyEm5lVSIjTmcdoB9HA==} - engines: {node: '>=12'} - cpu: [x64] + '@esbuild/win32-ia32@0.28.0': + resolution: {integrity: sha512-zF3ag/gfiCe6U2iczcRzSYJKH1DCI+ByzSENHlM2FcDbEeo5Zd2C86Aq0tKUYAJJ1obRP84ymxIAksZUcdztHA==} + engines: {node: '>=18'} + cpu: [ia32] os: [win32] '@esbuild/win32-x64@0.25.12': @@ -3803,6 +3934,12 @@ packages: cpu: [x64] os: [win32] + '@esbuild/win32-x64@0.28.0': + resolution: {integrity: sha512-pEl1bO9mfAmIC+tW5btTmrKaujg3zGtUmWNdCw/xs70FBjwAL3o9OEKNHvNmnyylD6ubxUERiEhdsL0xBQ9efw==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + '@eslint-community/eslint-utils@4.4.0': resolution: {integrity: sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} @@ -3940,6 +4077,10 @@ packages: resolution: {integrity: sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==} engines: {node: '>=18.0.0'} + '@isaacs/ttlcache@1.4.1': + resolution: {integrity: sha512-RQgQ4uQ+pLbqXfOmieB91ejmLwvSgv9nLx6sT6sD83s7umBypgg+OIBOBbEUiJXrfpnp9j0mRhYYdzp9uqq3lA==} + engines: {node: '>=12'} + '@jridgewell/gen-mapping@0.3.5': resolution: {integrity: sha512-IzL8ZoEDIBRWEzlCcRhOaCupYyN5gdIK+Q6fbFdPDg6HqX6jpkItn7DFIpW9LQzXG6Df9sA7+OKnq0qlz/GaQg==} engines: {node: '>=6.0.0'} @@ -4018,6 +4159,10 @@ packages: '@nangohq/types@0.69.22': resolution: {integrity: sha512-3p7KMZ3GDXrt+wo5BKn/ouEX93TPTBtHRzFWq8AIRLl9aaOi3T0CraHz94NlHye1od5N2mWeN04sCu9f4WTyxA==} + '@noble/hashes@1.8.0': + resolution: {integrity: sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==} + engines: {node: ^14.21.3 || >=16} + '@nodable/entities@2.1.0': resolution: {integrity: sha512-nyT7T3nbMyBI/lvr6L5TyWbFJAI9FTgVRakNoBqCD+PmID8DzFrrNdLLtHMwMszOtqZa8PAOV24ZqDnQrhQINA==} @@ -4208,16 +4353,19 @@ packages: resolution: {integrity: sha512-OWlrQAnWn9577PhVgqjUvMr1pg57Bc4jv0iL4w0PRuOSRvq67rvHW9Ie/dZVMvCzhSCB+UxhcY/PmCmFj33Q+g==} engines: {node: '>=8.0.0'} - '@opentelemetry/core@1.24.0': - resolution: {integrity: sha512-FP2oN7mVPqcdxJDTTnKExj4mi91EH+DNuArKfHTjPuJWe2K1JfMIVXNfahw1h3onJxQnxS8K0stKkogX05s+Aw==} + '@opentelemetry/core@1.30.1': + resolution: {integrity: sha512-OOCM2C/QIURhJMuKaekP3TRBxBKxG/TWWA0TL2J6nXUtDnuCtccy49LUJF8xPFXMX+0LMcxFpCo8M9cGY1W6rQ==} engines: {node: '>=14'} peerDependencies: - '@opentelemetry/api': '>=1.0.0 <1.9.0' + '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/semantic-conventions@1.24.0': - resolution: {integrity: sha512-yL0jI6Ltuz8R+Opj7jClGrul6pOoYrdfVmzQS4SITXRPH7I5IRZbrwe/6/v8v4WYMa6MYZG480S1+uc/IGfqsA==} + '@opentelemetry/semantic-conventions@1.28.0': + resolution: {integrity: sha512-lp4qAiMTD4sNWW4DbKLBkfiMZ4jbAboJIGOQr5DvciMRI494OapieI9qiODpOt0XBr1LjIDy1xAGAnVs5supTA==} engines: {node: '>=14'} + '@paralleldrive/cuid2@2.3.1': + resolution: {integrity: sha512-XO7cAxhnTZl0Yggq6jOgjiOHhbgcO4NqFqwSmQpjK3b6TEE6Uj/jfSk6wzYyemh3+I0sHirKSetjQwn5cZktFw==} + '@pkgjs/parseargs@0.11.0': resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} @@ -4474,12 +4622,12 @@ packages: resolution: {integrity: sha512-tA7GG7Tj479vojfV3AoxbckalA48aK6giGjNtgH6ihpLwTyHE3fIgRrvt8TWfLwW8X8dyu7vgmAsGLRG7hWWOg==} engines: {node: '>= 8.9.0', npm: '>= 5.5.1'} - '@slack/types@2.11.0': - resolution: {integrity: sha512-UlIrDWvuLaDly3QZhCPnwUSI/KYmV1N9LyhuH6EDKCRS1HWZhyTG3Ja46T3D0rYfqdltKYFXbJSSRPwZpwO0cQ==} + '@slack/types@2.20.1': + resolution: {integrity: sha512-eWX2mdt1ktpn8+40iiMc404uGrih+2fxiky3zBcPjtXKj6HLRdYlmhrPkJi7JTJm8dpXR6BWVWEDBXtaWMKD6A==} engines: {node: '>= 12.13.0', npm: '>= 6.12.0'} - '@slack/web-api@6.12.0': - resolution: {integrity: sha512-RPw6F8rWfGveGkZEJ4+4jUin5iazxRK2q3FpQDz/FvdgzC3nZmPyLx8WRzc6nh0w3MBjEbphNnp2VZksfhpBIQ==} + '@slack/web-api@6.13.0': + resolution: {integrity: sha512-dv65crIgdh9ZYHrevLU6XFHTQwTyDmNqEqzuIrV+Vqe/vgiG6w37oex5ePDU1RGm2IJ90H8iOvHFvzdEO/vB+g==} engines: {node: '>= 12.13.0', npm: '>= 6.12.0'} '@slack/webhook@6.1.0': @@ -4498,14 +4646,26 @@ packages: resolution: {integrity: sha512-lX9Ay+6LisTfpLid2zZtIhSEjHMZoAR5hHCR4H7tBz/Zkfr5ea8RcQ7Tk4mi0P76p4cN+Btz16Ffno7YHpKXnQ==} engines: {node: '>=18.0.0'} + '@smithy/chunked-blob-reader-native@4.2.3': + resolution: {integrity: sha512-jA5k5Udn7Y5717L86h4EIv06wIr3xn8GM1qHRi/Nf31annXcXHJjBKvgztnbn2TxH3xWrPBfgwHsOwZf0UmQWw==} + engines: {node: '>=18.0.0'} + '@smithy/chunked-blob-reader@5.2.0': resolution: {integrity: sha512-WmU0TnhEAJLWvfSeMxBNe5xtbselEO8+4wG0NtZeL8oR21WgH1xiO37El+/Y+H/Ie4SCwBy3MxYWmOYaGgZueA==} engines: {node: '>=18.0.0'} + '@smithy/chunked-blob-reader@5.2.2': + resolution: {integrity: sha512-St+kVicSyayWQca+I1rGitaOEH6uKgE8IUWoYnnEX26SWdWQcL6LvMSD19Lg+vYHKdT9B2Zuu7rd3i6Wnyb/iw==} + engines: {node: '>=18.0.0'} + '@smithy/config-resolver@2.2.0': resolution: {integrity: sha512-fsiMgd8toyUba6n1WRmr+qACzXltpdDkPTAaDqc8QqPBUzO+/JKwL6bUBseHVi8tu9l+3JOK+tSf7cay+4B3LA==} engines: {node: '>=14.0.0'} + '@smithy/config-resolver@4.4.17': + resolution: {integrity: sha512-TzDZcAnhTyAHbXVxWZo7/tEcrIeFq20IBk8So3OLOetWpR8EwY/yEqBMBFaJMeyEiREDq4NfEl+qO3OAUD+vbQ==} + engines: {node: '>=18.0.0'} + '@smithy/config-resolver@4.4.6': resolution: {integrity: sha512-qJpzYC64kaj3S0fueiu3kXm8xPrR3PcXDPEgnaNMRn0EjNSZFoFjvbUp0YUDsRhN1CB90EnHJtbxWKevnH99UQ==} engines: {node: '>=18.0.0'} @@ -4518,10 +4678,18 @@ packages: resolution: {integrity: sha512-x3ie6Crr58MWrm4viHqqy2Du2rHYZjwu8BekasrQx4ca+Y24dzVAwq3yErdqIbc2G3I0kLQA13PQ+/rde+u65g==} engines: {node: '>=18.0.0'} + '@smithy/core@3.23.16': + resolution: {integrity: sha512-JStomOrINQA1VqNEopLsgcdgwd42au7mykKqVr30XFw89wLt9sDxJDi4djVPRwQmmzyTGy/uOvTc2ultMpFi1w==} + engines: {node: '>=18.0.0'} + '@smithy/credential-provider-imds@2.3.0': resolution: {integrity: sha512-BWB9mIukO1wjEOo1Ojgl6LrG4avcaC7T/ZP6ptmAaW4xluhSIPZhY+/PI5YKzlk+jsm+4sQZB45Bt1OfMeQa3w==} engines: {node: '>=14.0.0'} + '@smithy/credential-provider-imds@4.2.14': + resolution: {integrity: sha512-Au28zBN48ZAoXdooGUHemuVBrkE+Ie6RPmGNIAJsFqj33Vhb6xAgRifUydZ2aY+M+KaMAETAlKk5NC5h1G7wpg==} + engines: {node: '>=18.0.0'} + '@smithy/credential-provider-imds@4.2.8': resolution: {integrity: sha512-FNT0xHS1c/CPN8upqbMFP83+ul5YgdisfCfkZ86Jh2NSmnqw/AJ6x5pEogVCTVvSm7j9MopRU89bmDelxuDMYw==} engines: {node: '>=18.0.0'} @@ -4529,6 +4697,10 @@ packages: '@smithy/eventstream-codec@2.2.0': resolution: {integrity: sha512-8janZoJw85nJmQZc4L8TuePp2pk1nxLgkxIR0TUjKJ5Dkj5oelB9WtiSSGXCQvNsJl0VSTvK/2ueMXxvpa9GVw==} + '@smithy/eventstream-codec@4.2.14': + resolution: {integrity: sha512-erZq0nOIpzfeZdCyzZjdJb4nVSKLUmSkaQUVkRGQTXs30gyUGeKnrYEg+Xe1W5gE3aReS7IgsvANwVPxSzY6Pw==} + engines: {node: '>=18.0.0'} + '@smithy/eventstream-codec@4.2.8': resolution: {integrity: sha512-jS/O5Q14UsufqoGhov7dHLOPCzkYJl9QDzusI2Psh4wyYx/izhzvX9P4D69aTxcdfVhEPhjK+wYyn/PzLjKbbw==} engines: {node: '>=18.0.0'} @@ -4537,6 +4709,10 @@ packages: resolution: {integrity: sha512-UaPf8jKbcP71BGiO0CdeLmlg+RhWnlN8ipsMSdwvqBFigl5nil3rHOI/5GE3tfiuX8LvY5Z9N0meuU7Rab7jWw==} engines: {node: '>=14.0.0'} + '@smithy/eventstream-serde-browser@4.2.14': + resolution: {integrity: sha512-8IelTCtTctWRbb+0Dcy+C0aICh1qa0qWXqgjcXDmMuCvPJRnv26hiDZoAau2ILOniki65mCPKqOQs/BaWvO4CQ==} + engines: {node: '>=18.0.0'} + '@smithy/eventstream-serde-browser@4.2.8': resolution: {integrity: sha512-MTfQT/CRQz5g24ayXdjg53V0mhucZth4PESoA5IhvaWVDTOQLfo8qI9vzqHcPsdd2v6sqfTYqF5L/l+pea5Uyw==} engines: {node: '>=18.0.0'} @@ -4545,6 +4721,10 @@ packages: resolution: {integrity: sha512-RHhbTw/JW3+r8QQH7PrganjNCiuiEZmpi6fYUAetFfPLfZ6EkiA08uN3EFfcyKubXQxOwTeJRZSQmDDCdUshaA==} engines: {node: '>=14.0.0'} + '@smithy/eventstream-serde-config-resolver@4.3.14': + resolution: {integrity: sha512-sqHiHpYRYo3FJlaIxD1J8PhbcmJAm7IuM16mVnwSkCToD7g00IBZzKuiLNMGmftULmEUX6/UAz8/NN5uMP8bVA==} + engines: {node: '>=18.0.0'} + '@smithy/eventstream-serde-config-resolver@4.3.8': resolution: {integrity: sha512-ah12+luBiDGzBruhu3efNy1IlbwSEdNiw8fOZksoKoWW1ZHvO/04MQsdnws/9Aj+5b0YXSSN2JXKy/ClIsW8MQ==} engines: {node: '>=18.0.0'} @@ -4553,6 +4733,10 @@ packages: resolution: {integrity: sha512-zpQMtJVqCUMn+pCSFcl9K/RPNtQE0NuMh8sKpCdEHafhwRsjP50Oq/4kMmvxSRy6d8Jslqd8BLvDngrUtmN9iA==} engines: {node: '>=14.0.0'} + '@smithy/eventstream-serde-node@4.2.14': + resolution: {integrity: sha512-Ht/8BuGlKfFTy0H3+8eEu0vdpwGztCnaLLXtpXNdQqiR7Hj4vFScU3T436vRAjATglOIPjJXronY+1WxxNLSiw==} + engines: {node: '>=18.0.0'} + '@smithy/eventstream-serde-node@4.2.8': resolution: {integrity: sha512-cYpCpp29z6EJHa5T9WL0KAlq3SOKUQkcgSoeRfRVwjGgSFl7Uh32eYGt7IDYCX20skiEdRffyDpvF2efEZPC0A==} engines: {node: '>=18.0.0'} @@ -4561,6 +4745,10 @@ packages: resolution: {integrity: sha512-pvoe/vvJY0mOpuF84BEtyZoYfbehiFj8KKWk1ds2AT0mTLYFVs+7sBJZmioOFdBXKd48lfrx1vumdPdmGlCLxA==} engines: {node: '>=14.0.0'} + '@smithy/eventstream-serde-universal@4.2.14': + resolution: {integrity: sha512-lWyt4T2XQZUZgK3tQ3Wn0w3XBvZsK/vjTuJl6bXbnGZBHH0ZUSONTYiK9TgjTTzU54xQr3DRFwpjmhp0oLm3gg==} + engines: {node: '>=18.0.0'} + '@smithy/eventstream-serde-universal@4.2.8': resolution: {integrity: sha512-iJ6YNJd0bntJYnX6s52NC4WFYcZeKrPUr1Kmmr5AwZcwCSzVpS7oavAmxMR7pMq7V+D1G4s9F5NJK0xwOsKAlQ==} engines: {node: '>=18.0.0'} @@ -4568,10 +4756,18 @@ packages: '@smithy/fetch-http-handler@2.5.0': resolution: {integrity: sha512-BOWEBeppWhLn/no/JxUL/ghTfANTjT7kg3Ww2rPqTUY9R4yHPXxJ9JhMe3Z03LN3aPwiwlpDIUcVw1xDyHqEhw==} + '@smithy/fetch-http-handler@5.3.17': + resolution: {integrity: sha512-bXOvQzaSm6MnmLaWA1elgfQcAtN4UP3vXqV97bHuoOrHQOJiLT3ds6o9eo5bqd0TJfRFpzdGnDQdW3FACiAVdw==} + engines: {node: '>=18.0.0'} + '@smithy/fetch-http-handler@5.3.9': resolution: {integrity: sha512-I4UhmcTYXBrct03rwzQX1Y/iqQlzVQaPxWjCjula++5EmWq9YGBrx6bbGqluGc1f0XEfhSkiY4jhLgbsJUMKRA==} engines: {node: '>=18.0.0'} + '@smithy/hash-blob-browser@4.2.15': + resolution: {integrity: sha512-0PJ4Al3fg2nM4qKrAIxyNcApgqHAXcBkN8FeizOz69z0rb26uZ6lMESYtxegaTlXB5Hj84JfwMPavMrwDMjucA==} + engines: {node: '>=18.0.0'} + '@smithy/hash-blob-browser@4.2.9': resolution: {integrity: sha512-m80d/iicI7DlBDxyQP6Th7BW/ejDGiF0bgI754+tiwK0lgMkcaIBgvwwVc7OFbY4eUzpGtnig52MhPAEJ7iNYg==} engines: {node: '>=18.0.0'} @@ -4584,10 +4780,18 @@ packages: resolution: {integrity: sha512-zLWaC/5aWpMrHKpoDF6nqpNtBhlAYKF/7+9yMN7GpdR8CzohnWfGtMznPybnwSS8saaXBMxIGwJqR4HmRp6b3g==} engines: {node: '>=14.0.0'} + '@smithy/hash-node@4.2.14': + resolution: {integrity: sha512-8ZBDY2DD4wr+GGjTpPtiglEsqr0lUP+KHqgZcWczFf6qeZ/YRjMIOoQWVQlmwu7EtxKTd8YXD8lblmYcpBIA1g==} + engines: {node: '>=18.0.0'} + '@smithy/hash-node@4.2.8': resolution: {integrity: sha512-7ZIlPbmaDGxVoxErDZnuFG18WekhbA/g2/i97wGj+wUBeS6pcUeAym8u4BXh/75RXWhgIJhyC11hBzig6MljwA==} engines: {node: '>=18.0.0'} + '@smithy/hash-stream-node@4.2.14': + resolution: {integrity: sha512-tw4GANWkZPb6+BdD4Fgucqzey2+r73Z/GRo9zklsCdwrnxxumUV83ZIaBDdudV4Ylazw3EPTiJZhpX42105ruQ==} + engines: {node: '>=18.0.0'} + '@smithy/hash-stream-node@4.2.8': resolution: {integrity: sha512-v0FLTXgHrTeheYZFGhR+ehX5qUm4IQsjAiL9qehad2cyjMWcN2QG6/4mSwbSgEQzI7jwfoXj7z4fxZUx/Mhj2w==} engines: {node: '>=18.0.0'} @@ -4595,6 +4799,10 @@ packages: '@smithy/invalid-dependency@2.2.0': resolution: {integrity: sha512-nEDASdbKFKPXN2O6lOlTgrEEOO9NHIeO+HVvZnkqc8h5U9g3BIhWsvzFo+UcUbliMHvKNPD/zVxDrkP1Sbgp8Q==} + '@smithy/invalid-dependency@4.2.14': + resolution: {integrity: sha512-c21qJiTSb25xvvOp+H2TNZzPCngrvl5vIPqPB8zQ/DmJF4QWXO19x1dWfMJZ6wZuuWUPPm0gV8C0cU3+ifcWuw==} + engines: {node: '>=18.0.0'} + '@smithy/invalid-dependency@4.2.8': resolution: {integrity: sha512-N9iozRybwAQ2dn9Fot9kI6/w9vos2oTXLhtK7ovGqwZjlOcxu6XhPlpLpC+INsxktqHinn5gS2DXDjDF2kG5sQ==} engines: {node: '>=18.0.0'} @@ -4611,6 +4819,14 @@ packages: resolution: {integrity: sha512-DZZZBvC7sjcYh4MazJSGiWMI2L7E0oCiRHREDzIxi/M2LY79/21iXt6aPLHge82wi5LsuRF5A06Ds3+0mlh6CQ==} engines: {node: '>=18.0.0'} + '@smithy/is-array-buffer@4.2.2': + resolution: {integrity: sha512-n6rQ4N8Jj4YTQO3YFrlgZuwKodf4zUFs7EJIWH86pSCWBaAtAGBFfCM7Wx6D2bBJ2xqFNxGBSrUWswT3M0VJow==} + engines: {node: '>=18.0.0'} + + '@smithy/md5-js@4.2.14': + resolution: {integrity: sha512-V2v0vx+h0iUSNG1Alt+GNBMSLGCrl9iVsdd+Ap67HPM9PN479x12V8LkuMoKImNZxn3MXeuyUjls+/7ZACZghA==} + engines: {node: '>=18.0.0'} + '@smithy/md5-js@4.2.8': resolution: {integrity: sha512-oGMaLj4tVZzLi3itBa9TCswgMBr7k9b+qKYowQ6x1rTyTuO1IU2YHdHUa+891OsOH+wCsH7aTPRsTJO3RMQmjQ==} engines: {node: '>=18.0.0'} @@ -4619,6 +4835,10 @@ packages: resolution: {integrity: sha512-5bl2LG1Ah/7E5cMSC+q+h3IpVHMeOkG0yLRyQT1p2aMJkSrZG7RlXHPuAgb7EyaFeidKEnnd/fNaLLaKlHGzDQ==} engines: {node: '>=14.0.0'} + '@smithy/middleware-content-length@4.2.14': + resolution: {integrity: sha512-xhHq7fX4/3lv5NHxLUk3OeEvl0xZ+Ek3qIbWaCL4f9JwgDZEclPBElljaZCAItdGPQl/kSM4LPMOpy1MYgprpw==} + engines: {node: '>=18.0.0'} + '@smithy/middleware-content-length@4.2.8': resolution: {integrity: sha512-RO0jeoaYAB1qBRhfVyq0pMgBoUK34YEJxVxyjOWYZiOKOq2yMZ4MnVXMZCUDenpozHue207+9P5ilTV1zeda0A==} engines: {node: '>=18.0.0'} @@ -4631,6 +4851,10 @@ packages: resolution: {integrity: sha512-x6vn0PjYmGdNuKh/juUJJewZh7MoQ46jYaJ2mvekF4EesMuFfrl4LaW/k97Zjf8PTCPQmPgMvwewg7eNoH9n5w==} engines: {node: '>=18.0.0'} + '@smithy/middleware-endpoint@4.4.31': + resolution: {integrity: sha512-KJPdCIN2kOE2aGmqZd7eUTr4WQwOGgtLWgUkswGJggs7rBcQYQjcZMEDa3C0DwbOiXS9L8/wDoQHkfxBYLfiLw==} + engines: {node: '>=18.0.0'} + '@smithy/middleware-retry@2.3.1': resolution: {integrity: sha512-P2bGufFpFdYcWvqpyqqmalRtwFUNUA8vHjJR5iGqbfR6mp65qKOLcUd6lTr4S9Gn/enynSrSf3p3FVgVAf6bXA==} engines: {node: '>=14.0.0'} @@ -4639,10 +4863,18 @@ packages: resolution: {integrity: sha512-CBGyFvN0f8hlnqKH/jckRDz78Snrp345+PVk8Ux7pnkUCW97Iinse59lY78hBt04h1GZ6hjBN94BRwZy1xC8Bg==} engines: {node: '>=18.0.0'} + '@smithy/middleware-retry@4.5.4': + resolution: {integrity: sha512-/z7nIFK+ZRW3Ie/l3NEVGdy34LvmEOzBrtBAvgWZ/4PrKX0xP3kWm8pkfcwUk523SqxZhdbQP9JSXgjF77Uhpw==} + engines: {node: '>=18.0.0'} + '@smithy/middleware-serde@2.3.0': resolution: {integrity: sha512-sIADe7ojwqTyvEQBe1nc/GXB9wdHhi9UwyX0lTyttmUWDJLP655ZYE1WngnNyXREme8I27KCaUhyhZWRXL0q7Q==} engines: {node: '>=14.0.0'} + '@smithy/middleware-serde@4.2.19': + resolution: {integrity: sha512-Q6y+W9h3iYVMCKWDoVge+OC1LKFqbEKaq8SIWG2X2bWJRpd/6dDLyICcNLT6PbjH3Rr6bmg/SeDB25XFOFfeEw==} + engines: {node: '>=18.0.0'} + '@smithy/middleware-serde@4.2.9': resolution: {integrity: sha512-eMNiej0u/snzDvlqRGSN3Vl0ESn3838+nKyVfF2FKNXFbi4SERYT6PR392D39iczngbqqGG0Jl1DlCnp7tBbXQ==} engines: {node: '>=18.0.0'} @@ -4651,6 +4883,10 @@ packages: resolution: {integrity: sha512-Qntc3jrtwwrsAC+X8wms8zhrTr0sFXnyEGhZd9sLtsJ/6gGQKFzNB+wWbOcpJd7BR8ThNCoKt76BuQahfMvpeA==} engines: {node: '>=14.0.0'} + '@smithy/middleware-stack@4.2.14': + resolution: {integrity: sha512-2dvkUKLuFdKsCRmOE4Mn63co0Djtsm+JMh0bYZQupN1pJwMeE8FmQmRLLzzEMN0dnNi7CDCYYH8F0EVwWiPBeA==} + engines: {node: '>=18.0.0'} + '@smithy/middleware-stack@4.2.8': resolution: {integrity: sha512-w6LCfOviTYQjBctOKSwy6A8FIkQy7ICvglrZFl6Bw4FmcQ1Z420fUtIhxaUZZshRe0VCq4kvDiPiXrPZAe8oRA==} engines: {node: '>=18.0.0'} @@ -4659,6 +4895,10 @@ packages: resolution: {integrity: sha512-0elK5/03a1JPWMDPaS726Iw6LpQg80gFut1tNpPfxFuChEEklo2yL823V94SpTZTxmKlXFtFgsP55uh3dErnIg==} engines: {node: '>=14.0.0'} + '@smithy/node-config-provider@4.3.14': + resolution: {integrity: sha512-S+gFjyo/weSVL0P1b9Ts8C/CwIfNCgUPikk3sl6QVsfE/uUuO+QsF+NsE/JkpvWqqyz1wg7HFdiaZuj5CoBMRg==} + engines: {node: '>=18.0.0'} + '@smithy/node-config-provider@4.3.8': resolution: {integrity: sha512-aFP1ai4lrbVlWjfpAfRSL8KFcnJQYfTl5QxLJXY32vghJrDuFyPZ6LtUL+JEGYiFRG1PfPLHLoxj107ulncLIg==} engines: {node: '>=18.0.0'} @@ -4671,10 +4911,18 @@ packages: resolution: {integrity: sha512-KX5Wml5mF+luxm1szW4QDz32e3NObgJ4Fyw+irhph4I/2geXwUy4jkIMUs5ZPGflRBeR6BUkC2wqIab4Llgm3w==} engines: {node: '>=18.0.0'} + '@smithy/node-http-handler@4.6.0': + resolution: {integrity: sha512-P734cAoTFtuGfWa/R3jgBnGlURt2w9bYEBwQNMKf58sRM9RShirB2mKwLsVP+jlG/wxpCu8abv8NxdUts8tdLA==} + engines: {node: '>=18.0.0'} + '@smithy/property-provider@2.2.0': resolution: {integrity: sha512-+xiil2lFhtTRzXkx8F053AV46QnIw6e7MV8od5Mi68E1ICOjCeCHw2XfLnDEUHnT9WGUIkwcqavXjfwuJbGlpg==} engines: {node: '>=14.0.0'} + '@smithy/property-provider@4.2.14': + resolution: {integrity: sha512-WuM31CgfsnQ/10i7NYr0PyxqknD72Y5uMfUMVSniPjbEPceiTErb4eIqJQ+pdxNEAUEWrewrGjIRjVbVHsxZiQ==} + engines: {node: '>=18.0.0'} + '@smithy/property-provider@4.2.8': resolution: {integrity: sha512-EtCTbyIveCKeOXDSWSdze3k612yCPq1YbXsbqX3UHhkOSW8zKsM9NOJG5gTIya0vbY2DIaieG8pKo1rITHYL0w==} engines: {node: '>=18.0.0'} @@ -4687,6 +4935,10 @@ packages: resolution: {integrity: sha512-Xy5XK1AFWW2nlY/biWZXu6/krgbaf2dg0q492D8M5qthsnU2H+UgFeZLbM76FnH7s6RO/xhQRkj+T6KBO3JzgQ==} engines: {node: '>=14.0.0'} + '@smithy/protocol-http@5.3.14': + resolution: {integrity: sha512-dN5F8kHx8RNU0r+pCwNmFZyz6ChjMkzShy/zup6MtkRmmix4vZzJdW+di7x//b1LiynIev88FM18ie+wwPcQtQ==} + engines: {node: '>=18.0.0'} + '@smithy/protocol-http@5.3.8': resolution: {integrity: sha512-QNINVDhxpZ5QnP3aviNHQFlRogQZDfYlCkQT+7tJnErPQbDhysondEjhikuANxgMsZrkGeiAxXy4jguEGsDrWQ==} engines: {node: '>=18.0.0'} @@ -4695,6 +4947,10 @@ packages: resolution: {integrity: sha512-L1kSeviUWL+emq3CUVSgdogoM/D9QMFaqxL/dd0X7PCNWmPXqt+ExtrBjqT0V7HLN03Vs9SuiLrG3zy3JGnE5A==} engines: {node: '>=14.0.0'} + '@smithy/querystring-builder@4.2.14': + resolution: {integrity: sha512-XYA5Z0IqTeF+5XDdh4BBmSA0HvbgVZIyv4cmOoUheDNR57K1HgBp9ukUMx3Cr3XpDHHpLBnexPE3LAtDsZkj2A==} + engines: {node: '>=18.0.0'} + '@smithy/querystring-builder@4.2.8': resolution: {integrity: sha512-Xr83r31+DrE8CP3MqPgMJl+pQlLLmOfiEUnoyAlGzzJIrEsbKsPy1hqH0qySaQm4oWrCBlUqRt+idEgunKB+iw==} engines: {node: '>=18.0.0'} @@ -4707,6 +4963,10 @@ packages: resolution: {integrity: sha512-BvHCDrKfbG5Yhbpj4vsbuPV2GgcpHiAkLeIlcA1LtfpMz3jrqizP1+OguSNSj1MwBHEiN+jwNisXLGdajGDQJA==} engines: {node: '>=14.0.0'} + '@smithy/querystring-parser@4.2.14': + resolution: {integrity: sha512-hr+YyqBD23GVvRxGGrcc/oOeNlK3PzT5Fu4dzrDXxzS1LpFiuL2PQQqKPs87M79aW7ziMs+nvB3qdw77SqE7Lw==} + engines: {node: '>=18.0.0'} + '@smithy/querystring-parser@4.2.8': resolution: {integrity: sha512-vUurovluVy50CUlazOiXkPq40KGvGWSdmusa3130MwrR1UNnNgKAlj58wlOe61XSHRpUfIIh6cE0zZ8mzKaDPA==} engines: {node: '>=18.0.0'} @@ -4719,6 +4979,10 @@ packages: resolution: {integrity: sha512-mZ5xddodpJhEt3RkCjbmUQuXUOaPNTkbMGR0bcS8FE0bJDLMZlhmpgrvPNCYglVw5rsYTpSnv19womw9WWXKQQ==} engines: {node: '>=18.0.0'} + '@smithy/service-error-classification@4.3.0': + resolution: {integrity: sha512-9jKsBYQRPR0xBLgc2415RsA5PIcP2sis4oBdN9s0D13cg1B1284mNTjx9Yc+BEERXzuPm5ObktI96OxsKh8E9A==} + engines: {node: '>=18.0.0'} + '@smithy/shared-ini-file-loader@2.4.0': resolution: {integrity: sha512-WyujUJL8e1B6Z4PBfAqC/aGY1+C7T0w20Gih3yrvJSk97gpiVfB+y7c46T4Nunk+ZngLq0rOIdeVeIklk0R3OA==} engines: {node: '>=14.0.0'} @@ -4727,10 +4991,18 @@ packages: resolution: {integrity: sha512-DfQjxXQnzC5UbCUPeC3Ie8u+rIWZTvuDPAGU/BxzrOGhRvgUanaP68kDZA+jaT3ZI+djOf+4dERGlm9mWfFDrg==} engines: {node: '>=18.0.0'} + '@smithy/shared-ini-file-loader@4.4.9': + resolution: {integrity: sha512-495/V2I15SHgedSJoDPD23JuSfKAp726ZI1V0wtjB07Wh7q/0tri/0e0DLefZCHgxZonrGKt/OCTpAtP1wE1kQ==} + engines: {node: '>=18.0.0'} + '@smithy/signature-v4@2.3.0': resolution: {integrity: sha512-ui/NlpILU+6HAQBfJX8BBsDXuKSNrjTSuOYArRblcrErwKFutjrCNb/OExfVRyj9+26F9J+ZmfWT+fKWuDrH3Q==} engines: {node: '>=14.0.0'} + '@smithy/signature-v4@5.3.14': + resolution: {integrity: sha512-1D9Y/nmlVjCeSivCbhZ7hgEpmHyY1h0GvpSZt3l0xcD9JjmjVC1CHOozS6+Gh+/ldMH8JuJ6cujObQqfayAVFA==} + engines: {node: '>=18.0.0'} + '@smithy/signature-v4@5.3.8': resolution: {integrity: sha512-6A4vdGj7qKNRF16UIcO8HhHjKW27thsxYci+5r/uVRkdcBEkOEiY8OMPuydLX4QHSrJqGHPJzPRwwVTqbLZJhg==} engines: {node: '>=18.0.0'} @@ -4743,6 +5015,10 @@ packages: resolution: {integrity: sha512-SCkGmFak/xC1n7hKRsUr6wOnBTJ3L22Qd4e8H1fQIuKTAjntwgU8lrdMe7uHdiT2mJAOWA/60qaW9tiMu69n1A==} engines: {node: '>=18.0.0'} + '@smithy/smithy-client@4.12.12': + resolution: {integrity: sha512-daO7SJn4eM6ArbmrEs+/BTbH7af8AEbSL3OMQdcRvvn8tuUcR5rU2n6DgxIV53aXMS42uwK8NgKKCh5XgqYOPQ==} + engines: {node: '>=18.0.0'} + '@smithy/types@1.2.0': resolution: {integrity: sha512-z1r00TvBqF3dh4aHhya7nz1HhvCg4TRmw51fjMrh5do3h+ngSstt/yKlNbHeb9QxJmFbmN8KEVSWgb1bRvfEoA==} engines: {node: '>=14.0.0'} @@ -4755,12 +5031,20 @@ packages: resolution: {integrity: sha512-9YcuJVTOBDjg9LWo23Qp0lTQ3D7fQsQtwle0jVfpbUHy9qBwCEgKuVH4FqFB3VYu0nwdHKiEMA+oXz7oV8X1kw==} engines: {node: '>=18.0.0'} + '@smithy/types@4.14.1': + resolution: {integrity: sha512-59b5HtSVrVR/eYNei3BUj3DCPKD/G7EtDDe7OEJE7i7FtQFugYo6MxbotS8mVJkLNVf8gYaAlEBwwtJ9HzhWSg==} + engines: {node: '>=18.0.0'} + '@smithy/url-parser@1.1.0': resolution: {integrity: sha512-tpvi761kzboiLNGEWczuybMPCJh6WHB3cz9gWAG95mSyaKXmmX8ZcMxoV+irZfxDqLwZVJ22XTumu32S7Ow8aQ==} '@smithy/url-parser@2.2.0': resolution: {integrity: sha512-hoA4zm61q1mNTpksiSWp2nEl1dt3j726HdRhiNgVJQMj7mLp7dprtF57mOB6JvEk/x9d2bsuL5hlqZbBuHQylQ==} + '@smithy/url-parser@4.2.14': + resolution: {integrity: sha512-p06BiBigJ8bTA3MgnOfCtDUWnAMY0YfedO/GRpmc7p+wg3KW8vbXy1xwSu5ASy0wV7rRYtlfZOIKH4XqfhjSQQ==} + engines: {node: '>=18.0.0'} + '@smithy/url-parser@4.2.8': resolution: {integrity: sha512-NQho9U68TGMEU639YkXnVMV3GEFFULmmaWdlu1E9qzyIePOHsoSnagTGSDv1Zi8DCNN6btxOSdgmy5E/hsZwhA==} engines: {node: '>=18.0.0'} @@ -4773,6 +5057,10 @@ packages: resolution: {integrity: sha512-GkXZ59JfyxsIwNTWFnjmFEI8kZpRNIBfxKjv09+nkAWPt/4aGaEWMM04m4sxgNVWkbt2MdSvE3KF/PfX4nFedQ==} engines: {node: '>=18.0.0'} + '@smithy/util-base64@4.3.2': + resolution: {integrity: sha512-XRH6b0H/5A3SgblmMa5ErXQ2XKhfbQB+Fm/oyLZ2O2kCUrwgg55bU0RekmzAhuwOjA9qdN5VU2BprOvGGUkOOQ==} + engines: {node: '>=18.0.0'} + '@smithy/util-body-length-browser@2.2.0': resolution: {integrity: sha512-dtpw9uQP7W+n3vOtx0CfBD5EWd7EPdIdsQnWTDoFf77e3VUf05uA7R7TGipIo8e4WL2kuPdnsr3hMQn9ziYj5w==} @@ -4780,6 +5068,10 @@ packages: resolution: {integrity: sha512-Fkoh/I76szMKJnBXWPdFkQJl2r9SjPt3cMzLdOB6eJ4Pnpas8hVoWPYemX/peO0yrrvldgCUVJqOAjUrOLjbxg==} engines: {node: '>=18.0.0'} + '@smithy/util-body-length-browser@4.2.2': + resolution: {integrity: sha512-JKCrLNOup3OOgmzeaKQwi4ZCTWlYR5H4Gm1r2uTMVBXoemo1UEghk5vtMi1xSu2ymgKVGW631e2fp9/R610ZjQ==} + engines: {node: '>=18.0.0'} + '@smithy/util-body-length-node@2.3.0': resolution: {integrity: sha512-ITWT1Wqjubf2CJthb0BuT9+bpzBfXeMokH/AAa5EJQgbv9aPMVfnM76iFIZVFf50hYXGbtiV71BHAthNWd6+dw==} engines: {node: '>=14.0.0'} @@ -4788,6 +5080,10 @@ packages: resolution: {integrity: sha512-h53dz/pISVrVrfxV1iqXlx5pRg3V2YWFcSQyPyXZRrZoZj4R4DeWRDo1a7dd3CPTcFi3kE+98tuNyD2axyZReA==} engines: {node: '>=18.0.0'} + '@smithy/util-body-length-node@4.2.3': + resolution: {integrity: sha512-ZkJGvqBzMHVHE7r/hcuCxlTY8pQr1kMtdsVPs7ex4mMU+EAbcXppfo5NmyxMYi2XU49eqaz56j2gsk4dHHPG/g==} + engines: {node: '>=18.0.0'} + '@smithy/util-buffer-from@1.1.0': resolution: {integrity: sha512-9m6NXE0ww+ra5HKHCHig20T+FAwxBAm7DIdwc/767uGWbRcY720ybgPacQNB96JMOI7xVr/CDa3oMzKmW4a+kw==} engines: {node: '>=14.0.0'} @@ -4800,6 +5096,10 @@ packages: resolution: {integrity: sha512-kAY9hTKulTNevM2nlRtxAG2FQ3B2OR6QIrPY3zE5LqJy1oxzmgBGsHLWTcNhWXKchgA0WHW+mZkQrng/pgcCew==} engines: {node: '>=18.0.0'} + '@smithy/util-buffer-from@4.2.2': + resolution: {integrity: sha512-FDXD7cvUoFWwN6vtQfEta540Y/YBe5JneK3SoZg9bThSoOAC/eGeYEua6RkBgKjGa/sz6Y+DuBZj3+YEY21y4Q==} + engines: {node: '>=18.0.0'} + '@smithy/util-config-provider@2.3.0': resolution: {integrity: sha512-HZkzrRcuFN1k70RLqlNK4FnPXKOpkik1+4JaBoHNJn+RnJGYqaa3c5/+XtLOXhlKzlRgNvyaLieHTW2VwGN0VQ==} engines: {node: '>=14.0.0'} @@ -4808,6 +5108,10 @@ packages: resolution: {integrity: sha512-YEjpl6XJ36FTKmD+kRJJWYvrHeUvm5ykaUS5xK+6oXffQPHeEM4/nXlZPe+Wu0lsgRUcNZiliYNh/y7q9c2y6Q==} engines: {node: '>=18.0.0'} + '@smithy/util-config-provider@4.2.2': + resolution: {integrity: sha512-dWU03V3XUprJwaUIFVv4iOnS1FC9HnMHDfUrlNDSh4315v0cWyaIErP8KiqGVbf5z+JupoVpNM7ZB3jFiTejvQ==} + engines: {node: '>=18.0.0'} + '@smithy/util-defaults-mode-browser@2.2.1': resolution: {integrity: sha512-RtKW+8j8skk17SYowucwRUjeh4mCtnm5odCL0Lm2NtHQBsYKrNW0od9Rhopu9wF1gHMfHeWF7i90NwBz/U22Kw==} engines: {node: '>= 10.0.0'} @@ -4816,6 +5120,10 @@ packages: resolution: {integrity: sha512-nIGy3DNRmOjaYaaKcQDzmWsro9uxlaqUOhZDHQed9MW/GmkBZPtnU70Pu1+GT9IBmUXwRdDuiyaeiy9Xtpn3+Q==} engines: {node: '>=18.0.0'} + '@smithy/util-defaults-mode-browser@4.3.48': + resolution: {integrity: sha512-hxVRVPYaRDWa6YQdse1aWX1qrksmLsvNyGBKdc32q4jFzSjxYVNWfstknAfR228TnzS4tzgswXRuYIbhXBuXFQ==} + engines: {node: '>=18.0.0'} + '@smithy/util-defaults-mode-node@2.3.1': resolution: {integrity: sha512-vkMXHQ0BcLFysBMWgSBLSk3+leMpFSyyFj8zQtv5ZyUBx8/owVh1/pPEkzmW/DR/Gy/5c8vjLDD9gZjXNKbrpA==} engines: {node: '>= 10.0.0'} @@ -4824,6 +5132,10 @@ packages: resolution: {integrity: sha512-7dtFff6pu5fsjqrVve0YMhrnzJtccCWDacNKOkiZjJ++fmjGExmmSu341x+WU6Oc1IccL7lDuaUj7SfrHpWc5Q==} engines: {node: '>=18.0.0'} + '@smithy/util-defaults-mode-node@4.2.53': + resolution: {integrity: sha512-ybgCk+9JdBq8pYC8Y6U5fjyS8e4sboyAShetxPNL0rRBtaVl56GSFAxsolVBIea1tXR4LPIzL8i6xqmcf0+DCQ==} + engines: {node: '>=18.0.0'} + '@smithy/util-endpoints@1.2.0': resolution: {integrity: sha512-BuDHv8zRjsE5zXd3PxFXFknzBG3owCpjq8G3FcsXW3CykYXuEqM3nTSsmLzw5q+T12ZYuDlVUZKBdpNbhVtlrQ==} engines: {node: '>= 14.0.0'} @@ -4832,6 +5144,10 @@ packages: resolution: {integrity: sha512-8JaVTn3pBDkhZgHQ8R0epwWt+BqPSLCjdjXXusK1onwJlRuN69fbvSK66aIKKO7SwVFM6x2J2ox5X8pOaWcUEw==} engines: {node: '>=18.0.0'} + '@smithy/util-endpoints@3.4.2': + resolution: {integrity: sha512-a55Tr+3OKld4TTtnT+RhKOQHyPxm3j/xL4OR83WBUhLJaKDS9dnJ7arRMOp3t31dcLhApwG9bgvrRXBHlLdIkg==} + engines: {node: '>=18.0.0'} + '@smithy/util-hex-encoding@2.2.0': resolution: {integrity: sha512-7iKXR+/4TpLK194pVjKiasIyqMtTYJsgKgM242Y9uzt5dhHnUDvMNb+3xIhRJ9QhvqGii/5cRUt4fJn3dtXNHQ==} engines: {node: '>=14.0.0'} @@ -4840,10 +5156,18 @@ packages: resolution: {integrity: sha512-CCQBwJIvXMLKxVbO88IukazJD9a4kQ9ZN7/UMGBjBcJYvatpWk+9g870El4cB8/EJxfe+k+y0GmR9CAzkF+Nbw==} engines: {node: '>=18.0.0'} + '@smithy/util-hex-encoding@4.2.2': + resolution: {integrity: sha512-Qcz3W5vuHK4sLQdyT93k/rfrUwdJ8/HZ+nMUOyGdpeGA1Wxt65zYwi3oEl9kOM+RswvYq90fzkNDahPS8K0OIg==} + engines: {node: '>=18.0.0'} + '@smithy/util-middleware@2.2.0': resolution: {integrity: sha512-L1qpleXf9QD6LwLCJ5jddGkgWyuSvWBkJwWAZ6kFkdifdso+sk3L3O1HdmPvCdnCK3IS4qWyPxev01QMnfHSBw==} engines: {node: '>=14.0.0'} + '@smithy/util-middleware@4.2.14': + resolution: {integrity: sha512-1Su2vj9RYNDEv/V+2E+jXkkwGsgR7dc4sfHn9Z7ruzQHJIEni9zzw5CauvRXlFJfmgcqYP8fWa0dkh2Q2YaQyw==} + engines: {node: '>=18.0.0'} + '@smithy/util-middleware@4.2.8': resolution: {integrity: sha512-PMqfeJxLcNPMDgvPbbLl/2Vpin+luxqTGPpW3NAQVLbRrFRzTa4rNAASYeIGjRV9Ytuhzny39SpyU04EQreF+A==} engines: {node: '>=18.0.0'} @@ -4856,6 +5180,10 @@ packages: resolution: {integrity: sha512-CfJqwvoRY0kTGe5AkQokpURNCT1u/MkRzMTASWMPPo2hNSnKtF1D45dQl3DE2LKLr4m+PW9mCeBMJr5mCAVThg==} engines: {node: '>=18.0.0'} + '@smithy/util-retry@4.3.3': + resolution: {integrity: sha512-idjUvd4M9Jj6rXkhqw4H4reHoweuK4ZxYWyOrEp4N2rOF5VtaOlQGLDQJva/8WanNXk9ScQtsAb7o5UHGvFm4A==} + engines: {node: '>=18.0.0'} + '@smithy/util-stream@2.2.0': resolution: {integrity: sha512-17faEXbYWIRst1aU9SvPZyMdWmqIrduZjVOqCPMIsWFNxs5yQQgFrJL6b2SdiCzyW9mJoDjFtgi53xx7EH+BXA==} engines: {node: '>=14.0.0'} @@ -4864,6 +5192,10 @@ packages: resolution: {integrity: sha512-lKmZ0S/3Qj2OF5H1+VzvDLb6kRxGzZHq6f3rAsoSu5cTLGsn3v3VQBA8czkNNXlLjoFEtVu3OQT2jEeOtOE2CA==} engines: {node: '>=18.0.0'} + '@smithy/util-stream@4.5.24': + resolution: {integrity: sha512-na5vv2mBSDzXewLEEoWGI7LQQkfpmFEomBsmOpzLFjqGctm0iMwXY5lAwesY9pIaErkccW0qzEOUcYP+WKneXg==} + engines: {node: '>=18.0.0'} + '@smithy/util-uri-escape@2.2.0': resolution: {integrity: sha512-jtmJMyt1xMD/d8OtbVJ2gFZOSKc+ueYJZPW20ULW1GOp/q/YIM0wNh+u8ZFao9UaIGz4WoPW8hC64qlWLIfoDA==} engines: {node: '>=14.0.0'} @@ -4872,6 +5204,10 @@ packages: resolution: {integrity: sha512-igZpCKV9+E/Mzrpq6YacdTQ0qTiLm85gD6N/IrmyDvQFA4UnU3d5g3m8tMT/6zG/vVkWSU+VxeUyGonL62DuxA==} engines: {node: '>=18.0.0'} + '@smithy/util-uri-escape@4.2.2': + resolution: {integrity: sha512-2kAStBlvq+lTXHyAZYfJRb/DfS3rsinLiwb+69SstC9Vb0s9vNWkRwpnj918Pfi85mzi42sOqdV72OLxWAISnw==} + engines: {node: '>=18.0.0'} + '@smithy/util-utf8@1.1.0': resolution: {integrity: sha512-p/MYV+JmqmPyjdgyN2UxAeYDj9cBqCjp0C/NsTWnnjoZUVqoeZ6IrW915L9CAKWVECgv9lVQGc4u/yz26/bI1A==} engines: {node: '>=14.0.0'} @@ -4884,6 +5220,14 @@ packages: resolution: {integrity: sha512-zBPfuzoI8xyBtR2P6WQj63Rz8i3AmfAaJLuNG8dWsfvPe8lO4aCPYLn879mEgHndZH1zQ2oXmG8O1GGzzaoZiw==} engines: {node: '>=18.0.0'} + '@smithy/util-utf8@4.2.2': + resolution: {integrity: sha512-75MeYpjdWRe8M5E3AW0O4Cx3UadweS+cwdXjwYGBW5h/gxxnbeZ877sLPX/ZJA9GVTlL/qG0dXP29JWFCD1Ayw==} + engines: {node: '>=18.0.0'} + + '@smithy/util-waiter@4.2.16': + resolution: {integrity: sha512-GtclrKoZ3Lt7jPQ7aTIYKfjY92OgceScftVnkTsG8e1KV8rkvZgN+ny6YSRhd9hxB8rZtwVbmln7NTvE5O3GmQ==} + engines: {node: '>=18.0.0'} + '@smithy/util-waiter@4.2.8': resolution: {integrity: sha512-n+lahlMWk+aejGuax7DPWtqav8HYnWxQwR+LCG2BgCUmaGcTe9qZCFsmw8TMg9iG75HOwhrJCX9TCJRLH+Yzqg==} engines: {node: '>=18.0.0'} @@ -4892,6 +5236,10 @@ packages: resolution: {integrity: sha512-4aUIteuyxtBUhVdiQqcDhKFitwfd9hqoSDYY2KRXiWtgoWJ9Bmise+KfEPDiVHWeJepvF8xJO9/9+WDIciMFFw==} engines: {node: '>=18.0.0'} + '@smithy/uuid@1.1.2': + resolution: {integrity: sha512-O/IEdcCUKkubz60tFbGA7ceITTAJsty+lBjNoorP4Z6XRqaFb/OjQjZODophEcuq68nKm6/0r+6/lLQ+XVpk8g==} + engines: {node: '>=18.0.0'} + '@socket.io/component-emitter@3.1.2': resolution: {integrity: sha512-9BCxFwvbGg/RsZK9tjXd8s4UcwR0MWeFQ1XEKIQVVvAGJyINdrqKMcTRyLoK8Rse1GjzLV9cwjWV1olXRWEXVA==} @@ -5140,12 +5488,15 @@ packages: '@types/node-int64@0.4.32': resolution: {integrity: sha512-xf/JsSlnXQ+mzvc0IpXemcrO4BrCfpgNpMco+GLcXkFk01k/gW9lGJu+Vof0ZSvHK6DsHJDPSbjFPs36QkWXqw==} - '@types/node@20.12.7': - resolution: {integrity: sha512-wq0cICSkRLVaf3UGLMGItu/PtdY7oaXaI/RVU+xliKVOtRna3PRY57ZDfztpDL0n11vfymMUnXv8QwYCO7L1wg==} + '@types/node@20.19.43': + resolution: {integrity: sha512-6oYBAi5ikg4Pl+kGsoYtawUMBT2zZMCvPNF7pVLnHZfd1zf38DRiWn/gT01RYCdUqkv7Fhr+C9ot4/tb+2sVvA==} '@types/node@22.19.10': resolution: {integrity: sha512-tF5VOugLS/EuDlTBijk0MqABfP8UxgYazTLo3uIn3b4yJgg26QRbVYJYsDtHrjdDUIRfP70+VfhTTc+CE1yskw==} + '@types/node@24.13.3': + resolution: {integrity: sha512-Dh8vAsV36ig5wa9OX4pXvMc9D3Veibfw2wix0CUwYODLD8nkj9UsLjASr49nPg+2eKzxhBV+v7L8pXvT4e639Q==} + '@types/q@1.5.8': resolution: {integrity: sha512-hroOstUScF6zhIi+5+x0dzqrHA1EJi+Irri6b1fxolMTqqHIV/Cg77EtnQcZqZCu8hR3mX2BzIxN4/GzI68Kfw==} @@ -5722,27 +6073,15 @@ packages: axios@0.27.2: resolution: {integrity: sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==} - axios@1.11.0: - resolution: {integrity: sha512-1Lx3WLFQWm3ooKDYZD1eXmoGO9fxYQjrycfHFC8P0sCfQVXyROp0p9PFWBehewBOdCwHc+f/b8I0fMto5eSfwA==} - axios@1.12.0: resolution: {integrity: sha512-oXTDccv8PcfjZmPGlWsPSwtOJCZ/b6W5jAMCNcfwJbCzDckwG0jrYJFaWH1yvivfCXjVzV/SPDEhMB3Q+DSurg==} - axios@1.13.1: - resolution: {integrity: sha512-hU4EGxxt+j7TQijx1oYdAjw4xuIp1wRQSsbMFwSthCWeBQur1eF+qJ5iQ5sN3Tw8YRzQNKb8jszgBdMDVqwJcw==} - - axios@1.16.1: - resolution: {integrity: sha512-caYkukvroVPO8KrzuJEb50Hm07KwfBZPEC3VeFHTsqWHvKTsy54hjJz9BS/cdaypROE2rH6xvm9mHX4fgWkr3A==} + axios@1.19.0: + resolution: {integrity: sha512-ht/iuYZXEjFxLH/Hkezgd7m6JKlHHXEUSneaDz8uZe1Gj5QZtCnpyDsckvAiEnT89OEbCLmnte4R4sn7P0EKFw==} axios@1.6.8: resolution: {integrity: sha512-v/ZHtJDU39mDpyBoFVkETcd/uNdxrWRrg3bKpOKzXFA6Bvqopts6ALSMU3y6ijYxbw2B+wPrIv46egTzJXCLGQ==} - axios@1.8.1: - resolution: {integrity: sha512-NN+fvwH/kV01dYUQ3PTOZns4LWtWhOFCAhQ/pHb88WQ1hNe5V/dvFwc4VJcDL11LT9xSX0QtsR8sWUuyOuOq7g==} - - axios@1.8.4: - resolution: {integrity: sha512-eBSYY4Y68NNlHbHBMdeDmKNtDgXWhQsJcGqzO3iLUM0GraQFSS9cVgPX5I9b3lbdFKyYoAEGAZF1DwhTaljNAw==} - balanced-match@1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} @@ -5801,9 +6140,6 @@ packages: bn.js@4.12.1: resolution: {integrity: sha512-k8TVBiPkPJT9uHLdOKfFpqcfprwBFOAAXXozRubr7R7PfIuKvQlzcI4M0pALeqXN09vdaMbUdUj+pass+uULAg==} - bn.js@5.2.1: - resolution: {integrity: sha512-eXRvHzWyYPBuB4NBy0cmYQjGitUrtqwbvlzP3G6VFnNRbsZQIxQ10PbKKHt8gZ/HW/D/747aDl+QkDqg3KQLMQ==} - body-parser@1.19.0: resolution: {integrity: sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw==} engines: {node: '>= 0.8'} @@ -6000,8 +6336,8 @@ packages: ci-info@2.0.0: resolution: {integrity: sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==} - cjs-module-lexer@1.3.1: - resolution: {integrity: sha512-a3KdPAANPbNE4ZUv9h6LckSl9zLsYOP4MBmhIPkRaeyybt+r4UghLvq+xw/YwUcC1gqylCkL4rdVs3Lwupjm4Q==} + cjs-module-lexer@1.4.3: + resolution: {integrity: sha512-9z8TZaGM1pfswYeXrUpzPrkx8UnWYdhJclsiYMm6x/w5+nN+8Tf/LnAgfLGQCm59qAOxU8WwHEq2vNwF6i4j+Q==} clearbit@1.3.5: resolution: {integrity: sha512-tlzF6XzgrT22+9tobCPnlJ1575QIy/wuAoVLUe/7Y6/Y4ZCbXSmVms5l9lbcrGqwmaiTZmhB/EQnimkt/m5ViQ==} @@ -6149,8 +6485,8 @@ packages: config-chain@1.1.13: resolution: {integrity: sha512-qj+f8APARXHrM0hraqXYb2/bOVSV4PvJQlNZ/DVj0QrmNM2q2euizkeuVckQ57J+W0mRH6Hvi+k50M4Jul2VRQ==} - config@3.3.11: - resolution: {integrity: sha512-Dhn63ZoWCW5EMg4P0Sl/XNsj/7RLiUIA1x1npCy+m2cRwRHzLnt3UtYtxRDMZW/6oOMdWhCzaGYkOcajGgrAOA==} + config@3.3.12: + resolution: {integrity: sha512-Vmx389R/QVM3foxqBzXO8t2tUikYZP64Q6vQxGrsMpREeJc/aWRnPRERXWsYzOHAumx/AOoILWe6nU3ZJL+6Sw==} engines: {node: '>= 10.0.0'} configstore@5.0.1: @@ -6338,12 +6674,12 @@ packages: date-and-time@0.14.2: resolution: {integrity: sha512-EFTCh9zRSEpGPmJaexg7HTuzZHh6cnJj1ui7IGCFNXzd2QdpsNh05Db5TF3xzJm30YN+A8/6xHSuRcQqoc3kFA==} - dc-polyfill@0.1.4: - resolution: {integrity: sha512-8iwEduR2jR9wWYggeaYtYZWRiUe3XZPyAQtMTL1otv8X3kfR8xUIVb4l5awHEeyDrH6Je7N324lKzMKlMMN6Yw==} + dc-polyfill@0.1.10: + resolution: {integrity: sha512-9iSbB8XZ7aIrhUtWI5ulEOJ+IyUN+axquodHK+bZO4r7HfY/xwmo6I4fYYf+aiDom+WMcN/wnzCz+pKvHDDCug==} engines: {node: '>=12.17'} - dd-trace@4.38.0: - resolution: {integrity: sha512-tEcCFz3drKoifSo4U7EyT+PDg0TXBBjl07z51NkouIou7eiKK/vGWQ5zKBDmt0qCPLnvNuNFT3T6ErSoLZNSOw==} + dd-trace@4.55.0: + resolution: {integrity: sha512-fGB4ljdpk2iTeFVkpF9CNb+6ty2x/em+hvvspJm83IEf8/bn5/tLRWj4ONBqAbVvlHIPM4b3AwVNhMvCygnZlg==} engines: {node: '>=16'} debug@2.6.9: @@ -6681,10 +7017,6 @@ packages: resolution: {integrity: sha512-e+HfNH61Bj1X9/jLc5v1owaLYuHdeHHSQlkhCBiTK8rBvKaULl/beGMxwrMXjpYrv4pz22BlY570vVePA2ho4A==} engines: {node: '>= 0.4'} - es-define-property@1.0.0: - resolution: {integrity: sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==} - engines: {node: '>= 0.4'} - es-define-property@1.0.1: resolution: {integrity: sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==} engines: {node: '>= 0.4'} @@ -6735,16 +7067,16 @@ packages: es6-weak-map@2.0.3: resolution: {integrity: sha512-p5um32HOTO1kP+w7PRnB+5lQ43Z6muuMuIMffvDN8ZB4GcnjLBV6zGStpbASIMk4DCAvEaamhe2zhyCb/QXXsA==} - esbuild@0.19.12: - resolution: {integrity: sha512-aARqgq8roFBj054KvQr5f1sFu0D65G+miZRCuJyJ0G13Zwx7vRar5Zhn2tkQNzIXcBrNVsv/8stehpj+GAjgbg==} - engines: {node: '>=12'} - hasBin: true - esbuild@0.25.12: resolution: {integrity: sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==} engines: {node: '>=18'} hasBin: true + esbuild@0.28.0: + resolution: {integrity: sha512-sNR9MHpXSUV/XB4zmsFKN+QgVG82Cc7+/aaxJ8Adi8hyOac+EXptIp45QBPaVyX3N70664wRbTcLTOemCAnyqw==} + engines: {node: '>=18'} + hasBin: true + escalade@3.1.2: resolution: {integrity: sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==} engines: {node: '>=6'} @@ -6918,9 +7250,6 @@ packages: event-emitter@0.3.5: resolution: {integrity: sha512-D9rRn9y7kLPnJ+hMq7S/nhvoKwwvVJahBi2BPmx3bvbsEdK3W9ii8cBSGjP+72/LnM4n6fo3+dkCX5FeTQruXA==} - event-lite@0.1.3: - resolution: {integrity: sha512-8qz9nOz5VeD2z96elrEKD2U433+L3DWdUdDkOINLGOJvx1GsMBbMn0aCeu28y8/e85A6mCigBiFlYMnTBEGlSw==} - event-target-shim@5.0.1: resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} engines: {node: '>=6'} @@ -7057,6 +7386,10 @@ packages: resolution: {integrity: sha512-JeaA2Vm9ffQKp9VjvfzObuMCjUYAp5WDYhRYL5LrBPY/jUDlUtOvDfot0vKSkB9tuX885BDHjtw4fZadD95wnA==} hasBin: true + fast-xml-parser@5.5.8: + resolution: {integrity: sha512-Z7Fh2nVQSb2d+poDViM063ix2ZGt9jmY1nWhPfHBOK2Hgnb/OW3P4Et3P/81SEej0J7QbWtJqxO05h8QYfK7LQ==} + hasBin: true + fast-xml-parser@5.8.0: resolution: {integrity: sha512-6bIM7fsJxeo3uXv7OncQYsBAMPJ7V16Slahl/6M98C/i2q+vB1+4a0MtrvYwDFEUrwDSbAmeLDRXsOBwrL7yAg==} hasBin: true @@ -7138,15 +7471,6 @@ packages: fn.name@1.1.0: resolution: {integrity: sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==} - follow-redirects@1.15.6: - resolution: {integrity: sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==} - engines: {node: '>=4.0'} - peerDependencies: - debug: '*' - peerDependenciesMeta: - debug: - optional: true - follow-redirects@1.16.0: resolution: {integrity: sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==} engines: {node: '>=4.0'} @@ -7163,20 +7487,12 @@ packages: resolution: {integrity: sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==} engines: {node: '>=14'} - form-data@2.5.1: - resolution: {integrity: sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==} + form-data@2.5.6: + resolution: {integrity: sha512-Ogz/E85h9tlfJzpI6TuFpGcHZFhLrb9Gw8wq9v40CxSCPnv7ahKr6Xgtkn0KYCDQJ8DNn5VoMO8EXr9V5PadyA==} engines: {node: '>= 0.12'} - form-data@4.0.0: - resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==} - engines: {node: '>= 6'} - - form-data@4.0.4: - resolution: {integrity: sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==} - engines: {node: '>= 6'} - - form-data@4.0.5: - resolution: {integrity: sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==} + form-data@4.0.6: + resolution: {integrity: sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==} engines: {node: '>= 6'} formdata-polyfill@4.0.10: @@ -7190,9 +7506,8 @@ packages: resolution: {integrity: sha512-KcpbcpuLNOwrEjnbpMC0gS+X8ciDoZE1kkqzat4a8vrprf+s9pKNQ/QIwWfbfs4ltgmFl3MD177SNTkve3BwGQ==} deprecated: 'Please upgrade to latest, formidable@v2 or formidable@v3! Check these notes: https://bit.ly/2ZEqIau' - formidable@2.1.2: - resolution: {integrity: sha512-CM3GuJ57US06mlpQ47YcunuUZ9jpm8Vx+P2CGt2j7HpgkKZO/DJYQ0Bobim8G6PFQmK5lOqOOdUXboU+h73A4g==} - deprecated: 'ACTION REQUIRED: SWITCH TO v3 - v1 and v2 are VULNERABLE! v1 is DEPRECATED FOR OVER 2 YEARS! Use formidable@latest or try formidable-mini for fresh projects' + formidable@2.1.5: + resolution: {integrity: sha512-Oz5Hwvwak/DCaXVVUtPn4oLMLLy1CdclLKO1LFgU7XzDpVMUU5UjlSLpGMocyQNNk8F6IJW9M/YdooSn2MRI+Q==} forwarded@0.2.0: resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} @@ -7322,9 +7637,6 @@ packages: resolution: {integrity: sha512-g0QYk1dZBxGwk+Ngc+ltRH2IBp2f7zBkBMBJZCDerh6EhlhSR6+9irMCuT/09zD6qkarHUSn529sK/yL4S27mg==} engines: {node: '>= 0.4'} - get-tsconfig@4.7.3: - resolution: {integrity: sha512-ZvkrzoUA0PQZM6fy6+/Hce561s+faD1rsNwhnO5FelNjyy7EMGJ3Rz1AQ8GYDWjhRs/7dBLOEJvhK8MiEJOAFg==} - git-raw-commits@4.0.0: resolution: {integrity: sha512-ICsMM1Wk8xSGMowkOmPrzo2Fgmfo4bMHLNX6ytHjajRJUqvHOw/TFapQ+QG75c3X/tTDDhOSRPGC52dDbNM8FQ==} engines: {node: '>=16'} @@ -7350,11 +7662,11 @@ packages: glob@6.0.4: resolution: {integrity: sha512-MKZeRNyYZAVVVG1oZeLaWie1uweH40m9AZwIwxyPbTSX4hHrVYSzLg0Ro5Z5R7XKkIX+Cc6oD1rqeDJnwsB8/A==} - deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me + deprecated: Glob versions prior to v9 are no longer supported glob@7.2.3: resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==} - deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me + deprecated: Glob versions prior to v9 are no longer supported global-directory@4.0.1: resolution: {integrity: sha512-wHTUcDUoZ1H5/0iVqEudYW4/kAlN5cZ3j/bXn0Dpbizl9iaUVeWSHqiOjsgk6OW2bkLclbBjzewBz6weQ1zA2Q==} @@ -7406,9 +7718,6 @@ packages: deprecated: Package is no longer maintained hasBin: true - gopd@1.0.1: - resolution: {integrity: sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==} - gopd@1.2.0: resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} engines: {node: '>= 0.4'} @@ -7486,6 +7795,10 @@ packages: resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} engines: {node: '>= 0.4'} + hasown@2.0.4: + resolution: {integrity: sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==} + engines: {node: '>= 0.4'} + he@1.2.0: resolution: {integrity: sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==} hasBin: true @@ -7498,10 +7811,6 @@ packages: resolution: {integrity: sha512-Avg4XxSBrehD94mkRwEljnO+6RZx7AGfk8Wa6K1nxaU+hbXlFOhlOIMgPfFqOYQB/dBCsTpootTGuiOG+CHiQA==} engines: {node: '>=10.0.0'} - hexoid@1.0.0: - resolution: {integrity: sha512-QFLV0taWQOZtvIRIAdBChesmogZrtuXvVWsFHZTk2SU+anspqZ2vMnoLg7IE1+Uk16N19APic1BuF8bC8c2m5g==} - engines: {node: '>=8'} - highlight.js@10.7.3: resolution: {integrity: sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A==} @@ -7629,8 +7938,8 @@ packages: resolution: {integrity: sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==} engines: {node: '>=6'} - import-in-the-middle@1.8.1: - resolution: {integrity: sha512-yhRwoHtiLGvmSozNOALgjRPFI6uYsds60EoMqqnXyyv+JOIW/BrrLejuTGBt+bq0T5tLzOHrN0T7xYTm4Qt/ng==} + import-in-the-middle@1.11.2: + resolution: {integrity: sha512-gK6Rr6EykBcc6cVWRSBR5TWf8nn6hZMYSRYqCcHa0l0d1fPK7JSYo6+Mlmck76jIX9aL/IZ71c06U2VpFwl1zA==} import-lazy@2.1.0: resolution: {integrity: sha512-m7ZEHgtw69qOGw+jwxXkHlrlIPdTGkyh66zXZ1ajZbxkDBNjSY/LGbmjc7h0s2ELsUDTAhFr55TrPSSqJGPG0A==} @@ -7674,9 +7983,6 @@ packages: int53@1.0.0: resolution: {integrity: sha512-u8BMiMa05OPBgd32CKTead0CVTsFVgwFk23nNXo1teKPF6Sxcu0lXxEzP//zTcaKzXbGgPDXGmj/woyv+I4C5w==} - int64-buffer@0.1.10: - resolution: {integrity: sha512-v7cSY1J8ydZ0GyjUHqF+1bshJ6cnEVLo9EnjB8p+4HDRPZc9N5jjmvUV7NvEsqQOKyH0pmIBFWXVQbiS0+OBbA==} - internal-slot@1.0.7: resolution: {integrity: sha512-NGnrKwXzSms2qUUih/ILZ5JBqNTSa1+ZmP6flaIp6KmSElgE9qdndzS3cqjrDovwFdmwsGsLdeFgB6suw+1e9g==} engines: {node: '>= 0.4'} @@ -7693,10 +7999,6 @@ packages: resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==} engines: {node: '>= 0.10'} - ipaddr.js@2.2.0: - resolution: {integrity: sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA==} - engines: {node: '>= 10'} - is-array-buffer@3.0.4: resolution: {integrity: sha512-wcjaerHw0ydZwfhiKbXJWLDY8A7yV7KhjQOpb83hGgGfId/aQa4TOvwyzn2PuswW2gPCYEL/nEAiSVpdOj1lXw==} engines: {node: '>= 0.4'} @@ -8069,8 +8371,8 @@ packages: resolution: {integrity: sha512-MT/xP0CrubFRNLNKvxJ2BYfy53Zkm++5bX9dtuPbqAeQpTVe0MQTFhao8+Cp//EmJp244xt6Drw/GVEGCUj40g==} engines: {node: '>=12', npm: '>=6'} - jwa@1.4.1: - resolution: {integrity: sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==} + jwa@1.4.2: + resolution: {integrity: sha512-eeH5JO+21J78qMvTIDdBXidBd6nG2kZjg5Ohz/1fpa28Z4CcsWUzJ1ZZyFq/3z3N17aZy+ZuBoHljASbL1WfOw==} jwa@2.0.1: resolution: {integrity: sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==} @@ -8079,8 +8381,8 @@ packages: resolution: {integrity: sha512-v7nqlfezb9YfHHzYII3ef2a2j1XnGeSE/bK3WfumaYCqONAIstJbrEGapz4kadScZzEt7zYCN7bucj8C0Mv/Rg==} engines: {node: '>=14'} - jws@3.2.2: - resolution: {integrity: sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==} + jws@3.2.3: + resolution: {integrity: sha512-byiJ0FLRdLdSVSReO/U4E7RoEyOCKnEnEPMjq3HxWtvzLsV08/i5RQKsFVNkCldrCaPr2vDNAOMsfs8T/Hze7g==} jws@4.0.1: resolution: {integrity: sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==} @@ -8487,8 +8789,8 @@ packages: engines: {node: '>=10'} hasBin: true - module-details-from-path@1.0.3: - resolution: {integrity: sha512-ySViT69/76t8VhE1xXHK6Ch4NcDd26gx0MzKXLO+F7NOtnqH68d9zF94nT8ZWSxXh8ELOERsnJO/sWt1xZYw5A==} + module-details-from-path@1.0.4: + resolution: {integrity: sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w==} moment-timezone@0.5.45: resolution: {integrity: sha512-HIWmqA86KcmCAhnMAN0wuDOARV/525R2+lOLotuGFzn4HO+FH+/645z2wx0Dt3iDv6/p61SIvKnDstISainhLQ==} @@ -8518,10 +8820,6 @@ packages: resolution: {integrity: sha512-kh8ARjh8rMN7Du2igDRO9QJnqCb2xYTJxyQYK7vJJS4TvLLmsbyhiKpSW+t+y26gyOyMd0riphX0GeWKU3ky5g==} engines: {node: '>=12.13'} - msgpack-lite@0.1.26: - resolution: {integrity: sha512-SZ2IxeqZ1oRFGo0xFGbvBJWMp3yLIY9rlIJyxy8CGrwZn1f0ZK4r6jV/AM1r0FZMDUkWkglOk/eeKIL9g77Nxw==} - hasBin: true - mute-stream@0.0.8: resolution: {integrity: sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==} @@ -8588,9 +8886,6 @@ packages: resolution: {integrity: sha512-IWjIExdVYlmwXuzHdY/Q3lXCv1gbqoAXPazQhy2w4Xgtgha3H0OOujEESVPQcFUFMWm+pAk2gKnb57g8S41JZg==} engines: {node: '>= 20.0.0'} - node-abort-controller@3.1.1: - resolution: {integrity: sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==} - node-addon-api@3.2.1: resolution: {integrity: sha512-mmcei9JghVNDYydghQmeDX8KoAm0FAiYyIcUt/N4nhyAipB17pllZQDOJD2fotxABnt4Mdz+dKTO7eftLg4d0A==} @@ -8631,6 +8926,10 @@ packages: resolution: {integrity: sha512-u6fs2AEUljNho3EYTJNBfImO5QTo/J/1Etd+NVdCj7qWKUSN/bSLkZwhDv7I+w/MSC6qJ4cknepkAYykDdK8og==} hasBin: true + node-gyp-build@4.8.4: + resolution: {integrity: sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==} + hasBin: true + node-html-markdown@1.3.0: resolution: {integrity: sha512-OeFi3QwC/cPjvVKZ114tzzu+YoR+v9UXW5RwSXGUqGb0qCl0DvP406tzdL7SFn8pZrMyzXoisfG2zcuF9+zw4g==} engines: {node: '>=10.0.0'} @@ -9069,9 +9368,6 @@ packages: path-to-regexp@0.1.7: resolution: {integrity: sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==} - path-to-regexp@0.1.8: - resolution: {integrity: sha512-EErxvEqTuliG5GCVHNt3K3UmfKhlOM26QtiJZ6XBnZgCd7n+P5aHNV37wFHGJSpbjN4danT+1CpOFT4giETmRQ==} - path-to-regexp@8.4.2: resolution: {integrity: sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==} @@ -9221,8 +9517,8 @@ packages: resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==} engines: {node: '>=0.10.0'} - pprof-format@2.1.0: - resolution: {integrity: sha512-0+G5bHH0RNr8E5hoZo/zJYsL92MhkZjwrHp3O2IxmY8RJL9ooKeuZ8Tm0ZNBw5sGZ9TiM71sthTjWoR2Vf5/xw==} + pprof-format@2.2.1: + resolution: {integrity: sha512-p4tVN7iK19ccDqQv8heyobzUmbHyds4N2FI6aBMcXz6y99MglTWDxIyhFkNaLeEXs6IFUEzT0zya0icbSLLY0g==} prelude-ls@1.2.1: resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==} @@ -9264,10 +9560,6 @@ packages: resolution: {integrity: sha512-SAzp/O4Yh02jGdRc+uIrGoe87dkN/XtwxfZ4ZyafJHymd79ozp5VG5nyZ7ygqPM5+cpLDjjGnYFUkngonyDPOQ==} engines: {node: '>=14.0.0'} - protobufjs@7.2.6: - resolution: {integrity: sha512-dgJaEDDL6x8ASUZ1YqWciTRrdOuYNzoOf27oHNfdyvKqHr5i0FV7FSLU+aIeFjyFgVxrpTOtQUi0BLLBymZaBw==} - engines: {node: '>=12.0.0'} - protobufjs@7.5.5: resolution: {integrity: sha512-3wY1AxV+VBNW8Yypfd1yQY9pXnqTAN+KwQxL8iYm3/BjKYMNg4i0owhEe26PWDOMaIrzeeF98Lqd5NGz4omiIg==} engines: {node: '>=12.0.0'} @@ -9322,18 +9614,10 @@ packages: resolution: {integrity: sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==} engines: {node: '>=0.6'} - qs@6.12.1: - resolution: {integrity: sha512-zWmv4RSuB9r2mYQw3zxQuHWeU+42aKi1wWig/j4ele4ygELZ7PEO6MM7rim9oAQH2A5MWfsAVf/jPvTPgCbvUQ==} - engines: {node: '>=0.6'} - qs@6.13.0: resolution: {integrity: sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==} engines: {node: '>=0.6'} - qs@6.14.0: - resolution: {integrity: sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==} - engines: {node: '>=0.6'} - qs@6.15.3: resolution: {integrity: sha512-O9gl3zCl5h5blw1KGUzQKhA5oUXSl8rwUIM5o0S3nCXMliSvy5Dzx7/DJcI+SwgICv+IneSZwhBh1oSyEHA71A==} engines: {node: '>=0.6'} @@ -9466,9 +9750,6 @@ packages: resolution: {integrity: sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==} engines: {node: '>=8'} - resolve-pkg-maps@1.0.0: - resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==} - resolve@1.22.8: resolution: {integrity: sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==} hasBin: true @@ -9582,8 +9863,9 @@ packages: sax@1.2.1: resolution: {integrity: sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA==} - sax@1.3.0: - resolution: {integrity: sha512-0s+oAmw9zLl1V1cS9BtZN7JAd0cW5e0QH4W3LWEK6a4LaLEA2OTpGYWDY+6XasBLtz6wkm3u1xRw95mRuJ59WA==} + sax@1.6.0: + resolution: {integrity: sha512-6R3J5M4AcbtLUdZmRv2SygeVaM7IhrLXu9BmnOGmmACak8fiUtOsYNWUS4uK7upbmHIBbLBeFeI//477BKLBzA==} + engines: {node: '>=11.0.0'} schema-utils@4.3.3: resolution: {integrity: sha512-eflK8wEtyOE6+hsaRVPxvUKYCpRgzLqDTb8krvAsRIwOGlHoSgYLgBXoubGgLd2fT41/OUYdb48v4k4WWHQurA==} @@ -9730,8 +10012,9 @@ packages: resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} engines: {node: '>=8'} - shell-quote@1.8.1: - resolution: {integrity: sha512-6j1W9l1iAs/4xYBI1SYOVZyFcCis9b4KCLQ8fgAGG07QvzaRLVVRQvAy85yNmmZSjYjg4MWh4gNvlPujU/5LpA==} + shell-quote@1.8.3: + resolution: {integrity: sha512-ObmnIF4hXNg1BqhnHmgbDETF8dLPCggZWBjkQfhZpbszZnYur5DUljTcCHii5LC3J5E0yeO/1LIMyH+UvHQgyw==} + engines: {node: '>= 0.4'} should-equal@2.0.0: resolution: {integrity: sha512-ZP36TMrK9euEuWQYBig9W55WPC7uo37qzAEmbjHz4gfyuXrEUgF8cUvQVO+w+d3OMfPvSRQJ22lSm8MQJ43LTA==} @@ -9751,10 +10034,6 @@ packages: should@13.2.3: resolution: {integrity: sha512-ggLesLtu2xp+ZxI+ysJTmNjh2U0TsC+rQ/pfED9bUZZ4DKefP27D+7YJVVTvKsmjLpIi9jAa7itwDGkDDmt1GQ==} - side-channel-list@1.0.0: - resolution: {integrity: sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==} - engines: {node: '>= 0.4'} - side-channel-list@1.0.1: resolution: {integrity: sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==} engines: {node: '>= 0.4'} @@ -9771,10 +10050,6 @@ packages: resolution: {integrity: sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==} engines: {node: '>= 0.4'} - side-channel@1.1.0: - resolution: {integrity: sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==} - engines: {node: '>= 0.4'} - side-channel@1.1.1: resolution: {integrity: sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==} engines: {node: '>= 0.4'} @@ -9824,8 +10099,9 @@ packages: snappyjs@0.7.0: resolution: {integrity: sha512-u5iEEXkMe2EInQio6Wv9LWHOQYRDbD2O9hzS27GpT/lwfIQhTCnHCTqedqHIHe9ZcvQo+9au6vngQayipz1NYw==} - snowflake-sdk@2.3.4: - resolution: {integrity: sha512-J+YIRLXDsE+nNn5UOtz7maWyMyrdHmvXoh/u8g079VEa9VwvjVaQa66TCt3bxL6fZQH1nmp5WhEbOPfbFn4Mag==} + snowflake-sdk@2.4.0: + resolution: {integrity: sha512-0nEQoGMPpCpe1Rvj9tlBp0z4QbOCxfyUdRXyFPKRDneR3ok7qNnlgUXEgldvryolUwSRNbsqyjRC4AyPdIyezg==} + engines: {node: '>=18'} peerDependencies: asn1.js: ^5.4.1 @@ -10318,8 +10594,8 @@ packages: peerDependencies: typescript: '>=2.8.0 || >= 3.2.0-dev || >= 3.3.0-dev || >= 3.4.0-dev || >= 3.5.0-dev || >= 3.6.0-dev || >= 3.6.0-beta || >= 3.7.0-dev || >= 3.7.0-beta' - tsx@4.7.3: - resolution: {integrity: sha512-+fQnMqIp/jxZEXLcj6WzYy9FhcS5/Dfk8y4AtzJ6ejKcKqmfTF8Gso/jtrzDggCF2zTU20gJa6n8XqPYwDAUYQ==} + tsx@4.23.12: + resolution: {integrity: sha512-FDf4L4sYzKtzWYhU/Xm0AQFdTjdIxNo9ElTf2mxXM6k8YMHXzYUe4yODVaXP4V9uMFbVg8c0qyBccK2OOxb45Q==} engines: {node: '>=18.0.0'} hasBin: true @@ -10402,12 +10678,12 @@ packages: undefsafe@2.0.5: resolution: {integrity: sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==} - undici-types@5.26.5: - resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} - undici-types@6.21.0: resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + undici-types@7.18.2: + resolution: {integrity: sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==} + undici@5.29.0: resolution: {integrity: sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg==} engines: {node: '>=14.0'} @@ -11137,6 +11413,66 @@ snapshots: transitivePeerDependencies: - aws-crt + '@aws-sdk/client-s3@3.1033.0': + dependencies: + '@aws-crypto/sha1-browser': 5.2.0 + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/core': 3.974.2 + '@aws-sdk/credential-provider-node': 3.972.33 + '@aws-sdk/middleware-bucket-endpoint': 3.972.10 + '@aws-sdk/middleware-expect-continue': 3.972.10 + '@aws-sdk/middleware-flexible-checksums': 3.974.10 + '@aws-sdk/middleware-host-header': 3.972.10 + '@aws-sdk/middleware-location-constraint': 3.972.10 + '@aws-sdk/middleware-logger': 3.972.10 + '@aws-sdk/middleware-recursion-detection': 3.972.11 + '@aws-sdk/middleware-sdk-s3': 3.972.31 + '@aws-sdk/middleware-ssec': 3.972.10 + '@aws-sdk/middleware-user-agent': 3.972.32 + '@aws-sdk/region-config-resolver': 3.972.12 + '@aws-sdk/signature-v4-multi-region': 3.996.19 + '@aws-sdk/types': 3.973.8 + '@aws-sdk/util-endpoints': 3.996.7 + '@aws-sdk/util-user-agent-browser': 3.972.10 + '@aws-sdk/util-user-agent-node': 3.973.18 + '@smithy/config-resolver': 4.4.17 + '@smithy/core': 3.23.16 + '@smithy/eventstream-serde-browser': 4.2.14 + '@smithy/eventstream-serde-config-resolver': 4.3.14 + '@smithy/eventstream-serde-node': 4.2.14 + '@smithy/fetch-http-handler': 5.3.17 + '@smithy/hash-blob-browser': 4.2.15 + '@smithy/hash-node': 4.2.14 + '@smithy/hash-stream-node': 4.2.14 + '@smithy/invalid-dependency': 4.2.14 + '@smithy/md5-js': 4.2.14 + '@smithy/middleware-content-length': 4.2.14 + '@smithy/middleware-endpoint': 4.4.31 + '@smithy/middleware-retry': 4.5.4 + '@smithy/middleware-serde': 4.2.19 + '@smithy/middleware-stack': 4.2.14 + '@smithy/node-config-provider': 4.3.14 + '@smithy/node-http-handler': 4.6.0 + '@smithy/protocol-http': 5.3.14 + '@smithy/smithy-client': 4.12.12 + '@smithy/types': 4.14.1 + '@smithy/url-parser': 4.2.14 + '@smithy/util-base64': 4.3.2 + '@smithy/util-body-length-browser': 4.2.2 + '@smithy/util-body-length-node': 4.2.3 + '@smithy/util-defaults-mode-browser': 4.3.48 + '@smithy/util-defaults-mode-node': 4.2.53 + '@smithy/util-endpoints': 3.4.2 + '@smithy/util-middleware': 4.2.14 + '@smithy/util-retry': 4.3.3 + '@smithy/util-stream': 4.5.24 + '@smithy/util-utf8': 4.2.2 + '@smithy/util-waiter': 4.2.16 + tslib: 2.8.1 + transitivePeerDependencies: + - aws-crt + '@aws-sdk/client-s3@3.985.0': dependencies: '@aws-crypto/sha1-browser': 5.2.0 @@ -11415,6 +11751,51 @@ snapshots: transitivePeerDependencies: - aws-crt + '@aws-sdk/client-sts@3.1033.0': + dependencies: + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/core': 3.974.2 + '@aws-sdk/credential-provider-node': 3.972.33 + '@aws-sdk/middleware-host-header': 3.972.10 + '@aws-sdk/middleware-logger': 3.972.10 + '@aws-sdk/middleware-recursion-detection': 3.972.11 + '@aws-sdk/middleware-user-agent': 3.972.32 + '@aws-sdk/region-config-resolver': 3.972.12 + '@aws-sdk/signature-v4-multi-region': 3.996.19 + '@aws-sdk/types': 3.973.8 + '@aws-sdk/util-endpoints': 3.996.7 + '@aws-sdk/util-user-agent-browser': 3.972.10 + '@aws-sdk/util-user-agent-node': 3.973.18 + '@smithy/config-resolver': 4.4.17 + '@smithy/core': 3.23.16 + '@smithy/fetch-http-handler': 5.3.17 + '@smithy/hash-node': 4.2.14 + '@smithy/invalid-dependency': 4.2.14 + '@smithy/middleware-content-length': 4.2.14 + '@smithy/middleware-endpoint': 4.4.31 + '@smithy/middleware-retry': 4.5.4 + '@smithy/middleware-serde': 4.2.19 + '@smithy/middleware-stack': 4.2.14 + '@smithy/node-config-provider': 4.3.14 + '@smithy/node-http-handler': 4.6.0 + '@smithy/protocol-http': 5.3.14 + '@smithy/smithy-client': 4.12.12 + '@smithy/types': 4.14.1 + '@smithy/url-parser': 4.2.14 + '@smithy/util-base64': 4.3.2 + '@smithy/util-body-length-browser': 4.2.2 + '@smithy/util-body-length-node': 4.2.3 + '@smithy/util-defaults-mode-browser': 4.3.48 + '@smithy/util-defaults-mode-node': 4.2.53 + '@smithy/util-endpoints': 3.4.2 + '@smithy/util-middleware': 4.2.14 + '@smithy/util-retry': 4.3.3 + '@smithy/util-utf8': 4.2.2 + tslib: 2.8.1 + transitivePeerDependencies: + - aws-crt + '@aws-sdk/client-sts@3.556.0(@aws-sdk/credential-provider-node@3.564.0)': dependencies: '@aws-crypto/sha256-browser': 3.0.0 @@ -11505,50 +11886,6 @@ snapshots: - '@aws-sdk/client-sso-oidc' - aws-crt - '@aws-sdk/client-sts@3.985.0': - dependencies: - '@aws-crypto/sha256-browser': 5.2.0 - '@aws-crypto/sha256-js': 5.2.0 - '@aws-sdk/core': 3.973.7 - '@aws-sdk/credential-provider-node': 3.972.6 - '@aws-sdk/middleware-host-header': 3.972.3 - '@aws-sdk/middleware-logger': 3.972.3 - '@aws-sdk/middleware-recursion-detection': 3.972.3 - '@aws-sdk/middleware-user-agent': 3.972.7 - '@aws-sdk/region-config-resolver': 3.972.3 - '@aws-sdk/types': 3.973.1 - '@aws-sdk/util-endpoints': 3.985.0 - '@aws-sdk/util-user-agent-browser': 3.972.3 - '@aws-sdk/util-user-agent-node': 3.972.5 - '@smithy/config-resolver': 4.4.6 - '@smithy/core': 3.22.1 - '@smithy/fetch-http-handler': 5.3.9 - '@smithy/hash-node': 4.2.8 - '@smithy/invalid-dependency': 4.2.8 - '@smithy/middleware-content-length': 4.2.8 - '@smithy/middleware-endpoint': 4.4.13 - '@smithy/middleware-retry': 4.4.30 - '@smithy/middleware-serde': 4.2.9 - '@smithy/middleware-stack': 4.2.8 - '@smithy/node-config-provider': 4.3.8 - '@smithy/node-http-handler': 4.4.9 - '@smithy/protocol-http': 5.3.8 - '@smithy/smithy-client': 4.11.2 - '@smithy/types': 4.12.0 - '@smithy/url-parser': 4.2.8 - '@smithy/util-base64': 4.3.0 - '@smithy/util-body-length-browser': 4.2.0 - '@smithy/util-body-length-node': 4.2.1 - '@smithy/util-defaults-mode-browser': 4.3.29 - '@smithy/util-defaults-mode-node': 4.2.32 - '@smithy/util-endpoints': 3.2.8 - '@smithy/util-middleware': 4.2.8 - '@smithy/util-retry': 4.2.8 - '@smithy/util-utf8': 4.2.0 - tslib: 2.8.1 - transitivePeerDependencies: - - aws-crt - '@aws-sdk/core@3.556.0': dependencies: '@smithy/core': 1.4.2 @@ -11585,11 +11922,32 @@ snapshots: '@smithy/util-utf8': 4.2.0 tslib: 2.8.1 + '@aws-sdk/core@3.974.2': + dependencies: + '@aws-sdk/types': 3.973.8 + '@aws-sdk/xml-builder': 3.972.18 + '@smithy/core': 3.23.16 + '@smithy/node-config-provider': 4.3.14 + '@smithy/property-provider': 4.2.14 + '@smithy/protocol-http': 5.3.14 + '@smithy/signature-v4': 5.3.14 + '@smithy/smithy-client': 4.12.12 + '@smithy/types': 4.14.1 + '@smithy/util-base64': 4.3.2 + '@smithy/util-middleware': 4.2.14 + '@smithy/util-utf8': 4.2.2 + tslib: 2.8.1 + '@aws-sdk/crc64-nvme@3.972.0': dependencies: '@smithy/types': 4.12.0 tslib: 2.8.1 + '@aws-sdk/crc64-nvme@3.972.7': + dependencies: + '@smithy/types': 4.14.1 + tslib: 2.8.1 + '@aws-sdk/credential-provider-env@3.535.0': dependencies: '@aws-sdk/types': 3.535.0 @@ -11604,6 +11962,14 @@ snapshots: '@smithy/types': 2.12.0 tslib: 2.8.1 + '@aws-sdk/credential-provider-env@3.972.28': + dependencies: + '@aws-sdk/core': 3.974.2 + '@aws-sdk/types': 3.973.8 + '@smithy/property-provider': 4.2.14 + '@smithy/types': 4.14.1 + tslib: 2.8.1 + '@aws-sdk/credential-provider-env@3.972.5': dependencies: '@aws-sdk/core': 3.973.7 @@ -11636,6 +12002,19 @@ snapshots: '@smithy/util-stream': 2.2.0 tslib: 2.8.1 + '@aws-sdk/credential-provider-http@3.972.30': + dependencies: + '@aws-sdk/core': 3.974.2 + '@aws-sdk/types': 3.973.8 + '@smithy/fetch-http-handler': 5.3.17 + '@smithy/node-http-handler': 4.6.0 + '@smithy/property-provider': 4.2.14 + '@smithy/protocol-http': 5.3.14 + '@smithy/smithy-client': 4.12.12 + '@smithy/types': 4.14.1 + '@smithy/util-stream': 4.5.24 + tslib: 2.8.1 + '@aws-sdk/credential-provider-http@3.972.7': dependencies: '@aws-sdk/core': 3.973.7 @@ -11683,6 +12062,25 @@ snapshots: - '@aws-sdk/client-sso-oidc' - aws-crt + '@aws-sdk/credential-provider-ini@3.972.32': + dependencies: + '@aws-sdk/core': 3.974.2 + '@aws-sdk/credential-provider-env': 3.972.28 + '@aws-sdk/credential-provider-http': 3.972.30 + '@aws-sdk/credential-provider-login': 3.972.32 + '@aws-sdk/credential-provider-process': 3.972.28 + '@aws-sdk/credential-provider-sso': 3.972.32 + '@aws-sdk/credential-provider-web-identity': 3.972.32 + '@aws-sdk/nested-clients': 3.997.0 + '@aws-sdk/types': 3.973.8 + '@smithy/credential-provider-imds': 4.2.14 + '@smithy/property-provider': 4.2.14 + '@smithy/shared-ini-file-loader': 4.4.9 + '@smithy/types': 4.14.1 + tslib: 2.8.1 + transitivePeerDependencies: + - aws-crt + '@aws-sdk/credential-provider-ini@3.972.5': dependencies: '@aws-sdk/core': 3.973.7 @@ -11702,6 +12100,19 @@ snapshots: transitivePeerDependencies: - aws-crt + '@aws-sdk/credential-provider-login@3.972.32': + dependencies: + '@aws-sdk/core': 3.974.2 + '@aws-sdk/nested-clients': 3.997.0 + '@aws-sdk/types': 3.973.8 + '@smithy/property-provider': 4.2.14 + '@smithy/protocol-http': 5.3.14 + '@smithy/shared-ini-file-loader': 4.4.9 + '@smithy/types': 4.14.1 + tslib: 2.8.1 + transitivePeerDependencies: + - aws-crt + '@aws-sdk/credential-provider-login@3.972.5': dependencies: '@aws-sdk/core': 3.973.7 @@ -11751,6 +12162,23 @@ snapshots: - '@aws-sdk/client-sts' - aws-crt + '@aws-sdk/credential-provider-node@3.972.33': + dependencies: + '@aws-sdk/credential-provider-env': 3.972.28 + '@aws-sdk/credential-provider-http': 3.972.30 + '@aws-sdk/credential-provider-ini': 3.972.32 + '@aws-sdk/credential-provider-process': 3.972.28 + '@aws-sdk/credential-provider-sso': 3.972.32 + '@aws-sdk/credential-provider-web-identity': 3.972.32 + '@aws-sdk/types': 3.973.8 + '@smithy/credential-provider-imds': 4.2.14 + '@smithy/property-provider': 4.2.14 + '@smithy/shared-ini-file-loader': 4.4.9 + '@smithy/types': 4.14.1 + tslib: 2.8.1 + transitivePeerDependencies: + - aws-crt + '@aws-sdk/credential-provider-node@3.972.6': dependencies: '@aws-sdk/credential-provider-env': 3.972.5 @@ -11784,6 +12212,15 @@ snapshots: '@smithy/types': 2.12.0 tslib: 2.8.1 + '@aws-sdk/credential-provider-process@3.972.28': + dependencies: + '@aws-sdk/core': 3.974.2 + '@aws-sdk/types': 3.973.8 + '@smithy/property-provider': 4.2.14 + '@smithy/shared-ini-file-loader': 4.4.9 + '@smithy/types': 4.14.1 + tslib: 2.8.1 + '@aws-sdk/credential-provider-process@3.972.5': dependencies: '@aws-sdk/core': 3.973.7 @@ -11819,6 +12256,19 @@ snapshots: - '@aws-sdk/client-sso-oidc' - aws-crt + '@aws-sdk/credential-provider-sso@3.972.32': + dependencies: + '@aws-sdk/core': 3.974.2 + '@aws-sdk/nested-clients': 3.997.0 + '@aws-sdk/token-providers': 3.1033.0 + '@aws-sdk/types': 3.973.8 + '@smithy/property-provider': 4.2.14 + '@smithy/shared-ini-file-loader': 4.4.9 + '@smithy/types': 4.14.1 + tslib: 2.8.1 + transitivePeerDependencies: + - aws-crt + '@aws-sdk/credential-provider-sso@3.972.5': dependencies: '@aws-sdk/client-sso': 3.985.0 @@ -11851,6 +12301,18 @@ snapshots: '@smithy/types': 2.12.0 tslib: 2.8.1 + '@aws-sdk/credential-provider-web-identity@3.972.32': + dependencies: + '@aws-sdk/core': 3.974.2 + '@aws-sdk/nested-clients': 3.997.0 + '@aws-sdk/types': 3.973.8 + '@smithy/property-provider': 4.2.14 + '@smithy/shared-ini-file-loader': 4.4.9 + '@smithy/types': 4.14.1 + tslib: 2.8.1 + transitivePeerDependencies: + - aws-crt + '@aws-sdk/credential-provider-web-identity@3.972.5': dependencies: '@aws-sdk/core': 3.973.7 @@ -11863,14 +12325,14 @@ snapshots: transitivePeerDependencies: - aws-crt - '@aws-sdk/ec2-metadata-service@3.985.0': + '@aws-sdk/ec2-metadata-service@3.1033.0': dependencies: - '@aws-sdk/types': 3.973.1 - '@smithy/node-config-provider': 4.3.8 - '@smithy/node-http-handler': 4.4.9 - '@smithy/protocol-http': 5.3.8 - '@smithy/types': 4.12.0 - '@smithy/util-stream': 4.5.11 + '@aws-sdk/types': 3.973.8 + '@smithy/node-config-provider': 4.3.14 + '@smithy/node-http-handler': 4.6.0 + '@smithy/protocol-http': 5.3.14 + '@smithy/types': 4.14.1 + '@smithy/util-stream': 4.5.24 tslib: 2.8.1 '@aws-sdk/hash-node@3.374.0': @@ -11878,6 +12340,16 @@ snapshots: '@smithy/hash-node': 1.1.0 tslib: 2.6.2 + '@aws-sdk/middleware-bucket-endpoint@3.972.10': + dependencies: + '@aws-sdk/types': 3.973.8 + '@aws-sdk/util-arn-parser': 3.972.3 + '@smithy/node-config-provider': 4.3.14 + '@smithy/protocol-http': 5.3.14 + '@smithy/types': 4.14.1 + '@smithy/util-config-provider': 4.2.2 + tslib: 2.8.1 + '@aws-sdk/middleware-bucket-endpoint@3.972.3': dependencies: '@aws-sdk/types': 3.973.1 @@ -11888,6 +12360,13 @@ snapshots: '@smithy/util-config-provider': 4.2.0 tslib: 2.8.1 + '@aws-sdk/middleware-expect-continue@3.972.10': + dependencies: + '@aws-sdk/types': 3.973.8 + '@smithy/protocol-http': 5.3.14 + '@smithy/types': 4.14.1 + tslib: 2.8.1 + '@aws-sdk/middleware-expect-continue@3.972.3': dependencies: '@aws-sdk/types': 3.973.1 @@ -11912,6 +12391,23 @@ snapshots: '@smithy/util-utf8': 4.2.0 tslib: 2.8.1 + '@aws-sdk/middleware-flexible-checksums@3.974.10': + dependencies: + '@aws-crypto/crc32': 5.2.0 + '@aws-crypto/crc32c': 5.2.0 + '@aws-crypto/util': 5.2.0 + '@aws-sdk/core': 3.974.2 + '@aws-sdk/crc64-nvme': 3.972.7 + '@aws-sdk/types': 3.973.8 + '@smithy/is-array-buffer': 4.2.2 + '@smithy/node-config-provider': 4.3.14 + '@smithy/protocol-http': 5.3.14 + '@smithy/types': 4.14.1 + '@smithy/util-middleware': 4.2.14 + '@smithy/util-stream': 4.5.24 + '@smithy/util-utf8': 4.2.2 + tslib: 2.8.1 + '@aws-sdk/middleware-host-header@3.535.0': dependencies: '@aws-sdk/types': 3.535.0 @@ -11926,6 +12422,13 @@ snapshots: '@smithy/types': 2.12.0 tslib: 2.6.2 + '@aws-sdk/middleware-host-header@3.972.10': + dependencies: + '@aws-sdk/types': 3.973.8 + '@smithy/protocol-http': 5.3.14 + '@smithy/types': 4.14.1 + tslib: 2.8.1 + '@aws-sdk/middleware-host-header@3.972.3': dependencies: '@aws-sdk/types': 3.973.1 @@ -11933,6 +12436,12 @@ snapshots: '@smithy/types': 4.12.0 tslib: 2.8.1 + '@aws-sdk/middleware-location-constraint@3.972.10': + dependencies: + '@aws-sdk/types': 3.973.8 + '@smithy/types': 4.14.1 + tslib: 2.8.1 + '@aws-sdk/middleware-location-constraint@3.972.3': dependencies: '@aws-sdk/types': 3.973.1 @@ -11951,6 +12460,12 @@ snapshots: '@smithy/types': 2.12.0 tslib: 2.6.2 + '@aws-sdk/middleware-logger@3.972.10': + dependencies: + '@aws-sdk/types': 3.973.8 + '@smithy/types': 4.14.1 + tslib: 2.8.1 + '@aws-sdk/middleware-logger@3.972.3': dependencies: '@aws-sdk/types': 3.973.1 @@ -11971,6 +12486,14 @@ snapshots: '@smithy/types': 2.12.0 tslib: 2.6.2 + '@aws-sdk/middleware-recursion-detection@3.972.11': + dependencies: + '@aws-sdk/types': 3.973.8 + '@aws/lambda-invoke-store': 0.2.3 + '@smithy/protocol-http': 5.3.14 + '@smithy/types': 4.14.1 + tslib: 2.8.1 + '@aws-sdk/middleware-recursion-detection@3.972.3': dependencies: '@aws-sdk/types': 3.973.1 @@ -11991,6 +12514,23 @@ snapshots: '@smithy/util-config-provider': 2.3.0 tslib: 2.8.1 + '@aws-sdk/middleware-sdk-s3@3.972.31': + dependencies: + '@aws-sdk/core': 3.974.2 + '@aws-sdk/types': 3.973.8 + '@aws-sdk/util-arn-parser': 3.972.3 + '@smithy/core': 3.23.16 + '@smithy/node-config-provider': 4.3.14 + '@smithy/protocol-http': 5.3.14 + '@smithy/signature-v4': 5.3.14 + '@smithy/smithy-client': 4.12.12 + '@smithy/types': 4.14.1 + '@smithy/util-config-provider': 4.2.2 + '@smithy/util-middleware': 4.2.14 + '@smithy/util-stream': 4.5.24 + '@smithy/util-utf8': 4.2.2 + tslib: 2.8.1 + '@aws-sdk/middleware-sdk-s3@3.972.7': dependencies: '@aws-sdk/core': 3.973.7 @@ -12008,6 +12548,12 @@ snapshots: '@smithy/util-utf8': 4.2.0 tslib: 2.8.1 + '@aws-sdk/middleware-ssec@3.972.10': + dependencies: + '@aws-sdk/types': 3.973.8 + '@smithy/types': 4.14.1 + tslib: 2.8.1 + '@aws-sdk/middleware-ssec@3.972.3': dependencies: '@aws-sdk/types': 3.973.1 @@ -12030,6 +12576,17 @@ snapshots: '@smithy/types': 2.12.0 tslib: 2.6.2 + '@aws-sdk/middleware-user-agent@3.972.32': + dependencies: + '@aws-sdk/core': 3.974.2 + '@aws-sdk/types': 3.973.8 + '@aws-sdk/util-endpoints': 3.996.7 + '@smithy/core': 3.23.16 + '@smithy/protocol-http': 5.3.14 + '@smithy/types': 4.14.1 + '@smithy/util-retry': 4.3.3 + tslib: 2.8.1 + '@aws-sdk/middleware-user-agent@3.972.7': dependencies: '@aws-sdk/core': 3.973.7 @@ -12083,6 +12640,50 @@ snapshots: transitivePeerDependencies: - aws-crt + '@aws-sdk/nested-clients@3.997.0': + dependencies: + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/core': 3.974.2 + '@aws-sdk/middleware-host-header': 3.972.10 + '@aws-sdk/middleware-logger': 3.972.10 + '@aws-sdk/middleware-recursion-detection': 3.972.11 + '@aws-sdk/middleware-user-agent': 3.972.32 + '@aws-sdk/region-config-resolver': 3.972.12 + '@aws-sdk/signature-v4-multi-region': 3.996.19 + '@aws-sdk/types': 3.973.8 + '@aws-sdk/util-endpoints': 3.996.7 + '@aws-sdk/util-user-agent-browser': 3.972.10 + '@aws-sdk/util-user-agent-node': 3.973.18 + '@smithy/config-resolver': 4.4.17 + '@smithy/core': 3.23.16 + '@smithy/fetch-http-handler': 5.3.17 + '@smithy/hash-node': 4.2.14 + '@smithy/invalid-dependency': 4.2.14 + '@smithy/middleware-content-length': 4.2.14 + '@smithy/middleware-endpoint': 4.4.31 + '@smithy/middleware-retry': 4.5.4 + '@smithy/middleware-serde': 4.2.19 + '@smithy/middleware-stack': 4.2.14 + '@smithy/node-config-provider': 4.3.14 + '@smithy/node-http-handler': 4.6.0 + '@smithy/protocol-http': 5.3.14 + '@smithy/smithy-client': 4.12.12 + '@smithy/types': 4.14.1 + '@smithy/url-parser': 4.2.14 + '@smithy/util-base64': 4.3.2 + '@smithy/util-body-length-browser': 4.2.2 + '@smithy/util-body-length-node': 4.2.3 + '@smithy/util-defaults-mode-browser': 4.3.48 + '@smithy/util-defaults-mode-node': 4.2.53 + '@smithy/util-endpoints': 3.4.2 + '@smithy/util-middleware': 4.2.14 + '@smithy/util-retry': 4.3.3 + '@smithy/util-utf8': 4.2.2 + tslib: 2.8.1 + transitivePeerDependencies: + - aws-crt + '@aws-sdk/protocol-http@3.374.0': dependencies: '@smithy/protocol-http': 1.2.0 @@ -12106,6 +12707,14 @@ snapshots: '@smithy/util-middleware': 2.2.0 tslib: 2.6.2 + '@aws-sdk/region-config-resolver@3.972.12': + dependencies: + '@aws-sdk/types': 3.973.8 + '@smithy/config-resolver': 4.4.17 + '@smithy/node-config-provider': 4.3.14 + '@smithy/types': 4.14.1 + tslib: 2.8.1 + '@aws-sdk/region-config-resolver@3.972.3': dependencies: '@aws-sdk/types': 3.973.1 @@ -12143,6 +12752,27 @@ snapshots: '@smithy/types': 4.12.0 tslib: 2.8.1 + '@aws-sdk/signature-v4-multi-region@3.996.19': + dependencies: + '@aws-sdk/middleware-sdk-s3': 3.972.31 + '@aws-sdk/types': 3.973.8 + '@smithy/protocol-http': 5.3.14 + '@smithy/signature-v4': 5.3.14 + '@smithy/types': 4.14.1 + tslib: 2.8.1 + + '@aws-sdk/token-providers@3.1033.0': + dependencies: + '@aws-sdk/core': 3.974.2 + '@aws-sdk/nested-clients': 3.997.0 + '@aws-sdk/types': 3.973.8 + '@smithy/property-provider': 4.2.14 + '@smithy/shared-ini-file-loader': 4.4.9 + '@smithy/types': 4.14.1 + tslib: 2.8.1 + transitivePeerDependencies: + - aws-crt + '@aws-sdk/token-providers@3.564.0(@aws-sdk/credential-provider-node@3.564.0)': dependencies: '@aws-sdk/client-sso-oidc': 3.564.0(@aws-sdk/credential-provider-node@3.564.0) @@ -12191,6 +12821,11 @@ snapshots: '@smithy/types': 4.12.0 tslib: 2.8.1 + '@aws-sdk/types@3.973.8': + dependencies: + '@smithy/types': 4.14.1 + tslib: 2.8.1 + '@aws-sdk/url-parser@3.374.0': dependencies: '@smithy/url-parser': 1.1.0 @@ -12204,6 +12839,10 @@ snapshots: dependencies: tslib: 2.8.1 + '@aws-sdk/util-arn-parser@3.972.3': + dependencies: + tslib: 2.8.1 + '@aws-sdk/util-endpoints@3.540.0': dependencies: '@aws-sdk/types': 3.535.0 @@ -12226,6 +12865,14 @@ snapshots: '@smithy/util-endpoints': 3.2.8 tslib: 2.8.1 + '@aws-sdk/util-endpoints@3.996.7': + dependencies: + '@aws-sdk/types': 3.973.8 + '@smithy/types': 4.14.1 + '@smithy/url-parser': 4.2.14 + '@smithy/util-endpoints': 3.4.2 + tslib: 2.8.1 + '@aws-sdk/util-format-url@3.535.0': dependencies: '@aws-sdk/types': 3.535.0 @@ -12251,6 +12898,13 @@ snapshots: bowser: 2.11.0 tslib: 2.6.2 + '@aws-sdk/util-user-agent-browser@3.972.10': + dependencies: + '@aws-sdk/types': 3.973.8 + '@smithy/types': 4.14.1 + bowser: 2.11.0 + tslib: 2.8.1 + '@aws-sdk/util-user-agent-browser@3.972.3': dependencies: '@aws-sdk/types': 3.973.1 @@ -12280,10 +12934,25 @@ snapshots: '@smithy/types': 4.12.0 tslib: 2.8.1 + '@aws-sdk/util-user-agent-node@3.973.18': + dependencies: + '@aws-sdk/middleware-user-agent': 3.972.32 + '@aws-sdk/types': 3.973.8 + '@smithy/node-config-provider': 4.3.14 + '@smithy/types': 4.14.1 + '@smithy/util-config-provider': 4.2.2 + tslib: 2.8.1 + '@aws-sdk/util-utf8-browser@3.259.0': dependencies: tslib: 2.8.1 + '@aws-sdk/xml-builder@3.972.18': + dependencies: + '@smithy/types': 4.14.1 + fast-xml-parser: 5.5.8 + tslib: 2.8.1 + '@aws-sdk/xml-builder@3.972.4': dependencies: '@smithy/types': 4.12.0 @@ -12304,12 +12973,6 @@ snapshots: transitivePeerDependencies: - supports-color - '@azure/core-auth@1.9.0': - dependencies: - '@azure/abort-controller': 2.1.2 - '@azure/core-util': 1.11.0 - tslib: 2.8.1 - '@azure/core-client@1.10.1': dependencies: '@azure/abort-controller': 2.1.2 @@ -12331,9 +12994,11 @@ snapshots: '@azure/core-lro@2.7.2': dependencies: '@azure/abort-controller': 2.1.2 - '@azure/core-util': 1.11.0 - '@azure/logger': 1.1.4 + '@azure/core-util': 1.13.1 + '@azure/logger': 1.3.0 tslib: 2.8.1 + transitivePeerDependencies: + - supports-color '@azure/core-paging@1.6.2': dependencies: @@ -12355,11 +13020,6 @@ snapshots: dependencies: tslib: 2.8.1 - '@azure/core-util@1.11.0': - dependencies: - '@azure/abort-controller': 2.1.2 - tslib: 2.8.1 - '@azure/core-util@1.13.1': dependencies: '@azure/abort-controller': 2.1.2 @@ -12370,18 +13030,18 @@ snapshots: '@azure/core-xml@1.5.0': dependencies: - fast-xml-parser: 5.3.5 + fast-xml-parser: 5.8.0 tslib: 2.8.1 '@azure/identity@4.13.0': dependencies: '@azure/abort-controller': 2.1.2 - '@azure/core-auth': 1.9.0 + '@azure/core-auth': 1.10.1 '@azure/core-client': 1.10.1 '@azure/core-rest-pipeline': 1.22.2 '@azure/core-tracing': 1.3.1 - '@azure/core-util': 1.11.0 - '@azure/logger': 1.1.4 + '@azure/core-util': 1.13.1 + '@azure/logger': 1.3.0 '@azure/msal-browser': 4.28.1 '@azure/msal-node': 3.8.6 open: 10.2.0 @@ -12389,10 +13049,6 @@ snapshots: transitivePeerDependencies: - supports-color - '@azure/logger@1.1.4': - dependencies: - tslib: 2.8.1 - '@azure/logger@1.3.0': dependencies: '@typespec/ts-http-runtime': 0.3.3 @@ -12415,16 +13071,16 @@ snapshots: '@azure/storage-blob@12.26.0': dependencies: '@azure/abort-controller': 2.1.2 - '@azure/core-auth': 1.9.0 + '@azure/core-auth': 1.10.1 '@azure/core-client': 1.10.1 '@azure/core-http-compat': 2.3.2(@azure/core-client@1.10.1)(@azure/core-rest-pipeline@1.22.2) '@azure/core-lro': 2.7.2 '@azure/core-paging': 1.6.2 '@azure/core-rest-pipeline': 1.22.2 '@azure/core-tracing': 1.3.1 - '@azure/core-util': 1.11.0 + '@azure/core-util': 1.13.1 '@azure/core-xml': 1.5.0 - '@azure/logger': 1.1.4 + '@azure/logger': 1.3.0 events: 3.3.0 tslib: 2.8.1 transitivePeerDependencies: @@ -12524,11 +13180,11 @@ snapshots: '@colors/colors@1.6.0': {} - '@commitlint/cli@19.8.0(@types/node@22.19.10)(typescript@5.6.3)': + '@commitlint/cli@19.8.0(@types/node@24.13.3)(typescript@5.6.3)': dependencies: '@commitlint/format': 19.8.0 '@commitlint/lint': 19.8.0 - '@commitlint/load': 19.8.0(@types/node@22.19.10)(typescript@5.6.3) + '@commitlint/load': 19.8.0(@types/node@24.13.3)(typescript@5.6.3) '@commitlint/read': 19.8.0 '@commitlint/types': 19.8.0 tinyexec: 0.3.2 @@ -12575,7 +13231,7 @@ snapshots: '@commitlint/rules': 19.8.0 '@commitlint/types': 19.8.0 - '@commitlint/load@19.8.0(@types/node@22.19.10)(typescript@5.6.3)': + '@commitlint/load@19.8.0(@types/node@24.13.3)(typescript@5.6.3)': dependencies: '@commitlint/config-validator': 19.8.0 '@commitlint/execute-rule': 19.8.0 @@ -12583,7 +13239,7 @@ snapshots: '@commitlint/types': 19.8.0 chalk: 5.4.1 cosmiconfig: 9.0.0(typescript@5.6.3) - cosmiconfig-typescript-loader: 6.1.0(@types/node@22.19.10)(cosmiconfig@9.0.0(typescript@5.6.3))(typescript@5.6.3) + cosmiconfig-typescript-loader: 6.1.0(@types/node@24.13.3)(cosmiconfig@9.0.0(typescript@5.6.3))(typescript@5.6.3) lodash.isplainobject: 4.0.6 lodash.merge: 4.6.2 lodash.uniq: 4.5.0 @@ -12644,30 +13300,32 @@ snapshots: enabled: 2.0.0 kuler: 2.0.0 - '@datadog/native-appsec@7.1.1': + '@datadog/libdatadog@0.3.0': {} + + '@datadog/native-appsec@8.4.0': dependencies: node-gyp-build: 3.9.0 - '@datadog/native-iast-rewriter@2.3.1': + '@datadog/native-iast-rewriter@2.6.1': dependencies: lru-cache: 7.18.3 - node-gyp-build: 4.8.0 + node-gyp-build: 4.8.4 - '@datadog/native-iast-taint-tracking@2.1.0': + '@datadog/native-iast-taint-tracking@3.2.0': dependencies: node-gyp-build: 3.9.0 - '@datadog/native-metrics@2.0.0': + '@datadog/native-metrics@3.1.1': dependencies: node-addon-api: 6.1.0 node-gyp-build: 3.9.0 - '@datadog/pprof@5.3.0': + '@datadog/pprof@5.4.1': dependencies: delay: 5.0.0 node-gyp-build: 3.9.0 p-limit: 3.1.0 - pprof-format: 2.1.0 + pprof-format: 2.2.1 source-map: 0.7.4 '@datadog/sketches-js@2.1.1': {} @@ -12692,153 +13350,162 @@ snapshots: - bufferutil - utf-8-validate - '@esbuild/aix-ppc64@0.19.12': - optional: true - '@esbuild/aix-ppc64@0.25.12': optional: true - '@esbuild/android-arm64@0.19.12': + '@esbuild/aix-ppc64@0.28.0': optional: true '@esbuild/android-arm64@0.25.12': optional: true - '@esbuild/android-arm@0.19.12': + '@esbuild/android-arm64@0.28.0': optional: true '@esbuild/android-arm@0.25.12': optional: true - '@esbuild/android-x64@0.19.12': + '@esbuild/android-arm@0.28.0': optional: true '@esbuild/android-x64@0.25.12': optional: true - '@esbuild/darwin-arm64@0.19.12': + '@esbuild/android-x64@0.28.0': optional: true '@esbuild/darwin-arm64@0.25.12': optional: true - '@esbuild/darwin-x64@0.19.12': + '@esbuild/darwin-arm64@0.28.0': optional: true '@esbuild/darwin-x64@0.25.12': optional: true - '@esbuild/freebsd-arm64@0.19.12': + '@esbuild/darwin-x64@0.28.0': optional: true '@esbuild/freebsd-arm64@0.25.12': optional: true - '@esbuild/freebsd-x64@0.19.12': + '@esbuild/freebsd-arm64@0.28.0': optional: true '@esbuild/freebsd-x64@0.25.12': optional: true - '@esbuild/linux-arm64@0.19.12': + '@esbuild/freebsd-x64@0.28.0': optional: true '@esbuild/linux-arm64@0.25.12': optional: true - '@esbuild/linux-arm@0.19.12': + '@esbuild/linux-arm64@0.28.0': optional: true '@esbuild/linux-arm@0.25.12': optional: true - '@esbuild/linux-ia32@0.19.12': + '@esbuild/linux-arm@0.28.0': optional: true '@esbuild/linux-ia32@0.25.12': optional: true - '@esbuild/linux-loong64@0.19.12': + '@esbuild/linux-ia32@0.28.0': optional: true '@esbuild/linux-loong64@0.25.12': optional: true - '@esbuild/linux-mips64el@0.19.12': + '@esbuild/linux-loong64@0.28.0': optional: true '@esbuild/linux-mips64el@0.25.12': optional: true - '@esbuild/linux-ppc64@0.19.12': + '@esbuild/linux-mips64el@0.28.0': optional: true '@esbuild/linux-ppc64@0.25.12': optional: true - '@esbuild/linux-riscv64@0.19.12': + '@esbuild/linux-ppc64@0.28.0': optional: true '@esbuild/linux-riscv64@0.25.12': optional: true - '@esbuild/linux-s390x@0.19.12': + '@esbuild/linux-riscv64@0.28.0': optional: true '@esbuild/linux-s390x@0.25.12': optional: true - '@esbuild/linux-x64@0.19.12': + '@esbuild/linux-s390x@0.28.0': optional: true '@esbuild/linux-x64@0.25.12': optional: true + '@esbuild/linux-x64@0.28.0': + optional: true + '@esbuild/netbsd-arm64@0.25.12': optional: true - '@esbuild/netbsd-x64@0.19.12': + '@esbuild/netbsd-arm64@0.28.0': optional: true '@esbuild/netbsd-x64@0.25.12': optional: true + '@esbuild/netbsd-x64@0.28.0': + optional: true + '@esbuild/openbsd-arm64@0.25.12': optional: true - '@esbuild/openbsd-x64@0.19.12': + '@esbuild/openbsd-arm64@0.28.0': optional: true '@esbuild/openbsd-x64@0.25.12': optional: true + '@esbuild/openbsd-x64@0.28.0': + optional: true + '@esbuild/openharmony-arm64@0.25.12': optional: true - '@esbuild/sunos-x64@0.19.12': + '@esbuild/openharmony-arm64@0.28.0': optional: true '@esbuild/sunos-x64@0.25.12': optional: true - '@esbuild/win32-arm64@0.19.12': + '@esbuild/sunos-x64@0.28.0': optional: true '@esbuild/win32-arm64@0.25.12': optional: true - '@esbuild/win32-ia32@0.19.12': + '@esbuild/win32-arm64@0.28.0': optional: true '@esbuild/win32-ia32@0.25.12': optional: true - '@esbuild/win32-x64@0.19.12': + '@esbuild/win32-ia32@0.28.0': optional: true '@esbuild/win32-x64@0.25.12': optional: true + '@esbuild/win32-x64@0.28.0': + optional: true + '@eslint-community/eslint-utils@4.4.0(eslint@8.57.0)': dependencies: eslint: 8.57.0 @@ -12871,13 +13538,13 @@ snapshots: '@gitbeaker/core@43.5.0': dependencies: '@gitbeaker/requester-utils': 43.5.0 - qs: 6.14.0 + qs: 6.15.3 xcase: 2.0.1 '@gitbeaker/requester-utils@43.5.0': dependencies: picomatch-browser: 2.2.6 - qs: 6.14.0 + qs: 6.15.3 rate-limiter-flexible: 7.4.0 xcase: 2.0.1 @@ -13046,6 +13713,8 @@ snapshots: dependencies: minipass: 7.1.3 + '@isaacs/ttlcache@1.4.1': {} + '@jridgewell/gen-mapping@0.3.5': dependencies: '@jridgewell/set-array': 1.2.1 @@ -13142,6 +13811,8 @@ snapshots: transitivePeerDependencies: - debug + '@noble/hashes@1.8.0': {} + '@nodable/entities@2.1.0': {} '@nodelib/fs.scandir@2.1.5': @@ -13464,12 +14135,16 @@ snapshots: '@opentelemetry/api@1.6.0': {} - '@opentelemetry/core@1.24.0(@opentelemetry/api@1.6.0)': + '@opentelemetry/core@1.30.1(@opentelemetry/api@1.6.0)': dependencies: '@opentelemetry/api': 1.6.0 - '@opentelemetry/semantic-conventions': 1.24.0 + '@opentelemetry/semantic-conventions': 1.28.0 - '@opentelemetry/semantic-conventions@1.24.0': {} + '@opentelemetry/semantic-conventions@1.28.0': {} + + '@paralleldrive/cuid2@2.3.1': + dependencies: + '@noble/hashes': 1.8.0 '@pkgjs/parseargs@0.11.0': optional: true @@ -13660,7 +14335,7 @@ snapshots: '@sendgrid/client@8.1.3': dependencies: '@sendgrid/helpers': 8.0.0 - axios: 1.16.1 + axios: 1.19.0 transitivePeerDependencies: - debug - supports-color @@ -13681,21 +14356,21 @@ snapshots: '@slack/logger@3.0.0': dependencies: - '@types/node': 20.12.7 + '@types/node': 20.19.43 '@slack/types@1.10.0': {} - '@slack/types@2.11.0': {} + '@slack/types@2.20.1': {} - '@slack/web-api@6.12.0': + '@slack/web-api@6.13.0': dependencies: '@slack/logger': 3.0.0 - '@slack/types': 2.11.0 + '@slack/types': 2.20.1 '@types/is-stream': 1.1.0 - '@types/node': 20.12.7 - axios: 1.16.1 + '@types/node': 20.19.43 + axios: 1.19.0 eventemitter3: 3.1.2 - form-data: 2.5.1 + form-data: 2.5.6 is-electron: 2.2.2 is-stream: 1.1.0 p-queue: 6.6.2 @@ -13707,7 +14382,7 @@ snapshots: '@slack/webhook@6.1.0': dependencies: '@slack/types': 1.10.0 - '@types/node': 20.12.7 + '@types/node': 20.19.43 axios: 0.21.4 transitivePeerDependencies: - debug @@ -13727,10 +14402,19 @@ snapshots: '@smithy/util-base64': 4.3.0 tslib: 2.8.1 + '@smithy/chunked-blob-reader-native@4.2.3': + dependencies: + '@smithy/util-base64': 4.3.2 + tslib: 2.8.1 + '@smithy/chunked-blob-reader@5.2.0': dependencies: tslib: 2.8.1 + '@smithy/chunked-blob-reader@5.2.2': + dependencies: + tslib: 2.8.1 + '@smithy/config-resolver@2.2.0': dependencies: '@smithy/node-config-provider': 2.3.0 @@ -13739,6 +14423,15 @@ snapshots: '@smithy/util-middleware': 2.2.0 tslib: 2.6.2 + '@smithy/config-resolver@4.4.17': + dependencies: + '@smithy/node-config-provider': 4.3.14 + '@smithy/types': 4.14.1 + '@smithy/util-config-provider': 4.2.2 + '@smithy/util-endpoints': 3.4.2 + '@smithy/util-middleware': 4.2.14 + tslib: 2.8.1 + '@smithy/config-resolver@4.4.6': dependencies: '@smithy/node-config-provider': 4.3.8 @@ -13772,6 +14465,19 @@ snapshots: '@smithy/uuid': 1.1.0 tslib: 2.8.1 + '@smithy/core@3.23.16': + dependencies: + '@smithy/protocol-http': 5.3.14 + '@smithy/types': 4.14.1 + '@smithy/url-parser': 4.2.14 + '@smithy/util-base64': 4.3.2 + '@smithy/util-body-length-browser': 4.2.2 + '@smithy/util-middleware': 4.2.14 + '@smithy/util-stream': 4.5.24 + '@smithy/util-utf8': 4.2.2 + '@smithy/uuid': 1.1.2 + tslib: 2.8.1 + '@smithy/credential-provider-imds@2.3.0': dependencies: '@smithy/node-config-provider': 2.3.0 @@ -13780,6 +14486,14 @@ snapshots: '@smithy/url-parser': 2.2.0 tslib: 2.8.1 + '@smithy/credential-provider-imds@4.2.14': + dependencies: + '@smithy/node-config-provider': 4.3.14 + '@smithy/property-provider': 4.2.14 + '@smithy/types': 4.14.1 + '@smithy/url-parser': 4.2.14 + tslib: 2.8.1 + '@smithy/credential-provider-imds@4.2.8': dependencies: '@smithy/node-config-provider': 4.3.8 @@ -13795,6 +14509,13 @@ snapshots: '@smithy/util-hex-encoding': 2.2.0 tslib: 2.8.1 + '@smithy/eventstream-codec@4.2.14': + dependencies: + '@aws-crypto/crc32': 5.2.0 + '@smithy/types': 4.14.1 + '@smithy/util-hex-encoding': 4.2.2 + tslib: 2.8.1 + '@smithy/eventstream-codec@4.2.8': dependencies: '@aws-crypto/crc32': 5.2.0 @@ -13808,6 +14529,12 @@ snapshots: '@smithy/types': 2.12.0 tslib: 2.6.2 + '@smithy/eventstream-serde-browser@4.2.14': + dependencies: + '@smithy/eventstream-serde-universal': 4.2.14 + '@smithy/types': 4.14.1 + tslib: 2.8.1 + '@smithy/eventstream-serde-browser@4.2.8': dependencies: '@smithy/eventstream-serde-universal': 4.2.8 @@ -13819,6 +14546,11 @@ snapshots: '@smithy/types': 2.12.0 tslib: 2.6.2 + '@smithy/eventstream-serde-config-resolver@4.3.14': + dependencies: + '@smithy/types': 4.14.1 + tslib: 2.8.1 + '@smithy/eventstream-serde-config-resolver@4.3.8': dependencies: '@smithy/types': 4.12.0 @@ -13830,6 +14562,12 @@ snapshots: '@smithy/types': 2.12.0 tslib: 2.6.2 + '@smithy/eventstream-serde-node@4.2.14': + dependencies: + '@smithy/eventstream-serde-universal': 4.2.14 + '@smithy/types': 4.14.1 + tslib: 2.8.1 + '@smithy/eventstream-serde-node@4.2.8': dependencies: '@smithy/eventstream-serde-universal': 4.2.8 @@ -13842,6 +14580,12 @@ snapshots: '@smithy/types': 2.12.0 tslib: 2.8.1 + '@smithy/eventstream-serde-universal@4.2.14': + dependencies: + '@smithy/eventstream-codec': 4.2.14 + '@smithy/types': 4.14.1 + tslib: 2.8.1 + '@smithy/eventstream-serde-universal@4.2.8': dependencies: '@smithy/eventstream-codec': 4.2.8 @@ -13856,6 +14600,14 @@ snapshots: '@smithy/util-base64': 2.3.0 tslib: 2.6.2 + '@smithy/fetch-http-handler@5.3.17': + dependencies: + '@smithy/protocol-http': 5.3.14 + '@smithy/querystring-builder': 4.2.14 + '@smithy/types': 4.14.1 + '@smithy/util-base64': 4.3.2 + tslib: 2.8.1 + '@smithy/fetch-http-handler@5.3.9': dependencies: '@smithy/protocol-http': 5.3.8 @@ -13864,6 +14616,13 @@ snapshots: '@smithy/util-base64': 4.3.0 tslib: 2.8.1 + '@smithy/hash-blob-browser@4.2.15': + dependencies: + '@smithy/chunked-blob-reader': 5.2.2 + '@smithy/chunked-blob-reader-native': 4.2.3 + '@smithy/types': 4.14.1 + tslib: 2.8.1 + '@smithy/hash-blob-browser@4.2.9': dependencies: '@smithy/chunked-blob-reader': 5.2.0 @@ -13885,6 +14644,13 @@ snapshots: '@smithy/util-utf8': 2.3.0 tslib: 2.6.2 + '@smithy/hash-node@4.2.14': + dependencies: + '@smithy/types': 4.14.1 + '@smithy/util-buffer-from': 4.2.2 + '@smithy/util-utf8': 4.2.2 + tslib: 2.8.1 + '@smithy/hash-node@4.2.8': dependencies: '@smithy/types': 4.12.0 @@ -13892,6 +14658,12 @@ snapshots: '@smithy/util-utf8': 4.2.0 tslib: 2.8.1 + '@smithy/hash-stream-node@4.2.14': + dependencies: + '@smithy/types': 4.14.1 + '@smithy/util-utf8': 4.2.2 + tslib: 2.8.1 + '@smithy/hash-stream-node@4.2.8': dependencies: '@smithy/types': 4.12.0 @@ -13903,6 +14675,11 @@ snapshots: '@smithy/types': 2.12.0 tslib: 2.6.2 + '@smithy/invalid-dependency@4.2.14': + dependencies: + '@smithy/types': 4.14.1 + tslib: 2.8.1 + '@smithy/invalid-dependency@4.2.8': dependencies: '@smithy/types': 4.12.0 @@ -13920,6 +14697,16 @@ snapshots: dependencies: tslib: 2.8.1 + '@smithy/is-array-buffer@4.2.2': + dependencies: + tslib: 2.8.1 + + '@smithy/md5-js@4.2.14': + dependencies: + '@smithy/types': 4.14.1 + '@smithy/util-utf8': 4.2.2 + tslib: 2.8.1 + '@smithy/md5-js@4.2.8': dependencies: '@smithy/types': 4.12.0 @@ -13932,6 +14719,12 @@ snapshots: '@smithy/types': 2.12.0 tslib: 2.6.2 + '@smithy/middleware-content-length@4.2.14': + dependencies: + '@smithy/protocol-http': 5.3.14 + '@smithy/types': 4.14.1 + tslib: 2.8.1 + '@smithy/middleware-content-length@4.2.8': dependencies: '@smithy/protocol-http': 5.3.8 @@ -13959,6 +14752,17 @@ snapshots: '@smithy/util-middleware': 4.2.8 tslib: 2.8.1 + '@smithy/middleware-endpoint@4.4.31': + dependencies: + '@smithy/core': 3.23.16 + '@smithy/middleware-serde': 4.2.19 + '@smithy/node-config-provider': 4.3.14 + '@smithy/shared-ini-file-loader': 4.4.9 + '@smithy/types': 4.14.1 + '@smithy/url-parser': 4.2.14 + '@smithy/util-middleware': 4.2.14 + tslib: 2.8.1 + '@smithy/middleware-retry@2.3.1': dependencies: '@smithy/node-config-provider': 2.3.0 @@ -13983,11 +14787,31 @@ snapshots: '@smithy/uuid': 1.1.0 tslib: 2.8.1 + '@smithy/middleware-retry@4.5.4': + dependencies: + '@smithy/core': 3.23.16 + '@smithy/node-config-provider': 4.3.14 + '@smithy/protocol-http': 5.3.14 + '@smithy/service-error-classification': 4.3.0 + '@smithy/smithy-client': 4.12.12 + '@smithy/types': 4.14.1 + '@smithy/util-middleware': 4.2.14 + '@smithy/util-retry': 4.3.3 + '@smithy/uuid': 1.1.2 + tslib: 2.8.1 + '@smithy/middleware-serde@2.3.0': dependencies: '@smithy/types': 2.12.0 tslib: 2.6.2 + '@smithy/middleware-serde@4.2.19': + dependencies: + '@smithy/core': 3.23.16 + '@smithy/protocol-http': 5.3.14 + '@smithy/types': 4.14.1 + tslib: 2.8.1 + '@smithy/middleware-serde@4.2.9': dependencies: '@smithy/protocol-http': 5.3.8 @@ -13999,6 +14823,11 @@ snapshots: '@smithy/types': 2.12.0 tslib: 2.6.2 + '@smithy/middleware-stack@4.2.14': + dependencies: + '@smithy/types': 4.14.1 + tslib: 2.8.1 + '@smithy/middleware-stack@4.2.8': dependencies: '@smithy/types': 4.12.0 @@ -14011,6 +14840,13 @@ snapshots: '@smithy/types': 2.12.0 tslib: 2.6.2 + '@smithy/node-config-provider@4.3.14': + dependencies: + '@smithy/property-provider': 4.2.14 + '@smithy/shared-ini-file-loader': 4.4.9 + '@smithy/types': 4.14.1 + tslib: 2.8.1 + '@smithy/node-config-provider@4.3.8': dependencies: '@smithy/property-provider': 4.2.8 @@ -14034,11 +14870,23 @@ snapshots: '@smithy/types': 4.12.0 tslib: 2.8.1 + '@smithy/node-http-handler@4.6.0': + dependencies: + '@smithy/protocol-http': 5.3.14 + '@smithy/querystring-builder': 4.2.14 + '@smithy/types': 4.14.1 + tslib: 2.8.1 + '@smithy/property-provider@2.2.0': dependencies: '@smithy/types': 2.12.0 tslib: 2.8.1 + '@smithy/property-provider@4.2.14': + dependencies: + '@smithy/types': 4.14.1 + tslib: 2.8.1 + '@smithy/property-provider@4.2.8': dependencies: '@smithy/types': 4.12.0 @@ -14054,6 +14902,11 @@ snapshots: '@smithy/types': 2.12.0 tslib: 2.6.2 + '@smithy/protocol-http@5.3.14': + dependencies: + '@smithy/types': 4.14.1 + tslib: 2.8.1 + '@smithy/protocol-http@5.3.8': dependencies: '@smithy/types': 4.12.0 @@ -14065,6 +14918,12 @@ snapshots: '@smithy/util-uri-escape': 2.2.0 tslib: 2.6.2 + '@smithy/querystring-builder@4.2.14': + dependencies: + '@smithy/types': 4.14.1 + '@smithy/util-uri-escape': 4.2.2 + tslib: 2.8.1 + '@smithy/querystring-builder@4.2.8': dependencies: '@smithy/types': 4.12.0 @@ -14081,6 +14940,11 @@ snapshots: '@smithy/types': 2.12.0 tslib: 2.8.1 + '@smithy/querystring-parser@4.2.14': + dependencies: + '@smithy/types': 4.14.1 + tslib: 2.8.1 + '@smithy/querystring-parser@4.2.8': dependencies: '@smithy/types': 4.12.0 @@ -14094,6 +14958,10 @@ snapshots: dependencies: '@smithy/types': 4.12.0 + '@smithy/service-error-classification@4.3.0': + dependencies: + '@smithy/types': 4.14.1 + '@smithy/shared-ini-file-loader@2.4.0': dependencies: '@smithy/types': 2.12.0 @@ -14104,6 +14972,11 @@ snapshots: '@smithy/types': 4.12.0 tslib: 2.8.1 + '@smithy/shared-ini-file-loader@4.4.9': + dependencies: + '@smithy/types': 4.14.1 + tslib: 2.8.1 + '@smithy/signature-v4@2.3.0': dependencies: '@smithy/is-array-buffer': 2.2.0 @@ -14114,6 +14987,17 @@ snapshots: '@smithy/util-utf8': 2.3.0 tslib: 2.8.1 + '@smithy/signature-v4@5.3.14': + dependencies: + '@smithy/is-array-buffer': 4.2.2 + '@smithy/protocol-http': 5.3.14 + '@smithy/types': 4.14.1 + '@smithy/util-hex-encoding': 4.2.2 + '@smithy/util-middleware': 4.2.14 + '@smithy/util-uri-escape': 4.2.2 + '@smithy/util-utf8': 4.2.2 + tslib: 2.8.1 + '@smithy/signature-v4@5.3.8': dependencies: '@smithy/is-array-buffer': 4.2.0 @@ -14144,6 +15028,16 @@ snapshots: '@smithy/util-stream': 4.5.11 tslib: 2.8.1 + '@smithy/smithy-client@4.12.12': + dependencies: + '@smithy/core': 3.23.16 + '@smithy/middleware-endpoint': 4.4.31 + '@smithy/middleware-stack': 4.2.14 + '@smithy/protocol-http': 5.3.14 + '@smithy/types': 4.14.1 + '@smithy/util-stream': 4.5.24 + tslib: 2.8.1 + '@smithy/types@1.2.0': dependencies: tslib: 2.8.1 @@ -14156,6 +15050,10 @@ snapshots: dependencies: tslib: 2.8.1 + '@smithy/types@4.14.1': + dependencies: + tslib: 2.8.1 + '@smithy/url-parser@1.1.0': dependencies: '@smithy/querystring-parser': 1.1.0 @@ -14168,6 +15066,12 @@ snapshots: '@smithy/types': 2.12.0 tslib: 2.6.2 + '@smithy/url-parser@4.2.14': + dependencies: + '@smithy/querystring-parser': 4.2.14 + '@smithy/types': 4.14.1 + tslib: 2.8.1 + '@smithy/url-parser@4.2.8': dependencies: '@smithy/querystring-parser': 4.2.8 @@ -14186,6 +15090,12 @@ snapshots: '@smithy/util-utf8': 4.2.0 tslib: 2.8.1 + '@smithy/util-base64@4.3.2': + dependencies: + '@smithy/util-buffer-from': 4.2.2 + '@smithy/util-utf8': 4.2.2 + tslib: 2.8.1 + '@smithy/util-body-length-browser@2.2.0': dependencies: tslib: 2.6.2 @@ -14194,6 +15104,10 @@ snapshots: dependencies: tslib: 2.8.1 + '@smithy/util-body-length-browser@4.2.2': + dependencies: + tslib: 2.8.1 + '@smithy/util-body-length-node@2.3.0': dependencies: tslib: 2.6.2 @@ -14202,6 +15116,10 @@ snapshots: dependencies: tslib: 2.8.1 + '@smithy/util-body-length-node@4.2.3': + dependencies: + tslib: 2.8.1 + '@smithy/util-buffer-from@1.1.0': dependencies: '@smithy/is-array-buffer': 1.1.0 @@ -14217,6 +15135,11 @@ snapshots: '@smithy/is-array-buffer': 4.2.0 tslib: 2.8.1 + '@smithy/util-buffer-from@4.2.2': + dependencies: + '@smithy/is-array-buffer': 4.2.2 + tslib: 2.8.1 + '@smithy/util-config-provider@2.3.0': dependencies: tslib: 2.8.1 @@ -14225,6 +15148,10 @@ snapshots: dependencies: tslib: 2.8.1 + '@smithy/util-config-provider@4.2.2': + dependencies: + tslib: 2.8.1 + '@smithy/util-defaults-mode-browser@2.2.1': dependencies: '@smithy/property-provider': 2.2.0 @@ -14240,6 +15167,13 @@ snapshots: '@smithy/types': 4.12.0 tslib: 2.8.1 + '@smithy/util-defaults-mode-browser@4.3.48': + dependencies: + '@smithy/property-provider': 4.2.14 + '@smithy/smithy-client': 4.12.12 + '@smithy/types': 4.14.1 + tslib: 2.8.1 + '@smithy/util-defaults-mode-node@2.3.1': dependencies: '@smithy/config-resolver': 2.2.0 @@ -14260,6 +15194,16 @@ snapshots: '@smithy/types': 4.12.0 tslib: 2.8.1 + '@smithy/util-defaults-mode-node@4.2.53': + dependencies: + '@smithy/config-resolver': 4.4.17 + '@smithy/credential-provider-imds': 4.2.14 + '@smithy/node-config-provider': 4.3.14 + '@smithy/property-provider': 4.2.14 + '@smithy/smithy-client': 4.12.12 + '@smithy/types': 4.14.1 + tslib: 2.8.1 + '@smithy/util-endpoints@1.2.0': dependencies: '@smithy/node-config-provider': 2.3.0 @@ -14272,6 +15216,12 @@ snapshots: '@smithy/types': 4.12.0 tslib: 2.8.1 + '@smithy/util-endpoints@3.4.2': + dependencies: + '@smithy/node-config-provider': 4.3.14 + '@smithy/types': 4.14.1 + tslib: 2.8.1 + '@smithy/util-hex-encoding@2.2.0': dependencies: tslib: 2.8.1 @@ -14280,11 +15230,20 @@ snapshots: dependencies: tslib: 2.8.1 + '@smithy/util-hex-encoding@4.2.2': + dependencies: + tslib: 2.8.1 + '@smithy/util-middleware@2.2.0': dependencies: '@smithy/types': 2.12.0 tslib: 2.6.2 + '@smithy/util-middleware@4.2.14': + dependencies: + '@smithy/types': 4.14.1 + tslib: 2.8.1 + '@smithy/util-middleware@4.2.8': dependencies: '@smithy/types': 4.12.0 @@ -14302,6 +15261,12 @@ snapshots: '@smithy/types': 4.12.0 tslib: 2.8.1 + '@smithy/util-retry@4.3.3': + dependencies: + '@smithy/service-error-classification': 4.3.0 + '@smithy/types': 4.14.1 + tslib: 2.8.1 + '@smithy/util-stream@2.2.0': dependencies: '@smithy/fetch-http-handler': 2.5.0 @@ -14324,6 +15289,17 @@ snapshots: '@smithy/util-utf8': 4.2.0 tslib: 2.8.1 + '@smithy/util-stream@4.5.24': + dependencies: + '@smithy/fetch-http-handler': 5.3.17 + '@smithy/node-http-handler': 4.6.0 + '@smithy/types': 4.14.1 + '@smithy/util-base64': 4.3.2 + '@smithy/util-buffer-from': 4.2.2 + '@smithy/util-hex-encoding': 4.2.2 + '@smithy/util-utf8': 4.2.2 + tslib: 2.8.1 + '@smithy/util-uri-escape@2.2.0': dependencies: tslib: 2.8.1 @@ -14332,6 +15308,10 @@ snapshots: dependencies: tslib: 2.8.1 + '@smithy/util-uri-escape@4.2.2': + dependencies: + tslib: 2.8.1 + '@smithy/util-utf8@1.1.0': dependencies: '@smithy/util-buffer-from': 1.1.0 @@ -14347,6 +15327,16 @@ snapshots: '@smithy/util-buffer-from': 4.2.0 tslib: 2.8.1 + '@smithy/util-utf8@4.2.2': + dependencies: + '@smithy/util-buffer-from': 4.2.2 + tslib: 2.8.1 + + '@smithy/util-waiter@4.2.16': + dependencies: + '@smithy/types': 4.14.1 + tslib: 2.8.1 + '@smithy/util-waiter@4.2.8': dependencies: '@smithy/abort-controller': 4.2.8 @@ -14357,6 +15347,10 @@ snapshots: dependencies: tslib: 2.8.1 + '@smithy/uuid@1.1.2': + dependencies: + tslib: 2.8.1 + '@socket.io/component-emitter@3.1.2': {} '@stablelib/base64@1.0.1': {} @@ -14538,17 +15532,17 @@ snapshots: '@types/body-parser@1.19.5': dependencies: '@types/connect': 3.4.38 - '@types/node': 20.12.7 + '@types/node': 20.19.43 '@types/btoa-lite@1.0.2': {} '@types/bunyan-format@0.2.9': dependencies: - '@types/node': 20.12.7 + '@types/node': 20.19.43 '@types/bunyan@1.8.11': dependencies: - '@types/node': 20.12.7 + '@types/node': 20.19.43 '@types/caseless@0.12.5': {} @@ -14561,11 +15555,11 @@ snapshots: '@types/connect@3.4.38': dependencies: - '@types/node': 20.12.7 + '@types/node': 20.19.43 '@types/conventional-commits-parser@5.0.1': dependencies: - '@types/node': 20.12.7 + '@types/node': 20.19.43 '@types/cookie@0.4.1': {} @@ -14573,7 +15567,7 @@ snapshots: '@types/cors@2.8.17': dependencies: - '@types/node': 20.12.7 + '@types/node': 20.19.43 '@types/cron@2.4.0': dependencies: @@ -14595,7 +15589,7 @@ snapshots: '@types/express-serve-static-core@4.19.0': dependencies: - '@types/node': 20.12.7 + '@types/node': 20.19.43 '@types/qs': 6.9.15 '@types/range-parser': 1.2.7 '@types/send': 0.17.4 @@ -14615,7 +15609,7 @@ snapshots: '@types/is-stream@1.1.0': dependencies: - '@types/node': 20.12.7 + '@types/node': 20.19.43 '@types/js-yaml@4.0.9': {} @@ -14625,11 +15619,11 @@ snapshots: '@types/jsonwebtoken@9.0.6': dependencies: - '@types/node': 20.12.7 + '@types/node': 20.19.43 '@types/keyv@3.1.4': dependencies: - '@types/node': 20.12.7 + '@types/node': 20.19.43 '@types/lru-cache@5.1.1': {} @@ -14641,16 +15635,20 @@ snapshots: '@types/node-int64@0.4.32': dependencies: - '@types/node': 20.12.7 + '@types/node': 20.19.43 - '@types/node@20.12.7': + '@types/node@20.19.43': dependencies: - undici-types: 5.26.5 + undici-types: 6.21.0 '@types/node@22.19.10': dependencies: undici-types: 6.21.0 + '@types/node@24.13.3': + dependencies: + undici-types: 7.18.2 + '@types/q@1.5.8': {} '@types/qs@6.9.15': {} @@ -14660,13 +15658,13 @@ snapshots: '@types/request@2.48.12': dependencies: '@types/caseless': 0.12.5 - '@types/node': 20.12.7 + '@types/node': 20.19.43 '@types/tough-cookie': 4.0.5 - form-data: 2.5.1 + form-data: 2.5.6 '@types/responselike@1.0.3': dependencies: - '@types/node': 20.12.7 + '@types/node': 20.19.43 '@types/retry@0.12.0': {} @@ -14679,22 +15677,22 @@ snapshots: '@types/send@0.17.4': dependencies: '@types/mime': 1.3.5 - '@types/node': 20.12.7 + '@types/node': 20.19.43 '@types/serve-static@1.15.7': dependencies: '@types/http-errors': 2.0.4 - '@types/node': 20.12.7 + '@types/node': 20.19.43 '@types/send': 0.17.4 '@types/superagent@4.1.24': dependencies: '@types/cookiejar': 2.1.5 - '@types/node': 20.12.7 + '@types/node': 20.19.43 '@types/thrift@0.10.17': dependencies: - '@types/node': 20.12.7 + '@types/node': 20.19.43 '@types/node-int64': 0.4.32 '@types/q': 1.5.8 @@ -14704,7 +15702,7 @@ snapshots: '@types/unzipper@0.10.11': dependencies: - '@types/node': 20.12.7 + '@types/node': 20.19.43 '@types/uuid@9.0.8': {} @@ -14909,29 +15907,29 @@ snapshots: chai: 6.2.2 tinyrainbow: 3.1.0 - '@vitest/mocker@3.2.4(vite@6.3.5(@types/node@20.12.7)(jiti@2.4.2)(terser@5.43.1)(tsx@4.7.3)(yaml@2.7.0))': + '@vitest/mocker@3.2.4(vite@6.3.5(@types/node@20.19.43)(jiti@2.4.2)(terser@5.43.1)(tsx@4.23.12)(yaml@2.7.0))': dependencies: '@vitest/spy': 3.2.4 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 6.3.5(@types/node@20.12.7)(jiti@2.4.2)(terser@5.43.1)(tsx@4.7.3)(yaml@2.7.0) + vite: 6.3.5(@types/node@20.19.43)(jiti@2.4.2)(terser@5.43.1)(tsx@4.23.12)(yaml@2.7.0) - '@vitest/mocker@4.1.7(vite@6.3.5(@types/node@20.12.7)(jiti@2.4.2)(terser@5.43.1)(yaml@2.7.0))': + '@vitest/mocker@4.1.7(vite@6.3.5(@types/node@20.19.43)(jiti@2.4.2)(terser@5.43.1)(tsx@4.23.12)(yaml@2.7.0))': dependencies: '@vitest/spy': 4.1.7 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 6.3.5(@types/node@20.12.7)(jiti@2.4.2)(terser@5.43.1)(tsx@4.7.3)(yaml@2.7.0) + vite: 6.3.5(@types/node@20.19.43)(jiti@2.4.2)(terser@5.43.1)(tsx@4.23.12)(yaml@2.7.0) - '@vitest/mocker@4.1.7(vite@6.3.5(@types/node@22.19.10)(jiti@2.4.2)(terser@5.43.1)(yaml@2.7.0))': + '@vitest/mocker@4.1.7(vite@6.3.5(@types/node@24.13.3)(jiti@2.4.2)(terser@5.43.1)(tsx@4.23.12)(yaml@2.7.0))': dependencies: '@vitest/spy': 4.1.7 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 6.3.5(@types/node@22.19.10)(jiti@2.4.2)(terser@5.43.1)(yaml@2.7.0) + vite: 6.3.5(@types/node@24.13.3)(jiti@2.4.2)(terser@5.43.1)(tsx@4.23.12)(yaml@2.7.0) '@vitest/pretty-format@3.2.4': dependencies: @@ -15097,9 +16095,9 @@ snapshots: mime-types: 3.0.2 negotiator: 1.0.0 - acorn-import-attributes@1.9.5(acorn@8.11.3): + acorn-import-attributes@1.9.5(acorn@8.16.0): dependencies: - acorn: 8.11.3 + acorn: 8.16.0 acorn-import-phases@1.0.4(acorn@8.16.0): dependencies: @@ -15361,39 +16359,23 @@ snapshots: axios@0.27.2: dependencies: - follow-redirects: 1.15.6 - form-data: 4.0.0 - transitivePeerDependencies: - - debug - - axios@1.11.0: - dependencies: - follow-redirects: 1.15.6 - form-data: 4.0.4 - proxy-from-env: 1.1.0 + follow-redirects: 1.16.0 + form-data: 4.0.6 transitivePeerDependencies: - debug axios@1.12.0: dependencies: follow-redirects: 1.16.0 - form-data: 4.0.5 - proxy-from-env: 1.1.0 - transitivePeerDependencies: - - debug - - axios@1.13.1: - dependencies: - follow-redirects: 1.15.6 - form-data: 4.0.4 + form-data: 4.0.6 proxy-from-env: 1.1.0 transitivePeerDependencies: - debug - axios@1.16.1: + axios@1.19.0: dependencies: follow-redirects: 1.16.0 - form-data: 4.0.5 + form-data: 4.0.6 https-proxy-agent: 5.0.1 proxy-from-env: 2.1.0 transitivePeerDependencies: @@ -15402,24 +16384,8 @@ snapshots: axios@1.6.8: dependencies: - follow-redirects: 1.15.6 - form-data: 4.0.0 - proxy-from-env: 1.1.0 - transitivePeerDependencies: - - debug - - axios@1.8.1: - dependencies: - follow-redirects: 1.15.6 - form-data: 4.0.0 - proxy-from-env: 1.1.0 - transitivePeerDependencies: - - debug - - axios@1.8.4: - dependencies: - follow-redirects: 1.15.6 - form-data: 4.0.0 + follow-redirects: 1.16.0 + form-data: 4.0.6 proxy-from-env: 1.1.0 transitivePeerDependencies: - debug @@ -15467,8 +16433,6 @@ snapshots: bn.js@4.12.1: {} - bn.js@5.2.1: {} - body-parser@1.19.0: dependencies: bytes: 3.1.0 @@ -15659,10 +16623,10 @@ snapshots: call-bind@1.0.7: dependencies: - es-define-property: 1.0.0 + es-define-property: 1.0.1 es-errors: 1.3.0 function-bind: 1.1.2 - get-intrinsic: 1.2.4 + get-intrinsic: 1.3.0 set-function-length: 1.2.2 call-bound@1.0.4: @@ -15734,7 +16698,7 @@ snapshots: ci-info@2.0.0: {} - cjs-module-lexer@1.3.1: {} + cjs-module-lexer@1.4.3: {} clearbit@1.3.5: dependencies: @@ -15902,7 +16866,7 @@ snapshots: ini: 1.3.8 proto-list: 1.2.4 - config@3.3.11: + config@3.3.12: dependencies: json5: 2.2.3 @@ -15985,9 +16949,9 @@ snapshots: object-assign: 4.1.1 vary: 1.1.2 - cosmiconfig-typescript-loader@6.1.0(@types/node@22.19.10)(cosmiconfig@9.0.0(typescript@5.6.3))(typescript@5.6.3): + cosmiconfig-typescript-loader@6.1.0(@types/node@24.13.3)(cosmiconfig@9.0.0(typescript@5.6.3))(typescript@5.6.3): dependencies: - '@types/node': 22.19.10 + '@types/node': 24.13.3 cosmiconfig: 9.0.0(typescript@5.6.3) jiti: 2.4.2 typescript: 5.6.3 @@ -16085,41 +17049,40 @@ snapshots: date-and-time@0.14.2: {} - dc-polyfill@0.1.4: {} + dc-polyfill@0.1.10: {} - dd-trace@4.38.0: + dd-trace@4.55.0: dependencies: - '@datadog/native-appsec': 7.1.1 - '@datadog/native-iast-rewriter': 2.3.1 - '@datadog/native-iast-taint-tracking': 2.1.0 - '@datadog/native-metrics': 2.0.0 - '@datadog/pprof': 5.3.0 + '@datadog/libdatadog': 0.3.0 + '@datadog/native-appsec': 8.4.0 + '@datadog/native-iast-rewriter': 2.6.1 + '@datadog/native-iast-taint-tracking': 3.2.0 + '@datadog/native-metrics': 3.1.1 + '@datadog/pprof': 5.4.1 '@datadog/sketches-js': 2.1.1 + '@isaacs/ttlcache': 1.4.1 '@opentelemetry/api': 1.6.0 - '@opentelemetry/core': 1.24.0(@opentelemetry/api@1.6.0) + '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.6.0) crypto-randomuuid: 1.0.0 - dc-polyfill: 0.1.4 + dc-polyfill: 0.1.10 ignore: 5.3.1 - import-in-the-middle: 1.8.1 - int64-buffer: 0.1.10 - ipaddr.js: 2.2.0 + import-in-the-middle: 1.11.2 istanbul-lib-coverage: 3.2.0 jest-docblock: 29.7.0 koalas: 1.0.2 limiter: 1.1.5 lodash.sortby: 4.7.0 lru-cache: 7.18.3 - methods: 1.1.2 - module-details-from-path: 1.0.3 - msgpack-lite: 0.1.26 - node-abort-controller: 3.1.1 + module-details-from-path: 1.0.4 opentracing: 0.14.7 - path-to-regexp: 0.1.8 - pprof-format: 2.1.0 - protobufjs: 7.2.6 + path-to-regexp: 0.1.12 + pprof-format: 2.2.1 + protobufjs: 7.6.2 retry: 0.13.1 + rfdc: 1.4.1 semver: 7.6.0 - shell-quote: 1.8.1 + shell-quote: 1.8.3 + source-map: 0.7.4 tlhunter-sorted-set: 0.1.0 debug@2.6.9: @@ -16388,7 +17351,7 @@ snapshots: dependencies: '@types/cookie': 0.4.1 '@types/cors': 2.8.17 - '@types/node': 20.12.7 + '@types/node': 20.19.43 accepts: 1.3.8 base64id: 2.0.0 cookie: 0.4.2 @@ -16434,20 +17397,20 @@ snapshots: data-view-buffer: 1.0.1 data-view-byte-length: 1.0.1 data-view-byte-offset: 1.0.0 - es-define-property: 1.0.0 + es-define-property: 1.0.1 es-errors: 1.3.0 - es-object-atoms: 1.0.0 + es-object-atoms: 1.1.1 es-set-tostringtag: 2.1.0 es-to-primitive: 1.2.1 function.prototype.name: 1.1.6 - get-intrinsic: 1.2.4 + get-intrinsic: 1.3.0 get-symbol-description: 1.0.2 globalthis: 1.0.3 - gopd: 1.0.1 + gopd: 1.2.0 has-property-descriptors: 1.0.2 has-proto: 1.0.3 - has-symbols: 1.0.3 - hasown: 2.0.2 + has-symbols: 1.1.0 + hasown: 2.0.4 internal-slot: 1.0.7 is-array-buffer: 3.0.4 is-callable: 1.2.7 @@ -16474,10 +17437,6 @@ snapshots: unbox-primitive: 1.0.2 which-typed-array: 1.1.15 - es-define-property@1.0.0: - dependencies: - get-intrinsic: 1.3.0 - es-define-property@1.0.1: {} es-errors@1.3.0: {} @@ -16499,11 +17458,11 @@ snapshots: es-errors: 1.3.0 get-intrinsic: 1.3.0 has-tostringtag: 1.0.2 - hasown: 2.0.2 + hasown: 2.0.4 es-shim-unscopables@1.0.2: dependencies: - hasown: 2.0.2 + hasown: 2.0.4 es-to-primitive@1.2.1: dependencies: @@ -16538,32 +17497,6 @@ snapshots: es6-iterator: 2.0.3 es6-symbol: 3.1.4 - esbuild@0.19.12: - optionalDependencies: - '@esbuild/aix-ppc64': 0.19.12 - '@esbuild/android-arm': 0.19.12 - '@esbuild/android-arm64': 0.19.12 - '@esbuild/android-x64': 0.19.12 - '@esbuild/darwin-arm64': 0.19.12 - '@esbuild/darwin-x64': 0.19.12 - '@esbuild/freebsd-arm64': 0.19.12 - '@esbuild/freebsd-x64': 0.19.12 - '@esbuild/linux-arm': 0.19.12 - '@esbuild/linux-arm64': 0.19.12 - '@esbuild/linux-ia32': 0.19.12 - '@esbuild/linux-loong64': 0.19.12 - '@esbuild/linux-mips64el': 0.19.12 - '@esbuild/linux-ppc64': 0.19.12 - '@esbuild/linux-riscv64': 0.19.12 - '@esbuild/linux-s390x': 0.19.12 - '@esbuild/linux-x64': 0.19.12 - '@esbuild/netbsd-x64': 0.19.12 - '@esbuild/openbsd-x64': 0.19.12 - '@esbuild/sunos-x64': 0.19.12 - '@esbuild/win32-arm64': 0.19.12 - '@esbuild/win32-ia32': 0.19.12 - '@esbuild/win32-x64': 0.19.12 - esbuild@0.25.12: optionalDependencies: '@esbuild/aix-ppc64': 0.25.12 @@ -16593,6 +17526,35 @@ snapshots: '@esbuild/win32-ia32': 0.25.12 '@esbuild/win32-x64': 0.25.12 + esbuild@0.28.0: + optionalDependencies: + '@esbuild/aix-ppc64': 0.28.0 + '@esbuild/android-arm': 0.28.0 + '@esbuild/android-arm64': 0.28.0 + '@esbuild/android-x64': 0.28.0 + '@esbuild/darwin-arm64': 0.28.0 + '@esbuild/darwin-x64': 0.28.0 + '@esbuild/freebsd-arm64': 0.28.0 + '@esbuild/freebsd-x64': 0.28.0 + '@esbuild/linux-arm': 0.28.0 + '@esbuild/linux-arm64': 0.28.0 + '@esbuild/linux-ia32': 0.28.0 + '@esbuild/linux-loong64': 0.28.0 + '@esbuild/linux-mips64el': 0.28.0 + '@esbuild/linux-ppc64': 0.28.0 + '@esbuild/linux-riscv64': 0.28.0 + '@esbuild/linux-s390x': 0.28.0 + '@esbuild/linux-x64': 0.28.0 + '@esbuild/netbsd-arm64': 0.28.0 + '@esbuild/netbsd-x64': 0.28.0 + '@esbuild/openbsd-arm64': 0.28.0 + '@esbuild/openbsd-x64': 0.28.0 + '@esbuild/openharmony-arm64': 0.28.0 + '@esbuild/sunos-x64': 0.28.0 + '@esbuild/win32-arm64': 0.28.0 + '@esbuild/win32-ia32': 0.28.0 + '@esbuild/win32-x64': 0.28.0 + escalade@3.1.2: {} escalade@3.2.0: {} @@ -16790,8 +17752,6 @@ snapshots: d: 1.0.2 es5-ext: 0.10.64 - event-lite@0.1.3: {} - event-target-shim@5.0.1: {} eventemitter3@3.1.2: {} @@ -17051,6 +18011,12 @@ snapshots: dependencies: strnum: 2.1.2 + fast-xml-parser@5.5.8: + dependencies: + fast-xml-builder: 1.2.0 + path-expression-matcher: 1.5.0 + strnum: 2.3.0 + fast-xml-parser@5.8.0: dependencies: '@nodable/entities': 2.1.0 @@ -17165,8 +18131,6 @@ snapshots: fn.name@1.1.0: {} - follow-redirects@1.15.6: {} - follow-redirects@1.16.0: {} for-each@0.3.3: @@ -17178,32 +18142,21 @@ snapshots: cross-spawn: 7.0.3 signal-exit: 4.1.0 - form-data@2.5.1: - dependencies: - asynckit: 0.4.0 - combined-stream: 1.0.8 - mime-types: 2.1.35 - - form-data@4.0.0: - dependencies: - asynckit: 0.4.0 - combined-stream: 1.0.8 - mime-types: 2.1.35 - - form-data@4.0.4: + form-data@2.5.6: dependencies: asynckit: 0.4.0 combined-stream: 1.0.8 es-set-tostringtag: 2.1.0 - hasown: 2.0.2 + hasown: 2.0.4 mime-types: 2.1.35 + safe-buffer: 5.2.1 - form-data@4.0.5: + form-data@4.0.6: dependencies: asynckit: 0.4.0 combined-stream: 1.0.8 es-set-tostringtag: 2.1.0 - hasown: 2.0.2 + hasown: 2.0.4 mime-types: 2.1.35 formdata-polyfill@4.0.10: @@ -17216,12 +18169,12 @@ snapshots: formidable@1.2.6: {} - formidable@2.1.2: + formidable@2.1.5: dependencies: + '@paralleldrive/cuid2': 2.3.1 dezalgo: 1.0.4 - hexoid: 1.0.0 once: 1.4.0 - qs: 6.13.0 + qs: 6.15.3 forwarded@0.2.0: {} @@ -17367,8 +18320,8 @@ snapshots: es-errors: 1.3.0 function-bind: 1.1.2 has-proto: 1.0.3 - has-symbols: 1.0.3 - hasown: 2.0.2 + has-symbols: 1.1.0 + hasown: 2.0.4 get-intrinsic@1.3.0: dependencies: @@ -17380,7 +18333,7 @@ snapshots: get-proto: 1.0.1 gopd: 1.2.0 has-symbols: 1.1.0 - hasown: 2.0.2 + hasown: 2.0.4 math-intrinsics: 1.1.0 get-proto@1.0.1: @@ -17406,10 +18359,6 @@ snapshots: es-errors: 1.3.0 get-intrinsic: 1.3.0 - get-tsconfig@4.7.3: - dependencies: - resolve-pkg-maps: 1.0.0 - git-raw-commits@4.0.0: dependencies: dargs: 8.1.0 @@ -17527,10 +18476,6 @@ snapshots: dependencies: node-forge: 1.3.1 - gopd@1.0.1: - dependencies: - get-intrinsic: 1.3.0 - gopd@1.2.0: {} got@9.6.0: @@ -17616,14 +18561,16 @@ snapshots: dependencies: function-bind: 1.1.2 + hasown@2.0.4: + dependencies: + function-bind: 1.1.2 + he@1.2.0: {} heap-js@2.7.1: {} helmet@4.1.1: {} - hexoid@1.0.0: {} - highlight.js@10.7.3: {} homedir-polyfill@1.0.3: @@ -17734,7 +18681,7 @@ snapshots: https-proxy-agent@7.0.4: dependencies: agent-base: 7.1.1 - debug: 4.4.0 + debug: 4.4.3(supports-color@5.5.0) transitivePeerDependencies: - supports-color @@ -17777,12 +18724,12 @@ snapshots: parent-module: 1.0.1 resolve-from: 4.0.0 - import-in-the-middle@1.8.1: + import-in-the-middle@1.11.2: dependencies: - acorn: 8.11.3 - acorn-import-attributes: 1.9.5(acorn@8.11.3) - cjs-module-lexer: 1.3.1 - module-details-from-path: 1.0.3 + acorn: 8.16.0 + acorn-import-attributes: 1.9.5(acorn@8.16.0) + cjs-module-lexer: 1.4.3 + module-details-from-path: 1.0.4 import-lazy@2.1.0: {} @@ -17811,13 +18758,11 @@ snapshots: int53@1.0.0: {} - int64-buffer@0.1.10: {} - internal-slot@1.0.7: dependencies: es-errors: 1.3.0 - hasown: 2.0.2 - side-channel: 1.0.6 + hasown: 2.0.4 + side-channel: 1.1.1 invert-kv@1.0.0: {} @@ -17825,8 +18770,6 @@ snapshots: ipaddr.js@1.9.1: {} - ipaddr.js@2.2.0: {} - is-array-buffer@3.0.4: dependencies: call-bind: 1.0.7 @@ -18022,7 +18965,7 @@ snapshots: jest-worker@27.5.1: dependencies: - '@types/node': 20.12.7 + '@types/node': 20.19.43 merge-stream: 2.0.0 supports-color: 8.1.1 @@ -18121,7 +19064,7 @@ snapshots: jsonwebtoken@8.5.1: dependencies: - jws: 3.2.2 + jws: 3.2.3 lodash.includes: 4.3.0 lodash.isboolean: 3.0.3 lodash.isinteger: 4.0.4 @@ -18145,7 +19088,7 @@ snapshots: ms: 2.1.3 semver: 7.6.0 - jwa@1.4.1: + jwa@1.4.2: dependencies: buffer-equal-constant-time: 1.0.1 ecdsa-sig-formatter: 1.0.11 @@ -18168,9 +19111,9 @@ snapshots: transitivePeerDependencies: - supports-color - jws@3.2.2: + jws@3.2.3: dependencies: - jwa: 1.4.1 + jwa: 1.4.2 safe-buffer: 5.2.1 jws@4.0.1: @@ -18526,7 +19469,7 @@ snapshots: mkdirp@1.0.4: {} - module-details-from-path@1.0.3: {} + module-details-from-path@1.0.4: {} moment-timezone@0.5.45: dependencies: @@ -18548,13 +19491,6 @@ snapshots: ms@3.0.0-canary.1: {} - msgpack-lite@0.1.26: - dependencies: - event-lite: 0.1.3 - ieee754: 1.2.1 - int64-buffer: 0.1.10 - isarray: 1.0.0 - mute-stream@0.0.8: {} mv@2.1.1: @@ -18593,7 +19529,7 @@ snapshots: dependencies: debug: 3.2.7(supports-color@5.5.0) iconv-lite: 0.4.24 - sax: 1.3.0 + sax: 1.6.0 transitivePeerDependencies: - supports-color @@ -18611,8 +19547,6 @@ snapshots: nexus-rpc@0.0.2: {} - node-abort-controller@3.1.1: {} - node-addon-api@3.2.1: {} node-addon-api@6.1.0: {} @@ -18641,6 +19575,8 @@ snapshots: node-gyp-build@4.8.0: {} + node-gyp-build@4.8.4: {} + node-html-markdown@1.3.0: dependencies: node-html-parser: 6.1.13 @@ -19150,8 +20086,6 @@ snapshots: path-to-regexp@0.1.7: {} - path-to-regexp@0.1.8: {} - path-to-regexp@8.4.2: {} path-type@2.0.0: @@ -19170,7 +20104,7 @@ snapshots: peopledatalabs@6.1.5: dependencies: - axios: 1.16.1 + axios: 1.19.0 copy-anything: 3.0.5 transitivePeerDependencies: - debug @@ -19279,7 +20213,7 @@ snapshots: dependencies: xtend: 4.0.2 - pprof-format@2.1.0: {} + pprof-format@2.2.1: {} prelude-ls@1.2.1: {} @@ -19308,21 +20242,6 @@ snapshots: dependencies: protobufjs: 7.6.2 - protobufjs@7.2.6: - dependencies: - '@protobufjs/aspromise': 1.1.2 - '@protobufjs/base64': 1.1.2 - '@protobufjs/codegen': 2.0.4 - '@protobufjs/eventemitter': 1.1.0 - '@protobufjs/fetch': 1.1.0 - '@protobufjs/float': 1.0.2 - '@protobufjs/inquire': 1.1.0 - '@protobufjs/path': 1.1.2 - '@protobufjs/pool': 1.1.0 - '@protobufjs/utf8': 1.1.0 - '@types/node': 20.12.7 - long: 5.3.2 - protobufjs@7.5.5: dependencies: '@protobufjs/aspromise': 1.1.2 @@ -19335,7 +20254,7 @@ snapshots: '@protobufjs/path': 1.1.2 '@protobufjs/pool': 1.1.0 '@protobufjs/utf8': 1.1.0 - '@types/node': 20.12.7 + '@types/node': 20.19.43 long: 5.3.2 protobufjs@7.6.2: @@ -19350,7 +20269,7 @@ snapshots: '@protobufjs/path': 1.1.2 '@protobufjs/pool': 1.1.0 '@protobufjs/utf8': 1.1.1 - '@types/node': 20.12.7 + '@types/node': 20.19.43 long: 5.3.2 proxy-addr@2.0.7: @@ -19391,18 +20310,10 @@ snapshots: dependencies: side-channel: 1.0.6 - qs@6.12.1: - dependencies: - side-channel: 1.0.6 - qs@6.13.0: dependencies: side-channel: 1.0.6 - qs@6.14.0: - dependencies: - side-channel: 1.1.0 - qs@6.15.3: dependencies: es-define-property: 1.0.1 @@ -19460,12 +20371,12 @@ snapshots: cli-table: 0.3.11 command-line-args: 5.2.1 command-line-usage: 6.1.3 - config: 3.3.11 + config: 3.3.12 configstore: 5.0.1 debug: 4.3.4 editor: 1.0.0 enquirer: 2.4.1 - form-data: 4.0.0 + form-data: 4.0.6 gray-matter: 4.0.3 isemail: 3.2.0 mime-types: 2.1.35 @@ -19577,8 +20488,6 @@ snapshots: resolve-from@5.0.0: {} - resolve-pkg-maps@1.0.0: {} - resolve@1.22.8: dependencies: is-core-module: 2.13.1 @@ -19734,7 +20643,7 @@ snapshots: sax@1.2.1: {} - sax@1.3.0: {} + sax@1.6.0: {} schema-utils@4.3.3: dependencies: @@ -19948,7 +20857,7 @@ snapshots: shebang-regex@3.0.0: {} - shell-quote@1.8.1: {} + shell-quote@1.8.3: {} should-equal@2.0.0: dependencies: @@ -19976,11 +20885,6 @@ snapshots: should-type-adaptors: 1.1.0 should-util: 1.0.1 - side-channel-list@1.0.0: - dependencies: - es-errors: 1.3.0 - object-inspect: 1.13.4 - side-channel-list@1.0.1: dependencies: es-errors: 1.3.0 @@ -20008,14 +20912,6 @@ snapshots: get-intrinsic: 1.3.0 object-inspect: 1.13.1 - side-channel@1.1.0: - dependencies: - es-errors: 1.3.0 - object-inspect: 1.13.4 - side-channel-list: 1.0.0 - side-channel-map: 1.0.1 - side-channel-weakmap: 1.0.2 - side-channel@1.1.1: dependencies: es-errors: 1.3.0 @@ -20062,16 +20958,15 @@ snapshots: snappyjs@0.7.0: {} - snowflake-sdk@2.3.4(asn1.js@5.4.1): + snowflake-sdk@2.4.0(asn1.js@5.4.1): dependencies: '@aws-crypto/sha256-js': 5.2.0 - '@aws-sdk/client-s3': 3.985.0 - '@aws-sdk/client-sts': 3.985.0 - '@aws-sdk/credential-provider-node': 3.972.6 - '@aws-sdk/ec2-metadata-service': 3.985.0 + '@aws-sdk/client-s3': 3.1033.0 + '@aws-sdk/client-sts': 3.1033.0 + '@aws-sdk/credential-provider-node': 3.972.33 + '@aws-sdk/ec2-metadata-service': 3.1033.0 '@azure/identity': 4.13.0 '@azure/storage-blob': 12.26.0 - '@google-cloud/storage': 7.19.0(encoding@0.1.13) '@smithy/node-http-handler': 4.4.9 '@smithy/protocol-http': 5.3.8 '@smithy/signature-v4': 5.3.8 @@ -20079,13 +20974,12 @@ snapshots: asn1.js: 5.4.1 asn1.js-rfc2560: 5.0.1(asn1.js@5.4.1) asn1.js-rfc5280: 3.0.0 - axios: 1.16.1 + axios: 1.19.0 big-integer: 1.6.52 bignumber.js: 9.1.2 - bn.js: 5.2.1 browser-request: 0.3.3 expand-tilde: 2.0.2 - fast-xml-parser: 5.3.5 + fast-xml-parser: 5.8.0 fastest-levenshtein: 1.0.16 generic-pool: 3.9.0 google-auth-library: 10.5.0 @@ -20103,7 +20997,6 @@ snapshots: transitivePeerDependencies: - aws-crt - debug - - encoding - supports-color socket.io-adapter@2.5.4(bufferutil@4.0.8)(utf-8-validate@5.0.10): @@ -20306,13 +21199,13 @@ snapshots: dependencies: component-emitter: 1.3.1 cookiejar: 2.1.4 - debug: 4.3.4 + debug: 4.4.3(supports-color@5.5.0) fast-safe-stringify: 2.1.1 - form-data: 4.0.0 - formidable: 2.1.2 + form-data: 4.0.6 + formidable: 2.1.5 methods: 1.1.2 mime: 2.6.0 - qs: 6.12.1 + qs: 6.15.3 semver: 7.6.0 transitivePeerDependencies: - supports-color @@ -20552,14 +21445,14 @@ snapshots: dependencies: typescript: 5.6.3 - ts-node@10.9.2(@swc/core@1.4.17)(@types/node@20.12.7)(typescript@5.6.3): + ts-node@10.9.2(@swc/core@1.4.17)(@types/node@20.19.43)(typescript@5.6.3): dependencies: '@cspotcode/source-map-support': 0.8.1 '@tsconfig/node10': 1.0.11 '@tsconfig/node12': 1.0.11 '@tsconfig/node14': 1.0.3 '@tsconfig/node16': 1.0.4 - '@types/node': 20.12.7 + '@types/node': 20.19.43 acorn: 8.11.3 acorn-walk: 8.3.2 arg: 4.1.3 @@ -20596,10 +21489,9 @@ snapshots: tslib: 1.14.1 typescript: 5.6.3 - tsx@4.7.3: + tsx@4.23.12: dependencies: - esbuild: 0.19.12 - get-tsconfig: 4.7.3 + esbuild: 0.28.0 optionalDependencies: fsevents: 2.3.3 @@ -20687,10 +21579,10 @@ snapshots: undefsafe@2.0.5: {} - undici-types@5.26.5: {} - undici-types@6.21.0: {} + undici-types@7.18.2: {} + undici@5.29.0: dependencies: '@fastify/busboy': 2.1.1 @@ -20819,13 +21711,13 @@ snapshots: verify-github-webhook@1.0.1: {} - vite-node@3.2.4(@types/node@20.12.7)(jiti@2.4.2)(terser@5.43.1)(tsx@4.7.3)(yaml@2.7.0): + vite-node@3.2.4(@types/node@20.19.43)(jiti@2.4.2)(terser@5.43.1)(tsx@4.23.12)(yaml@2.7.0): dependencies: cac: 6.7.14 debug: 4.4.3(supports-color@5.5.0) es-module-lexer: 1.7.0 pathe: 2.0.3 - vite: 6.3.5(@types/node@20.12.7)(jiti@2.4.2)(terser@5.43.1)(tsx@4.7.3)(yaml@2.7.0) + vite: 6.3.5(@types/node@20.19.43)(jiti@2.4.2)(terser@5.43.1)(tsx@4.23.12)(yaml@2.7.0) transitivePeerDependencies: - '@types/node' - jiti @@ -20840,7 +21732,7 @@ snapshots: - tsx - yaml - vite@6.3.5(@types/node@20.12.7)(jiti@2.4.2)(terser@5.43.1)(tsx@4.7.3)(yaml@2.7.0): + vite@6.3.5(@types/node@20.19.43)(jiti@2.4.2)(terser@5.43.1)(tsx@4.23.12)(yaml@2.7.0): dependencies: esbuild: 0.25.12 fdir: 6.5.0(picomatch@4.0.4) @@ -20849,14 +21741,14 @@ snapshots: rollup: 4.60.1 tinyglobby: 0.2.15 optionalDependencies: - '@types/node': 20.12.7 + '@types/node': 20.19.43 fsevents: 2.3.3 jiti: 2.4.2 terser: 5.43.1 - tsx: 4.7.3 + tsx: 4.23.12 yaml: 2.7.0 - vite@6.3.5(@types/node@22.19.10)(jiti@2.4.2)(terser@5.43.1)(yaml@2.7.0): + vite@6.3.5(@types/node@24.13.3)(jiti@2.4.2)(terser@5.43.1)(tsx@4.23.12)(yaml@2.7.0): dependencies: esbuild: 0.25.12 fdir: 6.5.0(picomatch@4.0.4) @@ -20865,17 +21757,18 @@ snapshots: rollup: 4.60.1 tinyglobby: 0.2.15 optionalDependencies: - '@types/node': 22.19.10 + '@types/node': 24.13.3 fsevents: 2.3.3 jiti: 2.4.2 terser: 5.43.1 + tsx: 4.23.12 yaml: 2.7.0 - vitest@3.2.4(@types/debug@4.1.12)(@types/node@20.12.7)(jiti@2.4.2)(terser@5.43.1)(tsx@4.7.3)(yaml@2.7.0): + vitest@3.2.4(@types/debug@4.1.12)(@types/node@20.19.43)(jiti@2.4.2)(terser@5.43.1)(tsx@4.23.12)(yaml@2.7.0): dependencies: '@types/chai': 5.2.3 '@vitest/expect': 3.2.4 - '@vitest/mocker': 3.2.4(vite@6.3.5(@types/node@20.12.7)(jiti@2.4.2)(terser@5.43.1)(tsx@4.7.3)(yaml@2.7.0)) + '@vitest/mocker': 3.2.4(vite@6.3.5(@types/node@20.19.43)(jiti@2.4.2)(terser@5.43.1)(tsx@4.23.12)(yaml@2.7.0)) '@vitest/pretty-format': 3.2.4 '@vitest/runner': 3.2.4 '@vitest/snapshot': 3.2.4 @@ -20893,12 +21786,12 @@ snapshots: tinyglobby: 0.2.15 tinypool: 1.1.1 tinyrainbow: 2.0.0 - vite: 6.3.5(@types/node@20.12.7)(jiti@2.4.2)(terser@5.43.1)(tsx@4.7.3)(yaml@2.7.0) - vite-node: 3.2.4(@types/node@20.12.7)(jiti@2.4.2)(terser@5.43.1)(tsx@4.7.3)(yaml@2.7.0) + vite: 6.3.5(@types/node@20.19.43)(jiti@2.4.2)(terser@5.43.1)(tsx@4.23.12)(yaml@2.7.0) + vite-node: 3.2.4(@types/node@20.19.43)(jiti@2.4.2)(terser@5.43.1)(tsx@4.23.12)(yaml@2.7.0) why-is-node-running: 2.3.0 optionalDependencies: '@types/debug': 4.1.12 - '@types/node': 20.12.7 + '@types/node': 20.19.43 transitivePeerDependencies: - jiti - less @@ -20913,10 +21806,10 @@ snapshots: - tsx - yaml - vitest@4.1.7(@types/node@20.12.7)(vite@6.3.5(@types/node@20.12.7)(jiti@2.4.2)(terser@5.43.1)(yaml@2.7.0)): + vitest@4.1.7(@types/node@20.19.43)(vite@6.3.5(@types/node@20.19.43)(jiti@2.4.2)(terser@5.43.1)(tsx@4.23.12)(yaml@2.7.0)): dependencies: '@vitest/expect': 4.1.7 - '@vitest/mocker': 4.1.7(vite@6.3.5(@types/node@20.12.7)(jiti@2.4.2)(terser@5.43.1)(yaml@2.7.0)) + '@vitest/mocker': 4.1.7(vite@6.3.5(@types/node@20.19.43)(jiti@2.4.2)(terser@5.43.1)(tsx@4.23.12)(yaml@2.7.0)) '@vitest/pretty-format': 4.1.7 '@vitest/runner': 4.1.7 '@vitest/snapshot': 4.1.7 @@ -20933,17 +21826,17 @@ snapshots: tinyexec: 1.2.4 tinyglobby: 0.2.15 tinyrainbow: 3.1.0 - vite: 6.3.5(@types/node@20.12.7)(jiti@2.4.2)(terser@5.43.1)(tsx@4.7.3)(yaml@2.7.0) + vite: 6.3.5(@types/node@20.19.43)(jiti@2.4.2)(terser@5.43.1)(tsx@4.23.12)(yaml@2.7.0) why-is-node-running: 2.3.0 optionalDependencies: - '@types/node': 20.12.7 + '@types/node': 20.19.43 transitivePeerDependencies: - msw - vitest@4.1.7(@types/node@22.19.10)(vite@6.3.5(@types/node@22.19.10)(jiti@2.4.2)(terser@5.43.1)(yaml@2.7.0)): + vitest@4.1.7(@types/node@24.13.3)(vite@6.3.5(@types/node@24.13.3)(jiti@2.4.2)(terser@5.43.1)(tsx@4.23.12)(yaml@2.7.0)): dependencies: '@vitest/expect': 4.1.7 - '@vitest/mocker': 4.1.7(vite@6.3.5(@types/node@22.19.10)(jiti@2.4.2)(terser@5.43.1)(yaml@2.7.0)) + '@vitest/mocker': 4.1.7(vite@6.3.5(@types/node@24.13.3)(jiti@2.4.2)(terser@5.43.1)(tsx@4.23.12)(yaml@2.7.0)) '@vitest/pretty-format': 4.1.7 '@vitest/runner': 4.1.7 '@vitest/snapshot': 4.1.7 @@ -20960,10 +21853,10 @@ snapshots: tinyexec: 1.2.4 tinyglobby: 0.2.15 tinyrainbow: 3.1.0 - vite: 6.3.5(@types/node@22.19.10)(jiti@2.4.2)(terser@5.43.1)(yaml@2.7.0) + vite: 6.3.5(@types/node@24.13.3)(jiti@2.4.2)(terser@5.43.1)(tsx@4.23.12)(yaml@2.7.0) why-is-node-running: 2.3.0 optionalDependencies: - '@types/node': 22.19.10 + '@types/node': 24.13.3 transitivePeerDependencies: - msw @@ -21087,7 +21980,7 @@ snapshots: wkx@0.5.0: dependencies: - '@types/node': 20.12.7 + '@types/node': 20.19.43 word-wrap@1.2.5: {} From 2bac076b21c139537dce2bc418acfb1307bdc90e Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Thu, 27 Aug 2026 11:27:46 +0100 Subject: [PATCH 22/69] fix: publish schema-parsed record instead of raw input in emit Signed-off-by: Mouad BANI --- services/libs/connectors/src/emit.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/services/libs/connectors/src/emit.ts b/services/libs/connectors/src/emit.ts index e724ccbf44..f2a12ad5e0 100644 --- a/services/libs/connectors/src/emit.ts +++ b/services/libs/connectors/src/emit.ts @@ -26,8 +26,9 @@ export function createEmit(deps: EmitterDeps): Emitter { const emit = async (records: unknown[]): Promise => { for (const record of records) { + let parsed: unknown try { - deps.schema.parse(record) + parsed = deps.schema.parse(record) } catch (err) { if (err instanceof ZodError) { throw new ConnectorError('connector.code', 'record failed schema validation', { @@ -37,7 +38,7 @@ export function createEmit(deps: EmitterDeps): Emitter { throw err } - const payload = { ...(record as Record), channel: deps.unit.channelName } + const payload = { ...(parsed as Record), channel: deps.unit.channelName } const resultId = await deps.publishResult(deps.unit.integrationId, { type: IntegrationResultType.ACTIVITY, From a0dafee71a0495eb6a06470c54bf44cc1c11bec8 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Thu, 27 Aug 2026 12:12:56 +0100 Subject: [PATCH 23/69] fix: address http client and emit schema review findings Signed-off-by: Mouad BANI --- services/libs/connectors/src/emit.ts | 6 +++--- services/libs/connectors/src/http/client.ts | 9 ++++++--- services/libs/connectors/src/types.ts | 2 +- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/services/libs/connectors/src/emit.ts b/services/libs/connectors/src/emit.ts index f2a12ad5e0..2deef9d510 100644 --- a/services/libs/connectors/src/emit.ts +++ b/services/libs/connectors/src/emit.ts @@ -12,7 +12,7 @@ export interface EmitterDeps { sinkEmitter: DataSinkWorkerEmitter unit: ISyncUnit segmentId: string - schema: ZodType + schema: ZodType> log: Logger } @@ -26,7 +26,7 @@ export function createEmit(deps: EmitterDeps): Emitter { const emit = async (records: unknown[]): Promise => { for (const record of records) { - let parsed: unknown + let parsed: Record try { parsed = deps.schema.parse(record) } catch (err) { @@ -38,7 +38,7 @@ export function createEmit(deps: EmitterDeps): Emitter { throw err } - const payload = { ...(parsed as Record), channel: deps.unit.channelName } + const payload = { ...parsed, channel: deps.unit.channelName } const resultId = await deps.publishResult(deps.unit.integrationId, { type: IntegrationResultType.ACTIVITY, diff --git a/services/libs/connectors/src/http/client.ts b/services/libs/connectors/src/http/client.ts index 4daa5b7196..9761a8d489 100644 --- a/services/libs/connectors/src/http/client.ts +++ b/services/libs/connectors/src/http/client.ts @@ -29,7 +29,6 @@ export interface HttpClientDeps { acquireToken: () => Promise parkToken: (tokenId: string, resumeAt: Date) => Promise quarantineToken: (tokenId: string) => Promise - correctBudget: (headers: Record) => Promise log: Logger applyToken?: TokenApplier interpretResponse?: ResponseInterpreter @@ -81,7 +80,6 @@ async function attemptRequest( const error = classifyResponse(deps, response.status, headers, response.data) if (!error) { - await deps.correctBudget(headers) return response.data } @@ -162,10 +160,15 @@ function isRateLimited(status: number, headers: Record): boolean } function computeResumeAt(headers: Record): Date { - const retryAfterSeconds = Number(headers['retry-after']) + const retryAfter = headers['retry-after'] + const retryAfterSeconds = Number(retryAfter) if (Number.isFinite(retryAfterSeconds) && retryAfterSeconds > 0) { return new Date(Date.now() + retryAfterSeconds * 1000) } + const retryAfterDateMs = Date.parse(retryAfter ?? '') + if (Number.isFinite(retryAfterDateMs) && retryAfterDateMs > Date.now()) { + return new Date(retryAfterDateMs) + } const resetEpochSeconds = Number(headers['x-ratelimit-reset']) if (Number.isFinite(resetEpochSeconds) && resetEpochSeconds > 0) { return new Date(resetEpochSeconds * 1000) diff --git a/services/libs/connectors/src/types.ts b/services/libs/connectors/src/types.ts index a1a6e4f562..b32728a781 100644 --- a/services/libs/connectors/src/types.ts +++ b/services/libs/connectors/src/types.ts @@ -31,7 +31,7 @@ export interface SyncContext { export interface SyncDefinition { name: string cadenceMinutes: number - schema: ZodType + schema: ZodType> run: (ctx: SyncContext) => Promise } From 12f2cddb41cb9188775d1d6b356d998e6cee5bcd Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Thu, 27 Aug 2026 12:24:21 +0100 Subject: [PATCH 24/69] fix: align node 24 targets, migration ordering and applier classification Signed-off-by: Mouad BANI --- ... => V1787829533__createSyncUnitsTable.sql} | 0 pnpm-lock.yaml | 35 ++++++++----------- .../docker/Dockerfile.connectors_worker | 4 +-- services/apps/connectors_worker/package.json | 2 +- services/libs/connectors/package.json | 2 +- services/libs/connectors/src/http/client.ts | 11 +++--- 6 files changed, 24 insertions(+), 30 deletions(-) rename backend/src/database/migrations/{V1786442761__createSyncUnitsTable.sql => V1787829533__createSyncUnitsTable.sql} (100%) diff --git a/backend/src/database/migrations/V1786442761__createSyncUnitsTable.sql b/backend/src/database/migrations/V1787829533__createSyncUnitsTable.sql similarity index 100% rename from backend/src/database/migrations/V1786442761__createSyncUnitsTable.sql rename to backend/src/database/migrations/V1787829533__createSyncUnitsTable.sql diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c1a1feeed4..eddf82e1e4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -648,8 +648,8 @@ importers: version: 5.6.3 devDependencies: '@types/node': - specifier: ^20.8.2 - version: 20.19.43 + specifier: ^24.13.3 + version: 24.13.3 nodemon: specifier: ^3.0.1 version: 3.1.0 @@ -2295,8 +2295,8 @@ importers: version: 3.25.76 devDependencies: '@types/node': - specifier: ^20.8.2 - version: 20.19.43 + specifier: ^24.13.3 + version: 24.13.3 typescript: specifier: ^5.6.3 version: 5.6.3 @@ -5365,9 +5365,6 @@ packages: '@types/node-int64@0.4.32': resolution: {integrity: sha512-xf/JsSlnXQ+mzvc0IpXemcrO4BrCfpgNpMco+GLcXkFk01k/gW9lGJu+Vof0ZSvHK6DsHJDPSbjFPs36QkWXqw==} - '@types/node@20.19.43': - resolution: {integrity: sha512-6oYBAi5ikg4Pl+kGsoYtawUMBT2zZMCvPNF7pVLnHZfd1zf38DRiWn/gT01RYCdUqkv7Fhr+C9ot4/tb+2sVvA==} - '@types/node@22.19.10': resolution: {integrity: sha512-tF5VOugLS/EuDlTBijk0MqABfP8UxgYazTLo3uIn3b4yJgg26QRbVYJYsDtHrjdDUIRfP70+VfhTTc+CE1yskw==} @@ -11056,8 +11053,8 @@ snapshots: dependencies: '@aws-crypto/sha256-browser': 3.0.0 '@aws-crypto/sha256-js': 3.0.0 - '@aws-sdk/client-sso-oidc': 3.572.0(@aws-sdk/client-sts@3.572.0) - '@aws-sdk/client-sts': 3.572.0 + '@aws-sdk/client-sso-oidc': 3.572.0 + '@aws-sdk/client-sts': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) '@aws-sdk/core': 3.572.0 '@aws-sdk/credential-provider-node': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0)(@aws-sdk/client-sts@3.572.0) '@aws-sdk/middleware-host-header': 3.567.0 @@ -11265,11 +11262,11 @@ snapshots: transitivePeerDependencies: - aws-crt - '@aws-sdk/client-sso-oidc@3.572.0(@aws-sdk/client-sts@3.572.0)': + '@aws-sdk/client-sso-oidc@3.572.0': dependencies: '@aws-crypto/sha256-browser': 3.0.0 '@aws-crypto/sha256-js': 3.0.0 - '@aws-sdk/client-sts': 3.572.0 + '@aws-sdk/client-sts': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) '@aws-sdk/core': 3.572.0 '@aws-sdk/credential-provider-node': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0)(@aws-sdk/client-sts@3.572.0) '@aws-sdk/middleware-host-header': 3.567.0 @@ -11308,7 +11305,6 @@ snapshots: '@smithy/util-utf8': 2.3.0 tslib: 2.6.2 transitivePeerDependencies: - - '@aws-sdk/client-sts' - aws-crt '@aws-sdk/client-sso@3.556.0': @@ -11496,11 +11492,11 @@ snapshots: transitivePeerDependencies: - aws-crt - '@aws-sdk/client-sts@3.572.0': + '@aws-sdk/client-sts@3.572.0(@aws-sdk/client-sso-oidc@3.572.0)': dependencies: '@aws-crypto/sha256-browser': 3.0.0 '@aws-crypto/sha256-js': 3.0.0 - '@aws-sdk/client-sso-oidc': 3.572.0(@aws-sdk/client-sts@3.572.0) + '@aws-sdk/client-sso-oidc': 3.572.0 '@aws-sdk/core': 3.572.0 '@aws-sdk/credential-provider-node': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0)(@aws-sdk/client-sts@3.572.0) '@aws-sdk/middleware-host-header': 3.567.0 @@ -11539,6 +11535,7 @@ snapshots: '@smithy/util-utf8': 2.3.0 tslib: 2.6.2 transitivePeerDependencies: + - '@aws-sdk/client-sso-oidc' - aws-crt '@aws-sdk/core@3.556.0': @@ -11689,7 +11686,7 @@ snapshots: '@aws-sdk/credential-provider-ini@3.572.0(@aws-sdk/client-sso-oidc@3.572.0)(@aws-sdk/client-sts@3.572.0)': dependencies: - '@aws-sdk/client-sts': 3.572.0 + '@aws-sdk/client-sts': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) '@aws-sdk/credential-provider-env': 3.568.0 '@aws-sdk/credential-provider-process': 3.572.0 '@aws-sdk/credential-provider-sso': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) @@ -11923,7 +11920,7 @@ snapshots: '@aws-sdk/credential-provider-web-identity@3.568.0(@aws-sdk/client-sts@3.572.0)': dependencies: - '@aws-sdk/client-sts': 3.572.0 + '@aws-sdk/client-sts': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) '@aws-sdk/types': 3.567.0 '@smithy/property-provider': 2.2.0 '@smithy/types': 2.12.0 @@ -12278,7 +12275,7 @@ snapshots: '@aws-sdk/token-providers@3.572.0(@aws-sdk/client-sso-oidc@3.572.0)': dependencies: - '@aws-sdk/client-sso-oidc': 3.572.0(@aws-sdk/client-sts@3.572.0) + '@aws-sdk/client-sso-oidc': 3.572.0 '@aws-sdk/types': 3.567.0 '@smithy/property-provider': 2.2.0 '@smithy/shared-ini-file-loader': 2.4.0 @@ -14907,10 +14904,6 @@ snapshots: dependencies: '@types/node': 24.13.3 - '@types/node@20.19.43': - dependencies: - undici-types: 6.21.0 - '@types/node@22.19.10': dependencies: undici-types: 6.21.0 diff --git a/scripts/services/docker/Dockerfile.connectors_worker b/scripts/services/docker/Dockerfile.connectors_worker index 8fb33c1aff..e17f7bcb4c 100644 --- a/scripts/services/docker/Dockerfile.connectors_worker +++ b/scripts/services/docker/Dockerfile.connectors_worker @@ -1,4 +1,4 @@ -FROM node:20-alpine as builder +FROM node:24-alpine as builder RUN apk add --no-cache python3 make g++ @@ -11,7 +11,7 @@ RUN pnpm fetch COPY ./services ./services RUN pnpm i --frozen-lockfile -FROM node:20-bookworm-slim as runner +FROM node:24-bookworm-slim as runner WORKDIR /usr/crowd/app RUN npm install -g corepack@latest && corepack enable pnpm && corepack prepare pnpm@9.15.0 --activate && apt update && apt install -y ca-certificates --no-install-recommends && rm -rf /var/lib/apt/lists/* diff --git a/services/apps/connectors_worker/package.json b/services/apps/connectors_worker/package.json index 5cb359796b..19e3c434fa 100644 --- a/services/apps/connectors_worker/package.json +++ b/services/apps/connectors_worker/package.json @@ -29,7 +29,7 @@ "typescript": "^5.6.3" }, "devDependencies": { - "@types/node": "^20.8.2", + "@types/node": "^24.13.3", "nodemon": "^3.0.1" } } diff --git a/services/libs/connectors/package.json b/services/libs/connectors/package.json index 5d697977c2..683a213372 100644 --- a/services/libs/connectors/package.json +++ b/services/libs/connectors/package.json @@ -9,7 +9,7 @@ "tsc-check": "tsc --noEmit" }, "devDependencies": { - "@types/node": "^20.8.2", + "@types/node": "^24.13.3", "typescript": "^5.6.3" }, "dependencies": { diff --git a/services/libs/connectors/src/http/client.ts b/services/libs/connectors/src/http/client.ts index 9761a8d489..4b779b2332 100644 --- a/services/libs/connectors/src/http/client.ts +++ b/services/libs/connectors/src/http/client.ts @@ -113,12 +113,13 @@ async function send( token: IPooledToken, ): Promise> { const applyToken = deps.applyToken ?? applyBearerToken + const authenticatedConfig = { + timeout: DEFAULT_TIMEOUT_MS, + ...applyToken(config, token), + validateStatus: () => true, + } try { - return await axios.request({ - timeout: DEFAULT_TIMEOUT_MS, - ...applyToken(config, token), - validateStatus: () => true, - }) + return await axios.request(authenticatedConfig) } catch (err) { throw new ProviderUnavailableError('no response from provider', { cause: err }) } From 47f6e53a9b6b11e4dd1be061d6e9c74846dcd275 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Thu, 27 Aug 2026 12:37:44 +0100 Subject: [PATCH 25/69] fix: park tokens on secondary rate limits and ignore stale reset headers Signed-off-by: Mouad BANI --- services/libs/connectors/src/http/client.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/services/libs/connectors/src/http/client.ts b/services/libs/connectors/src/http/client.ts index 4b779b2332..ed86f5d9ee 100644 --- a/services/libs/connectors/src/http/client.ts +++ b/services/libs/connectors/src/http/client.ts @@ -157,7 +157,10 @@ function isRateLimited(status: number, headers: Record): boolean if (status === 429) { return true } - return status === 403 && headers['x-ratelimit-remaining'] === '0' + return ( + status === 403 && + (headers['x-ratelimit-remaining'] === '0' || headers['retry-after'] !== undefined) + ) } function computeResumeAt(headers: Record): Date { @@ -171,7 +174,7 @@ function computeResumeAt(headers: Record): Date { return new Date(retryAfterDateMs) } const resetEpochSeconds = Number(headers['x-ratelimit-reset']) - if (Number.isFinite(resetEpochSeconds) && resetEpochSeconds > 0) { + if (Number.isFinite(resetEpochSeconds) && resetEpochSeconds * 1000 > Date.now()) { return new Date(resetEpochSeconds * 1000) } return new Date(Date.now() + RATE_LIMIT_FALLBACK_MS) From 26447464b748d96cb081d41cdc9cea3be055c4b8 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Thu, 27 Aug 2026 12:46:09 +0100 Subject: [PATCH 26/69] fix: rotate to a fresh token after quarantining on auth failure Signed-off-by: Mouad BANI --- services/libs/connectors/src/http/client.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/libs/connectors/src/http/client.ts b/services/libs/connectors/src/http/client.ts index ed86f5d9ee..1a2ea45417 100644 --- a/services/libs/connectors/src/http/client.ts +++ b/services/libs/connectors/src/http/client.ts @@ -102,6 +102,9 @@ async function attemptRequest( { tokenId: token.id, status: response.status }, 'token quarantined on auth failure', ) + if (allowTokenRotation) { + return attemptRequest(deps, config, false) + } } throw error From 57fc8af3c7eb5dbf5c790989f6dac2553503a27a Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Thu, 27 Aug 2026 14:06:58 +0100 Subject: [PATCH 27/69] feat: wire http client and emit into sync runs Signed-off-by: Mouad BANI --- .../src/activities/syncRunActivities.ts | 89 ++++++++++++++----- services/apps/connectors_worker/src/main.ts | 2 +- services/libs/connectors/src/types.ts | 3 + 3 files changed, 71 insertions(+), 23 deletions(-) diff --git a/services/apps/connectors_worker/src/activities/syncRunActivities.ts b/services/apps/connectors_worker/src/activities/syncRunActivities.ts index fd8390a381..c695a3178f 100644 --- a/services/apps/connectors_worker/src/activities/syncRunActivities.ts +++ b/services/apps/connectors_worker/src/activities/syncRunActivities.ts @@ -1,12 +1,22 @@ import { Context } from '@temporalio/activity' -import { getSync } from '@crowd/connectors' +import { + ConnectorError, + createEmit, + createHttpClient, + createTokenPool, + getCredential, + getSync, +} from '@crowd/connectors' import type { SyncContext } from '@crowd/connectors' import { getUnitById, recordRunFailure, recordRunSuccess, + rescheduleUnit, } from '@crowd/data-access-layer/src/connectors' +import { fetchIntegrationById } from '@crowd/data-access-layer/src/integrations' +import IntegrationStreamRepository from '@crowd/data-access-layer/src/old/apps/integration_stream_worker/integrationStream.repo' import { dbStoreQx } from '@crowd/data-access-layer/src/queryExecutor' import { getChildLogger } from '@crowd/logging' @@ -14,6 +24,7 @@ import { svc } from '../main' const DEAD_LETTER_AFTER = 5 const HEARTBEAT_INTERVAL_MS = 10_000 +const RATE_LIMIT_FALLBACK_MS = 60_000 export async function executeSync(unitId: string): Promise { const qx = dbStoreQx(svc.postgres.writer) @@ -32,22 +43,6 @@ export async function executeSync(unitId: string): Promise { channelName: unit.channelName, }) - let emittedCount = 0 - let committedWatermark = unit.watermark - - // POC only: emit collects counts in memory; the Kafka sink emitter arrives in M2 - const ctx: SyncContext = { - channel: { channelId: unit.channelId, channelName: unit.channelName }, - watermark: unit.watermark, - emit: async (records) => { - emittedCount += records.length - }, - commitWatermark: async (watermark) => { - committedWatermark = watermark - }, - log, - } - const heartbeat = setInterval(() => { try { activityContext.heartbeat() @@ -57,19 +52,69 @@ export async function executeSync(unitId: string): Promise { }, HEARTBEAT_INTERVAL_MS) try { + const integration = await fetchIntegrationById(qx, unit.integrationId) + if (!integration?.segmentId) { + throw new Error(`integration ${unit.integrationId} not found or has no segmentId`) + } + if (!svc.dataSinkWorkerEmitter) { + throw new Error('data sink worker emitter not initialized') + } + + // POC only: dummy has no credentials; token minting from the credential is M4 + if (unit.platform !== 'dummy') { + await getCredential(qx, unit.integrationId) + } + + const pool = createTokenPool(svc.redis, unit.platform, unit.integrationId) + const http = createHttpClient({ + acquireToken: pool.acquire, + parkToken: pool.park, + quarantineToken: pool.quarantine, + log, + }) + const sync = getSync(unit.platform, unit.syncName) + + const streamRepo = new IntegrationStreamRepository(svc.postgres.writer, log) + const emitter = createEmit({ + publishResult: streamRepo.publishExternalResult.bind(streamRepo), + sinkEmitter: svc.dataSinkWorkerEmitter, + unit, + segmentId: integration.segmentId, + schema: sync.schema, + log, + }) + + let committedWatermark = unit.watermark + + const ctx: SyncContext = { + channel: { channelId: unit.channelId, channelName: unit.channelName }, + watermark: unit.watermark, + emit: emitter.emit, + commitWatermark: async (watermark) => { + committedWatermark = watermark + }, + http, + log, + } + await sync.run(ctx) await recordRunSuccess(qx, unitId, { watermark: committedWatermark ?? {}, - emittedCount, + emittedCount: emitter.emittedCount(), }) - log.info({ emittedCount }, 'sync run succeeded') + log.info({ emittedCount: emitter.emittedCount() }, 'sync run succeeded') } catch (err) { + if (err instanceof ConnectorError && err.errorClass === 'provider.rate_limit') { + const resumeAt = err.options?.resumeAt ?? new Date(Date.now() + RATE_LIMIT_FALLBACK_MS) + await rescheduleUnit(qx, unitId, resumeAt) + log.info({ resumeAt }, 'sync run rate-limit parked') + return + } + const errorClass = err instanceof ConnectorError ? err.errorClass : 'unknown' log.error(err, 'sync run failed') - // POC only: everything unclassified is framework.internal; the 7-class - // error taxonomy arrives with the M2 HTTP client - await recordRunFailure(qx, unitId, 'framework.internal', DEAD_LETTER_AFTER) + await recordRunFailure(qx, unitId, errorClass, DEAD_LETTER_AFTER) throw err } finally { clearInterval(heartbeat) diff --git a/services/apps/connectors_worker/src/main.ts b/services/apps/connectors_worker/src/main.ts index ec10cb39a6..d35c038511 100644 --- a/services/apps/connectors_worker/src/main.ts +++ b/services/apps/connectors_worker/src/main.ts @@ -8,7 +8,7 @@ import { scheduleDispatcher } from './schedules/dispatcher' const config: Config = { envvars: [], producer: { - enabled: false, + enabled: true, }, temporal: { enabled: true, diff --git a/services/libs/connectors/src/types.ts b/services/libs/connectors/src/types.ts index b32728a781..a36d986d9e 100644 --- a/services/libs/connectors/src/types.ts +++ b/services/libs/connectors/src/types.ts @@ -2,6 +2,8 @@ import type { ZodType } from 'zod' import type { Logger } from '@crowd/logging' +import type { ConnectorHttp } from './http/client' + export interface Channel { channelId: string channelName: string @@ -25,6 +27,7 @@ export interface SyncContext { watermark: Record | null emit: (records: unknown[]) => Promise commitWatermark: (watermark: Record) => Promise + http: ConnectorHttp log: Logger } From 89eee852dd6a7994a7e41362b15f33537d768984 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Thu, 27 Aug 2026 15:22:54 +0100 Subject: [PATCH 28/69] fix: register dummy connector only in dev environments Signed-off-by: Mouad BANI --- services/apps/connectors_worker/src/main.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/services/apps/connectors_worker/src/main.ts b/services/apps/connectors_worker/src/main.ts index d35c038511..c0231969d9 100644 --- a/services/apps/connectors_worker/src/main.ts +++ b/services/apps/connectors_worker/src/main.ts @@ -1,5 +1,6 @@ import { Config } from '@crowd/archetype-standard' import { Options, ServiceWorker } from '@crowd/archetype-worker' +import { IS_DEV_ENV } from '@crowd/common' import { registerConnector } from '@crowd/connectors' import { dummyConnector } from '@crowd/connectors/src/testing/dummyConnector' @@ -31,7 +32,9 @@ export const svc = new ServiceWorker(config, options) // POC only: dummy connector drives the control-plane end-to-end; real // connectors register here starting with GitHub in M4 -registerConnector(dummyConnector) +if (IS_DEV_ENV) { + registerConnector(dummyConnector) +} setImmediate(async () => { await svc.init() From 871980beb199e0516c16cd966c2f9262842cdf1f Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Thu, 27 Aug 2026 16:10:31 +0100 Subject: [PATCH 29/69] fix: persist partial progress when a sync run parks on rate limit Signed-off-by: Mouad BANI --- .../src/activities/syncRunActivities.ts | 21 +++++++++++++---- .../src/connectors/syncUnits.ts | 23 +++++++++++++++++++ 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/services/apps/connectors_worker/src/activities/syncRunActivities.ts b/services/apps/connectors_worker/src/activities/syncRunActivities.ts index c695a3178f..2edd08b256 100644 --- a/services/apps/connectors_worker/src/activities/syncRunActivities.ts +++ b/services/apps/connectors_worker/src/activities/syncRunActivities.ts @@ -8,10 +8,11 @@ import { getCredential, getSync, } from '@crowd/connectors' -import type { SyncContext } from '@crowd/connectors' +import type { Emitter, SyncContext } from '@crowd/connectors' import { getUnitById, recordRunFailure, + recordRunPartial, recordRunSuccess, rescheduleUnit, } from '@crowd/data-access-layer/src/connectors' @@ -51,6 +52,9 @@ export async function executeSync(unitId: string): Promise { } }, HEARTBEAT_INTERVAL_MS) + let emitter: Emitter | null = null + let committedWatermark = unit.watermark + try { const integration = await fetchIntegrationById(qx, unit.integrationId) if (!integration?.segmentId) { @@ -76,7 +80,7 @@ export async function executeSync(unitId: string): Promise { const sync = getSync(unit.platform, unit.syncName) const streamRepo = new IntegrationStreamRepository(svc.postgres.writer, log) - const emitter = createEmit({ + emitter = createEmit({ publishResult: streamRepo.publishExternalResult.bind(streamRepo), sinkEmitter: svc.dataSinkWorkerEmitter, unit, @@ -85,8 +89,6 @@ export async function executeSync(unitId: string): Promise { log, }) - let committedWatermark = unit.watermark - const ctx: SyncContext = { channel: { channelId: unit.channelId, channelName: unit.channelName }, watermark: unit.watermark, @@ -108,7 +110,16 @@ export async function executeSync(unitId: string): Promise { } catch (err) { if (err instanceof ConnectorError && err.errorClass === 'provider.rate_limit') { const resumeAt = err.options?.resumeAt ?? new Date(Date.now() + RATE_LIMIT_FALLBACK_MS) - await rescheduleUnit(qx, unitId, resumeAt) + if (emitter && committedWatermark) { + await recordRunPartial( + qx, + unitId, + { watermark: committedWatermark, emittedCount: emitter.emittedCount() }, + resumeAt, + ) + } else { + await rescheduleUnit(qx, unitId, resumeAt) + } log.info({ resumeAt }, 'sync run rate-limit parked') return } diff --git a/services/libs/data-access-layer/src/connectors/syncUnits.ts b/services/libs/data-access-layer/src/connectors/syncUnits.ts index df20b7e5a5..fdc18b607b 100644 --- a/services/libs/data-access-layer/src/connectors/syncUnits.ts +++ b/services/libs/data-access-layer/src/connectors/syncUnits.ts @@ -91,6 +91,29 @@ export async function recordRunSuccess( ) } +export async function recordRunPartial( + qx: QueryExecutor, + id: string, + progress: ISyncRunSuccess, + resumeAt: Date, +): Promise { + await qx.result( + `UPDATE integration.sync_units + SET watermark = $(watermark)::jsonb, + "emittedCount" = $(emittedCount), + "nextRunAt" = $(resumeAt), + "lockedAt" = NULL, + "updatedAt" = now() + WHERE id = $(id)`, + { + id, + watermark: JSON.stringify(progress.watermark), + emittedCount: progress.emittedCount, + resumeAt, + }, + ) +} + export async function recordRunFailure( qx: QueryExecutor, id: string, From c19e860edbedd1c117b4b576834e22f4d50f4be7 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Thu, 27 Aug 2026 16:30:43 +0100 Subject: [PATCH 30/69] feat: add per-platform manifest hooks for tokens, budgets and response interpretation Signed-off-by: Mouad BANI --- .../src/activities/dispatcherActivities.ts | 7 +++++-- .../src/activities/syncRunActivities.ts | 14 +++++++++----- .../libs/connectors/src/pool/tokenPool.ts | 19 ++++++++++--------- services/libs/connectors/src/registry.ts | 4 ++++ services/libs/connectors/src/types.ts | 6 +++++- 5 files changed, 33 insertions(+), 17 deletions(-) diff --git a/services/apps/connectors_worker/src/activities/dispatcherActivities.ts b/services/apps/connectors_worker/src/activities/dispatcherActivities.ts index e64e2ed876..df13d44b7a 100644 --- a/services/apps/connectors_worker/src/activities/dispatcherActivities.ts +++ b/services/apps/connectors_worker/src/activities/dispatcherActivities.ts @@ -1,4 +1,4 @@ -import { createTokenPool, getSync } from '@crowd/connectors' +import { createTokenPool, findManifest, getSync } from '@crowd/connectors' import { claimDueUnits, rescheduleUnit } from '@crowd/data-access-layer/src/connectors' import type { IClaimedUnit } from '@crowd/data-access-layer/src/connectors' import { dbStoreQx } from '@crowd/data-access-layer/src/queryExecutor' @@ -23,7 +23,10 @@ export async function admitByBudget(units: IClaimedUnit[]): Promise { throw new Error('data sink worker emitter not initialized') } - // POC only: dummy has no credentials; token minting from the credential is M4 - if (unit.platform !== 'dummy') { - await getCredential(qx, unit.integrationId) + const manifest = getManifest(unit.platform) + const pool = createTokenPool(svc.redis, unit.platform, unit.integrationId, { + probeBudget: manifest.probeBudget, + }) + if (manifest.seedTokens) { + const credential = await getCredential(qx, unit.integrationId) + await manifest.seedTokens(credential, pool) } - - const pool = createTokenPool(svc.redis, unit.platform, unit.integrationId) const http = createHttpClient({ acquireToken: pool.acquire, parkToken: pool.park, quarantineToken: pool.quarantine, + interpretResponse: manifest.interpretResponse, log, }) diff --git a/services/libs/connectors/src/pool/tokenPool.ts b/services/libs/connectors/src/pool/tokenPool.ts index 53d74ce529..150c8a9aab 100644 --- a/services/libs/connectors/src/pool/tokenPool.ts +++ b/services/libs/connectors/src/pool/tokenPool.ts @@ -14,7 +14,7 @@ export interface BudgetSnapshot { export type BudgetProbe = ( platform: string, connectionId: string, - tokenId: string, + token: IPooledToken, ) => Promise // POC only: the probe is the single source of truth for budgets (github /rate_limit is free and @@ -122,14 +122,14 @@ export function createTokenPool( async function loadBucket( probe: BudgetProbe, - tokenId: string, + token: IPooledToken, nowMs: number, ): Promise { - const bucket = await readBucket(tokenId) + const bucket = await readBucket(token.id) if (!needsProbe(bucket, nowMs)) { return bucket } - const snapshot = await probe(platform, connectionId, tokenId) + const snapshot = await probe(platform, connectionId, token) if (!snapshot) { return null } @@ -139,7 +139,7 @@ export function createTokenPool( resetAtMs: snapshot.resetAt.getTime(), probedAtMs: nowMs, } - await redis.hSet(bucketKey(tokenId), { + await redis.hSet(bucketKey(token.id), { limit: String(probed.limit), remaining: String(probed.remaining), resetAt: String(probed.resetAtMs), @@ -161,7 +161,7 @@ export function createTokenPool( continue } if (probe) { - const bucket = await loadBucket(probe, id, nowMs) + const bucket = await loadBucket(probe, { id, value: state.value }, nowMs) if (bucket && bucket.remaining <= 0) { const resetAt = new Date(bucket.resetAtMs) if (!earliestBudgetResetAt || resetAt < earliestBudgetResetAt) { @@ -202,7 +202,7 @@ export function createTokenPool( if (!isHealthy(state, nowMs)) { continue } - const bucket = await loadBucket(probe, id, nowMs) + const bucket = await loadBucket(probe, { id, value: state.value }, nowMs) if (!bucket) { return true } @@ -225,8 +225,9 @@ export function createTokenPool( async seed(tokenId: string, value: string): Promise { const json = await redis.hGet(tokensKey, tokenId) - const state = json ? (JSON.parse(json) as ITokenState) : {} - await redis.hSet(tokensKey, tokenId, JSON.stringify({ ...state, value })) + const state = json ? (JSON.parse(json) as ITokenState) : null + const next = state && state.value === value ? { ...state, value } : { value } + await redis.hSet(tokensKey, tokenId, JSON.stringify(next)) await redis.zAdd(lruKey, { score: 0, value: tokenId }, { NX: true }) }, diff --git a/services/libs/connectors/src/registry.ts b/services/libs/connectors/src/registry.ts index 081751934b..ecd06e3314 100644 --- a/services/libs/connectors/src/registry.ts +++ b/services/libs/connectors/src/registry.ts @@ -6,6 +6,10 @@ export function registerConnector(manifest: Manifest): void { manifests.set(manifest.platform, manifest) } +export function findManifest(platform: string): Manifest | undefined { + return manifests.get(platform) +} + export function getManifest(platform: string): Manifest { const manifest = manifests.get(platform) if (!manifest) { diff --git a/services/libs/connectors/src/types.ts b/services/libs/connectors/src/types.ts index a36d986d9e..ada7994574 100644 --- a/services/libs/connectors/src/types.ts +++ b/services/libs/connectors/src/types.ts @@ -2,7 +2,8 @@ import type { ZodType } from 'zod' import type { Logger } from '@crowd/logging' -import type { ConnectorHttp } from './http/client' +import type { ConnectorHttp, ResponseInterpreter } from './http/client' +import type { BudgetProbe, TokenPool } from './pool/tokenPool' export interface Channel { channelId: string @@ -42,4 +43,7 @@ export interface Manifest { platform: string syncs: SyncDefinition[] discover: (credential: Credential) => Promise + seedTokens?: (credential: Credential, pool: TokenPool) => Promise + probeBudget?: BudgetProbe + interpretResponse?: ResponseInterpreter } From 84a0801dac0e82ba5759731580245ebdd5268142 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Thu, 27 Aug 2026 16:56:36 +0100 Subject: [PATCH 31/69] fix: admit units with no healthy tokens so seeding can revive the pool Signed-off-by: Mouad BANI --- services/libs/connectors/src/pool/tokenPool.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/services/libs/connectors/src/pool/tokenPool.ts b/services/libs/connectors/src/pool/tokenPool.ts index 150c8a9aab..f1852e8d87 100644 --- a/services/libs/connectors/src/pool/tokenPool.ts +++ b/services/libs/connectors/src/pool/tokenPool.ts @@ -194,14 +194,12 @@ export function createTokenPool( } const nowMs = Date.now() const states = await readStates() - if (states.size === 0) { + const healthy = [...states.entries()].filter(([, state]) => isHealthy(state, nowMs)) + if (healthy.length === 0) { return true } let pooledRemaining = 0 - for (const [id, state] of states.entries()) { - if (!isHealthy(state, nowMs)) { - continue - } + for (const [id, state] of healthy) { const bucket = await loadBucket(probe, { id, value: state.value }, nowMs) if (!bucket) { return true From 8c78a773dd25efe94d05237be9a90470eccf84c2 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Thu, 27 Aug 2026 17:41:37 +0100 Subject: [PATCH 32/69] feat: add github app auth, graphql client and connector skeleton Signed-off-by: Mouad BANI --- pnpm-lock.yaml | 33 ++-- services/apps/connectors_worker/src/main.ts | 6 +- services/libs/connectors/package.json | 3 + .../src/connectors/github/appToken.ts | 48 ++++++ .../src/connectors/github/budget.ts | 31 ++++ .../src/connectors/github/discover.ts | 35 ++++ .../connectors/src/connectors/github/gql.ts | 27 ++++ .../connectors/src/connectors/github/index.ts | 15 ++ .../src/connectors/github/interpret.ts | 17 ++ .../src/connectors/github/mappers/member.ts | 149 ++++++++++++++++++ .../src/connectors/github/schemas.ts | 65 ++++++++ 11 files changed, 415 insertions(+), 14 deletions(-) create mode 100644 services/libs/connectors/src/connectors/github/appToken.ts create mode 100644 services/libs/connectors/src/connectors/github/budget.ts create mode 100644 services/libs/connectors/src/connectors/github/discover.ts create mode 100644 services/libs/connectors/src/connectors/github/gql.ts create mode 100644 services/libs/connectors/src/connectors/github/index.ts create mode 100644 services/libs/connectors/src/connectors/github/interpret.ts create mode 100644 services/libs/connectors/src/connectors/github/mappers/member.ts create mode 100644 services/libs/connectors/src/connectors/github/schemas.ts diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index eddf82e1e4..199d6c5d85 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2278,6 +2278,9 @@ importers: '@crowd/data-access-layer': specifier: workspace:* version: link:../data-access-layer + '@crowd/integrations': + specifier: workspace:* + version: link:../integrations '@crowd/logging': specifier: workspace:* version: link:../logging @@ -2290,10 +2293,16 @@ importers: axios: specifier: ^1.6.8 version: 1.19.0 + jsonwebtoken: + specifier: ^9.0.0 + version: 9.0.3 zod: specifier: ^3.22.0 version: 3.25.76 devDependencies: + '@types/jsonwebtoken': + specifier: ^9.0.0 + version: 9.0.6 '@types/node': specifier: ^24.13.3 version: 24.13.3 @@ -7473,11 +7482,11 @@ packages: glob@6.0.4: resolution: {integrity: sha512-MKZeRNyYZAVVVG1oZeLaWie1uweH40m9AZwIwxyPbTSX4hHrVYSzLg0Ro5Z5R7XKkIX+Cc6oD1rqeDJnwsB8/A==} - deprecated: Glob versions prior to v9 are no longer supported + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me glob@7.2.3: resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==} - deprecated: Glob versions prior to v9 are no longer supported + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me global-directory@4.0.1: resolution: {integrity: sha512-wHTUcDUoZ1H5/0iVqEudYW4/kAlN5cZ3j/bXn0Dpbizl9iaUVeWSHqiOjsgk6OW2bkLclbBjzewBz6weQ1zA2Q==} @@ -11053,8 +11062,8 @@ snapshots: dependencies: '@aws-crypto/sha256-browser': 3.0.0 '@aws-crypto/sha256-js': 3.0.0 - '@aws-sdk/client-sso-oidc': 3.572.0 - '@aws-sdk/client-sts': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) + '@aws-sdk/client-sso-oidc': 3.572.0(@aws-sdk/client-sts@3.572.0) + '@aws-sdk/client-sts': 3.572.0 '@aws-sdk/core': 3.572.0 '@aws-sdk/credential-provider-node': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0)(@aws-sdk/client-sts@3.572.0) '@aws-sdk/middleware-host-header': 3.567.0 @@ -11262,11 +11271,11 @@ snapshots: transitivePeerDependencies: - aws-crt - '@aws-sdk/client-sso-oidc@3.572.0': + '@aws-sdk/client-sso-oidc@3.572.0(@aws-sdk/client-sts@3.572.0)': dependencies: '@aws-crypto/sha256-browser': 3.0.0 '@aws-crypto/sha256-js': 3.0.0 - '@aws-sdk/client-sts': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) + '@aws-sdk/client-sts': 3.572.0 '@aws-sdk/core': 3.572.0 '@aws-sdk/credential-provider-node': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0)(@aws-sdk/client-sts@3.572.0) '@aws-sdk/middleware-host-header': 3.567.0 @@ -11305,6 +11314,7 @@ snapshots: '@smithy/util-utf8': 2.3.0 tslib: 2.6.2 transitivePeerDependencies: + - '@aws-sdk/client-sts' - aws-crt '@aws-sdk/client-sso@3.556.0': @@ -11492,11 +11502,11 @@ snapshots: transitivePeerDependencies: - aws-crt - '@aws-sdk/client-sts@3.572.0(@aws-sdk/client-sso-oidc@3.572.0)': + '@aws-sdk/client-sts@3.572.0': dependencies: '@aws-crypto/sha256-browser': 3.0.0 '@aws-crypto/sha256-js': 3.0.0 - '@aws-sdk/client-sso-oidc': 3.572.0 + '@aws-sdk/client-sso-oidc': 3.572.0(@aws-sdk/client-sts@3.572.0) '@aws-sdk/core': 3.572.0 '@aws-sdk/credential-provider-node': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0)(@aws-sdk/client-sts@3.572.0) '@aws-sdk/middleware-host-header': 3.567.0 @@ -11535,7 +11545,6 @@ snapshots: '@smithy/util-utf8': 2.3.0 tslib: 2.6.2 transitivePeerDependencies: - - '@aws-sdk/client-sso-oidc' - aws-crt '@aws-sdk/core@3.556.0': @@ -11686,7 +11695,7 @@ snapshots: '@aws-sdk/credential-provider-ini@3.572.0(@aws-sdk/client-sso-oidc@3.572.0)(@aws-sdk/client-sts@3.572.0)': dependencies: - '@aws-sdk/client-sts': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) + '@aws-sdk/client-sts': 3.572.0 '@aws-sdk/credential-provider-env': 3.568.0 '@aws-sdk/credential-provider-process': 3.572.0 '@aws-sdk/credential-provider-sso': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) @@ -11920,7 +11929,7 @@ snapshots: '@aws-sdk/credential-provider-web-identity@3.568.0(@aws-sdk/client-sts@3.572.0)': dependencies: - '@aws-sdk/client-sts': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) + '@aws-sdk/client-sts': 3.572.0 '@aws-sdk/types': 3.567.0 '@smithy/property-provider': 2.2.0 '@smithy/types': 2.12.0 @@ -12275,7 +12284,7 @@ snapshots: '@aws-sdk/token-providers@3.572.0(@aws-sdk/client-sso-oidc@3.572.0)': dependencies: - '@aws-sdk/client-sso-oidc': 3.572.0 + '@aws-sdk/client-sso-oidc': 3.572.0(@aws-sdk/client-sts@3.572.0) '@aws-sdk/types': 3.567.0 '@smithy/property-provider': 2.2.0 '@smithy/shared-ini-file-loader': 2.4.0 diff --git a/services/apps/connectors_worker/src/main.ts b/services/apps/connectors_worker/src/main.ts index c0231969d9..4e0810eb7f 100644 --- a/services/apps/connectors_worker/src/main.ts +++ b/services/apps/connectors_worker/src/main.ts @@ -2,6 +2,7 @@ import { Config } from '@crowd/archetype-standard' import { Options, ServiceWorker } from '@crowd/archetype-worker' import { IS_DEV_ENV } from '@crowd/common' import { registerConnector } from '@crowd/connectors' +import { githubConnector } from '@crowd/connectors/src/connectors/github' import { dummyConnector } from '@crowd/connectors/src/testing/dummyConnector' import { scheduleDispatcher } from './schedules/dispatcher' @@ -30,8 +31,9 @@ const options: Options = { export const svc = new ServiceWorker(config, options) -// POC only: dummy connector drives the control-plane end-to-end; real -// connectors register here starting with GitHub in M4 +registerConnector(githubConnector) + +// POC only: dummy connector drives the control-plane end-to-end in dev if (IS_DEV_ENV) { registerConnector(dummyConnector) } diff --git a/services/libs/connectors/package.json b/services/libs/connectors/package.json index 683a213372..55a9d42958 100644 --- a/services/libs/connectors/package.json +++ b/services/libs/connectors/package.json @@ -9,6 +9,7 @@ "tsc-check": "tsc --noEmit" }, "devDependencies": { + "@types/jsonwebtoken": "^9.0.0", "@types/node": "^24.13.3", "typescript": "^5.6.3" }, @@ -16,10 +17,12 @@ "@crowd/common": "workspace:*", "@crowd/common_services": "workspace:*", "@crowd/data-access-layer": "workspace:*", + "@crowd/integrations": "workspace:*", "@crowd/logging": "workspace:*", "@crowd/redis": "workspace:*", "@crowd/types": "workspace:*", "axios": "^1.6.8", + "jsonwebtoken": "^9.0.0", "zod": "^3.22.0" } } diff --git a/services/libs/connectors/src/connectors/github/appToken.ts b/services/libs/connectors/src/connectors/github/appToken.ts new file mode 100644 index 0000000000..71ad54e187 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/appToken.ts @@ -0,0 +1,48 @@ +import axios from 'axios' +import * as jwt from 'jsonwebtoken' + +import type { TokenPool } from '../../pool/tokenPool' +import type { Credential } from '../../types' + +const GITHUB_API_VERSION = '2022-11-28' + +export async function mintInstallationToken( + credential: Credential, + installationId: string, +): Promise<{ token: string; expiresAt: string }> { + const now = Math.floor(Date.now() / 1000) + const appJwt = jwt.sign( + { iat: now - 60, exp: now + 600, iss: credential.data.appId }, + credential.data.privateKey, + { algorithm: 'RS256' }, + ) + + // POC only: minting cannot go through ConnectorHttp — it feeds the pool the client draws from + const response = await axios.post( + `https://api.github.com/app/installations/${installationId}/access_tokens`, + {}, + { + headers: { + Authorization: `Bearer ${appJwt}`, + Accept: 'application/vnd.github+json', + 'X-GitHub-Api-Version': GITHUB_API_VERSION, + }, + }, + ) + + return { token: response.data.token, expiresAt: response.data.expires_at } +} + +export function requireInstallationId(): string { + const installationId = process.env.CROWD_GITHUB_INSTALLATION_ID + if (!installationId) { + throw new Error('missing CROWD_GITHUB_INSTALLATION_ID environment variable') + } + return installationId +} + +export async function seedGithubTokens(credential: Credential, pool: TokenPool): Promise { + const installationId = requireInstallationId() + const { token } = await mintInstallationToken(credential, installationId) + await pool.seed(`install-${installationId}`, token) +} diff --git a/services/libs/connectors/src/connectors/github/budget.ts b/services/libs/connectors/src/connectors/github/budget.ts new file mode 100644 index 0000000000..4d770712ca --- /dev/null +++ b/services/libs/connectors/src/connectors/github/budget.ts @@ -0,0 +1,31 @@ +import axios from 'axios' + +import type { BudgetProbe } from '../../pool/tokenPool' + +interface RateLimitResource { + limit: number + remaining: number + reset: number +} + +export const probeGithubBudget: BudgetProbe = async (_platform, _connectionId, token) => { + try { + const response = await axios.get('https://api.github.com/rate_limit', { + headers: { + Authorization: `Bearer ${token.value}`, + Accept: 'application/vnd.github+json', + }, + }) + const graphql = response.data?.resources?.graphql as RateLimitResource | undefined + if (!graphql) { + return null + } + return { + limit: graphql.limit, + remaining: graphql.remaining, + resetAt: new Date(graphql.reset * 1000), + } + } catch { + return null + } +} diff --git a/services/libs/connectors/src/connectors/github/discover.ts b/services/libs/connectors/src/connectors/github/discover.ts new file mode 100644 index 0000000000..c57487d05b --- /dev/null +++ b/services/libs/connectors/src/connectors/github/discover.ts @@ -0,0 +1,35 @@ +import axios from 'axios' + +import type { Channel, Credential } from '../../types' + +import { mintInstallationToken, requireInstallationId } from './appToken' + +const PER_PAGE = 100 + +interface InstallationRepo { + node_id: string + html_url: string +} + +export async function discoverRepos(credential: Credential): Promise { + const installationId = requireInstallationId() + const { token } = await mintInstallationToken(credential, installationId) + + const channels: Channel[] = [] + let page = 1 + for (;;) { + const response = await axios.get('https://api.github.com/installation/repositories', { + params: { per_page: PER_PAGE, page }, + headers: { + Authorization: `Bearer ${token}`, + Accept: 'application/vnd.github+json', + }, + }) + const repos = (response.data?.repositories ?? []) as InstallationRepo[] + channels.push(...repos.map((r) => ({ channelId: r.node_id, channelName: r.html_url }))) + if (repos.length < PER_PAGE) { + return channels + } + page += 1 + } +} diff --git a/services/libs/connectors/src/connectors/github/gql.ts b/services/libs/connectors/src/connectors/github/gql.ts new file mode 100644 index 0000000000..baa4d70856 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/gql.ts @@ -0,0 +1,27 @@ +import type { ConnectorHttp } from '../../http/client' +import { ProviderContractError } from '../../http/errors' + +interface GraphqlEnvelope { + data?: T + errors?: { type?: string; message?: string }[] +} + +export async function githubGraphql( + http: ConnectorHttp, + query: string, + variables: Record, +): Promise { + const body = await http.request>({ + method: 'post', + url: 'https://api.github.com/graphql', + data: { query, variables }, + }) + if (body.errors?.length) { + const details = body.errors.map((e) => `${e.type ?? 'ERROR'}: ${e.message ?? ''}`).join('; ') + throw new ProviderContractError(`github graphql errors: ${details}`) + } + if (!body.data) { + throw new ProviderContractError('github graphql response has no data') + } + return body.data +} diff --git a/services/libs/connectors/src/connectors/github/index.ts b/services/libs/connectors/src/connectors/github/index.ts new file mode 100644 index 0000000000..64f8fc975b --- /dev/null +++ b/services/libs/connectors/src/connectors/github/index.ts @@ -0,0 +1,15 @@ +import type { Manifest } from '../../types' + +import { seedGithubTokens } from './appToken' +import { probeGithubBudget } from './budget' +import { discoverRepos } from './discover' +import { interpretGithubResponse } from './interpret' + +export const githubConnector: Manifest = { + platform: 'github', + syncs: [], + discover: discoverRepos, + seedTokens: seedGithubTokens, + probeBudget: probeGithubBudget, + interpretResponse: interpretGithubResponse, +} diff --git a/services/libs/connectors/src/connectors/github/interpret.ts b/services/libs/connectors/src/connectors/github/interpret.ts new file mode 100644 index 0000000000..40c0c7b4eb --- /dev/null +++ b/services/libs/connectors/src/connectors/github/interpret.ts @@ -0,0 +1,17 @@ +import type { ResponseInterpreter } from '../../http/client' +import { RateLimitError } from '../../http/errors' + +interface GraphqlErrorEnvelope { + errors?: { type?: string }[] +} + +export const interpretGithubResponse: ResponseInterpreter = (response) => { + if (response.status !== 200) { + return null + } + const body = response.data as GraphqlErrorEnvelope | null + if (body?.errors?.some((e) => e.type === 'RATE_LIMITED')) { + return new RateLimitError('github graphql rate limited') + } + return null +} diff --git a/services/libs/connectors/src/connectors/github/mappers/member.ts b/services/libs/connectors/src/connectors/github/mappers/member.ts new file mode 100644 index 0000000000..cd8681781d --- /dev/null +++ b/services/libs/connectors/src/connectors/github/mappers/member.ts @@ -0,0 +1,149 @@ +import type { GithubMember, GithubOrganization } from '../schemas' + +export interface GithubOrgNode { + databaseId?: number | null + login: string + name?: string | null + url?: string | null + websiteUrl?: string | null + description?: string | null + avatarUrl?: string | null + twitterUsername?: string | null + location?: string | null +} + +export interface GithubUserNode { + __typename?: string + login?: string | null + name?: string | null + avatarUrl?: string | null + isHireable?: boolean | null + url?: string | null + bio?: string | null + company?: string | null + location?: string | null + email?: string | null + websiteUrl?: string | null + databaseId?: number | null + id?: string | number | null + organizations?: { nodes?: (GithubOrgNode | null)[] | null } | null +} + +// https://github.com/ghost +const GHOST_MEMBER: GithubMember = { + displayName: 'ghost', + identities: [ + { + platform: 'github', + type: 'username', + verified: true, + value: 'ghost', + sourceId: '10137', + }, + ], + attributes: { + avatarUrl: 'https://avatars.githubusercontent.com/u/10137?v=4', + bio: "Hi, I'm @ghost! I take the place of user accounts that have been deleted.\n:ghost:\n", + company: '', + isBot: false, + isHireable: false, + location: 'Nothing to see here, move along.', + url: 'https://github.com/ghost', + websiteUrl: '', + }, +} + +function toAttributes(user: GithubUserNode): GithubMember['attributes'] { + return { + isHireable: user.isHireable ?? false, + url: `https://github.com/${user.login ?? ''}`, + bio: user.bio ?? '', + location: user.location ?? '', + avatarUrl: user.avatarUrl ?? '', + company: user.company ?? '', + isBot: user.__typename === 'Bot', + websiteUrl: user.websiteUrl ?? '', + } +} + +function toOrganizations(user: GithubUserNode): GithubOrganization[] { + const nodes = user.organizations?.nodes?.filter((node) => node !== null) ?? [] + + return nodes.map((org) => { + const organization: GithubOrganization = { + displayName: org.name || org.login, + names: [org.name, org.login].filter((name): name is string => Boolean(name)), + description: org.description ?? null, + location: org.location ?? null, + logo: org.avatarUrl ?? null, + source: 'github', + identities: [ + { + platform: 'github', + type: 'username', + value: org.login, + verified: true, + sourceId: org.databaseId?.toString() ?? '', + }, + ], + } + + if (org.websiteUrl) { + organization.identities.push({ + platform: 'github', + type: 'primary-domain', + value: org.websiteUrl, + verified: false, + }) + } + + if (org.twitterUsername) { + organization.identities.push({ + platform: 'twitter', + type: 'username', + value: org.twitterUsername, + verified: false, + }) + } + + return organization + }) +} + +export function toMember(user: GithubUserNode | null | undefined): GithubMember { + if (!user || !user.login) { + return GHOST_MEMBER + } + + if ((user.__typename !== 'User' || !user.databaseId) && user.__typename !== 'Bot') { + return { + displayName: user.name || user.login, + identities: [ + { + platform: 'github', + type: 'username', + verified: true, + value: user.login, + sourceId: user.id?.toString() ?? '', + }, + ], + attributes: toAttributes(user), + organizations: [], + } + } + + return { + displayName: user.login, + identities: [ + { + platform: 'github', + type: 'username', + verified: true, + value: user.login, + sourceId: user.databaseId?.toString() ?? '', + }, + ], + attributes: toAttributes(user), + organizations: toOrganizations(user), + } +} diff --git a/services/libs/connectors/src/connectors/github/schemas.ts b/services/libs/connectors/src/connectors/github/schemas.ts new file mode 100644 index 0000000000..e8ad03f273 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/schemas.ts @@ -0,0 +1,65 @@ +import { z } from 'zod' + +import { GithubActivityType } from '@crowd/integrations' + +const identitySchema = z.object({ + platform: z.string(), + value: z.string(), + type: z.string(), + verified: z.boolean(), + sourceId: z.string(), +}) + +const memberAttributesSchema = z.object({ + isHireable: z.boolean(), + url: z.string(), + bio: z.string(), + location: z.string(), + avatarUrl: z.string(), + company: z.string(), + isBot: z.boolean(), + websiteUrl: z.string().optional(), +}) + +const organizationIdentitySchema = z.object({ + platform: z.string(), + type: z.string(), + value: z.string(), + verified: z.boolean(), + sourceId: z.string().optional(), +}) + +const organizationSchema = z.object({ + displayName: z.string(), + names: z.array(z.string()), + description: z.string().nullable(), + location: z.string().nullable(), + logo: z.string().nullable(), + source: z.string(), + identities: z.array(organizationIdentitySchema), +}) + +const memberSchema = z.object({ + displayName: z.string(), + identities: z.array(identitySchema).min(1), + attributes: memberAttributesSchema, + organizations: z.array(organizationSchema).optional(), +}) + +export const githubActivitySchema = z.object({ + type: z.nativeEnum(GithubActivityType), + timestamp: z.string(), + sourceId: z.string(), + sourceParentId: z.string().optional(), + score: z.number(), + title: z.string().optional(), + body: z.string().optional(), + url: z.string().optional(), + attributes: z.record(z.unknown()).optional(), + member: memberSchema, + objectMember: memberSchema.optional(), +}) + +export type GithubActivity = z.infer +export type GithubMember = z.infer +export type GithubOrganization = z.infer From 8935e49e5623c9e336277edf1b8f57c1c4ae190f Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 28 Aug 2026 11:17:45 +0100 Subject: [PATCH 33/69] fix: address review findings on timeouts, partial-run recency and sink error class Signed-off-by: Mouad BANI --- .../src/connectors/github/appToken.ts | 3 +++ .../src/connectors/github/budget.ts | 3 +++ .../src/connectors/github/discover.ts | 3 ++- services/libs/connectors/src/emit.ts | 19 +++++++++++++------ services/libs/connectors/src/http/client.ts | 2 +- services/libs/connectors/src/http/errors.ts | 2 +- .../src/connectors/syncUnits.ts | 1 + 7 files changed, 24 insertions(+), 9 deletions(-) diff --git a/services/libs/connectors/src/connectors/github/appToken.ts b/services/libs/connectors/src/connectors/github/appToken.ts index 71ad54e187..f1d0d7340c 100644 --- a/services/libs/connectors/src/connectors/github/appToken.ts +++ b/services/libs/connectors/src/connectors/github/appToken.ts @@ -6,6 +6,8 @@ import type { Credential } from '../../types' const GITHUB_API_VERSION = '2022-11-28' +export const GITHUB_REQUEST_TIMEOUT_MS = 30_000 + export async function mintInstallationToken( credential: Credential, installationId: string, @@ -27,6 +29,7 @@ export async function mintInstallationToken( Accept: 'application/vnd.github+json', 'X-GitHub-Api-Version': GITHUB_API_VERSION, }, + timeout: GITHUB_REQUEST_TIMEOUT_MS, }, ) diff --git a/services/libs/connectors/src/connectors/github/budget.ts b/services/libs/connectors/src/connectors/github/budget.ts index 4d770712ca..18b36abff7 100644 --- a/services/libs/connectors/src/connectors/github/budget.ts +++ b/services/libs/connectors/src/connectors/github/budget.ts @@ -2,6 +2,8 @@ import axios from 'axios' import type { BudgetProbe } from '../../pool/tokenPool' +import { GITHUB_REQUEST_TIMEOUT_MS } from './appToken' + interface RateLimitResource { limit: number remaining: number @@ -15,6 +17,7 @@ export const probeGithubBudget: BudgetProbe = async (_platform, _connectionId, t Authorization: `Bearer ${token.value}`, Accept: 'application/vnd.github+json', }, + timeout: GITHUB_REQUEST_TIMEOUT_MS, }) const graphql = response.data?.resources?.graphql as RateLimitResource | undefined if (!graphql) { diff --git a/services/libs/connectors/src/connectors/github/discover.ts b/services/libs/connectors/src/connectors/github/discover.ts index c57487d05b..5d86e3f4cc 100644 --- a/services/libs/connectors/src/connectors/github/discover.ts +++ b/services/libs/connectors/src/connectors/github/discover.ts @@ -2,7 +2,7 @@ import axios from 'axios' import type { Channel, Credential } from '../../types' -import { mintInstallationToken, requireInstallationId } from './appToken' +import { GITHUB_REQUEST_TIMEOUT_MS, mintInstallationToken, requireInstallationId } from './appToken' const PER_PAGE = 100 @@ -24,6 +24,7 @@ export async function discoverRepos(credential: Credential): Promise Authorization: `Bearer ${token}`, Accept: 'application/vnd.github+json', }, + timeout: GITHUB_REQUEST_TIMEOUT_MS, }) const repos = (response.data?.repositories ?? []) as InstallationRepo[] channels.push(...repos.map((r) => ({ channelId: r.node_id, channelName: r.html_url }))) diff --git a/services/libs/connectors/src/emit.ts b/services/libs/connectors/src/emit.ts index 2deef9d510..4a955f8c4c 100644 --- a/services/libs/connectors/src/emit.ts +++ b/services/libs/connectors/src/emit.ts @@ -40,12 +40,19 @@ export function createEmit(deps: EmitterDeps): Emitter { const payload = { ...parsed, channel: deps.unit.channelName } - const resultId = await deps.publishResult(deps.unit.integrationId, { - type: IntegrationResultType.ACTIVITY, - segmentId: deps.segmentId, - data: payload, - }) - await deps.sinkEmitter.triggerResultProcessing(resultId, resultId, false) + try { + const resultId = await deps.publishResult(deps.unit.integrationId, { + type: IntegrationResultType.ACTIVITY, + segmentId: deps.segmentId, + data: payload, + }) + await deps.sinkEmitter.triggerResultProcessing(resultId, resultId, false) + } catch (err) { + if (err instanceof ConnectorError) { + throw err + } + throw new ConnectorError('sink.rejected', 'failed to hand record to sink', { cause: err }) + } emitted += 1 } diff --git a/services/libs/connectors/src/http/client.ts b/services/libs/connectors/src/http/client.ts index 1a2ea45417..35aa18f690 100644 --- a/services/libs/connectors/src/http/client.ts +++ b/services/libs/connectors/src/http/client.ts @@ -150,7 +150,7 @@ function classifyResponse( resumeAt: computeResumeAt(headers), }) } - if (status >= 400) { + if (status >= 300) { return errorFromHttpStatus(status) } return null diff --git a/services/libs/connectors/src/http/errors.ts b/services/libs/connectors/src/http/errors.ts index 7318cd12a6..bce5f79591 100644 --- a/services/libs/connectors/src/http/errors.ts +++ b/services/libs/connectors/src/http/errors.ts @@ -77,7 +77,7 @@ export function errorFromHttpStatus( if (status >= 500) { return new ProviderUnavailableError(message ?? `provider returned status ${status}`, opts) } - if (status >= 400) { + if (status >= 300) { return new ProviderContractError(message ?? `provider returned status ${status}`, opts) } return new ConnectorError('unknown', message ?? `unexpected status ${status}`, opts) diff --git a/services/libs/data-access-layer/src/connectors/syncUnits.ts b/services/libs/data-access-layer/src/connectors/syncUnits.ts index fdc18b607b..3faaec5fe7 100644 --- a/services/libs/data-access-layer/src/connectors/syncUnits.ts +++ b/services/libs/data-access-layer/src/connectors/syncUnits.ts @@ -102,6 +102,7 @@ export async function recordRunPartial( SET watermark = $(watermark)::jsonb, "emittedCount" = $(emittedCount), "nextRunAt" = $(resumeAt), + "lastRunAt" = now(), "lockedAt" = NULL, "updatedAt" = now() WHERE id = $(id)`, From bbf15c5bad1a48a120cae49e1c10955d91906ab8 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 28 Aug 2026 12:08:47 +0100 Subject: [PATCH 34/69] feat: add github sync-unit seeding script to connectors worker Signed-off-by: Mouad BANI --- services/apps/connectors_worker/package.json | 3 +- .../src/bin/seed-github-sync-units.ts | 142 ++++++++++++++++++ .../src/connectors/github/discover.ts | 15 ++ 3 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 services/apps/connectors_worker/src/bin/seed-github-sync-units.ts diff --git a/services/apps/connectors_worker/package.json b/services/apps/connectors_worker/package.json index 19e3c434fa..50bb105c79 100644 --- a/services/apps/connectors_worker/package.json +++ b/services/apps/connectors_worker/package.json @@ -10,7 +10,8 @@ "lint": "npx eslint --ext .ts src --max-warnings=0", "format": "npx prettier --write \"src/**/*.ts\"", "format-check": "npx prettier --check .", - "tsc-check": "tsc --noEmit" + "tsc-check": "tsc --noEmit", + "script:seed-github-sync-units": "tsx src/bin/seed-github-sync-units.ts" }, "dependencies": { "@crowd/archetype-standard": "workspace:*", diff --git a/services/apps/connectors_worker/src/bin/seed-github-sync-units.ts b/services/apps/connectors_worker/src/bin/seed-github-sync-units.ts new file mode 100644 index 0000000000..9d999e1356 --- /dev/null +++ b/services/apps/connectors_worker/src/bin/seed-github-sync-units.ts @@ -0,0 +1,142 @@ +import { generateUUIDv4 } from '@crowd/common' +import type { Channel } from '@crowd/connectors' +import { getCredential } from '@crowd/connectors' +import { githubConnector } from '@crowd/connectors/src/connectors/github' +import { + mintInstallationToken, + requireInstallationId, +} from '@crowd/connectors/src/connectors/github/appToken' +import { resolveRepoChannel } from '@crowd/connectors/src/connectors/github/discover' +import type { SyncUnitUpsert } from '@crowd/data-access-layer/src/connectors' +import { upsertSyncUnits } from '@crowd/data-access-layer/src/connectors' +import { WRITE_DB_CONFIG, getDbConnection } from '@crowd/data-access-layer/src/database' +import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' +import { pgpQx } from '@crowd/data-access-layer/src/queryExecutor' +import { upsertRepository } from '@crowd/data-access-layer/src/repositories' +import { getServiceLogger } from '@crowd/logging' + +const log = getServiceLogger() + +interface RepoRef { + owner: string + name: string +} + +function usage(): never { + log.error( + 'Usage: seed-github-sync-units --integration-id [more repos ...]', + ) + process.exit(1) +} + +function parseRepoArg(arg: string): RepoRef { + const path = arg.replace(/^https:\/\/github\.com\//, '').replace(/\/+$/, '') + const parts = path.split('/') + if (parts.length !== 2 || !parts[0] || !parts[1]) { + throw new Error(`invalid repo argument "${arg}" - expected owner/repo or a github repo URL`) + } + return { owner: parts[0], name: parts[1] } +} + +function parseArgs(argv: string[]): { integrationId: string; repos: RepoRef[] } { + const flagIndex = argv.indexOf('--integration-id') + if (flagIndex === -1 || !argv[flagIndex + 1]) { + usage() + } + const integrationId = argv[flagIndex + 1] + const repoArgs = argv.filter((_, i) => i !== flagIndex && i !== flagIndex + 1) + if (repoArgs.length === 0) { + usage() + } + return { integrationId, repos: repoArgs.map(parseRepoArg) } +} + +async function getSegmentContext( + qx: QueryExecutor, + integrationId: string, +): Promise<{ segmentId: string; gitIntegrationId: string; insightsProjectId: string }> { + const integration: { segmentId: string | null } | null = await qx.selectOneOrNone( + `SELECT "segmentId" FROM integrations WHERE id = $(integrationId) AND "deletedAt" IS NULL`, + { integrationId }, + ) + if (!integration?.segmentId) { + throw new Error(`integration ${integrationId} not found or has no segmentId`) + } + + const gitIntegration: { id: string } | null = await qx.selectOneOrNone( + `SELECT id FROM integrations + WHERE "segmentId" = $(segmentId) AND platform = 'git' AND "deletedAt" IS NULL`, + { segmentId: integration.segmentId }, + ) + if (!gitIntegration) { + throw new Error(`git integration not found for segment ${integration.segmentId}`) + } + + const insightsProject: { id: string } | null = await qx.selectOneOrNone( + `SELECT id FROM "insightsProjects" + WHERE "segmentId" = $(segmentId) AND "deletedAt" IS NULL`, + { segmentId: integration.segmentId }, + ) + if (!insightsProject) { + throw new Error(`insights project not found for segment ${integration.segmentId}`) + } + + return { + segmentId: integration.segmentId, + gitIntegrationId: gitIntegration.id, + insightsProjectId: insightsProject.id, + } +} + +setImmediate(async () => { + try { + const { integrationId, repos } = parseArgs(process.argv.slice(2)) + + const db = await getDbConnection(WRITE_DB_CONFIG()) + const qx = pgpQx(db) + + const credential = await getCredential(qx, integrationId) + const installationId = requireInstallationId() + const { token, expiresAt } = await mintInstallationToken(credential, installationId) + log.info({ installationId, expiresAt }, 'github app auth verified, installation token minted') + + const segmentContext = await getSegmentContext(qx, integrationId) + + const channels: Channel[] = [] + for (const repo of repos) { + const channel = await resolveRepoChannel(token, repo.owner, repo.name) + const repositoryUpsert = await upsertRepository(qx, { + id: generateUUIDv4(), + url: channel.channelName, + segmentId: segmentContext.segmentId, + gitIntegrationId: segmentContext.gitIntegrationId, + sourceIntegrationId: integrationId, + insightsProjectId: segmentContext.insightsProjectId, + }) + log.info({ ...channel, repositoryUpsert }, 'repo resolved') + channels.push(channel) + } + + const syncNames = githubConnector.syncs.map((s) => s.name) + if (syncNames.length === 0) { + log.warn('github manifest has no syncs registered yet - seeded repositories only') + } + + const units: SyncUnitUpsert[] = channels.flatMap((channel) => + syncNames.map((syncName) => ({ + integrationId, + platform: githubConnector.platform, + channelId: channel.channelId, + channelName: channel.channelName, + syncName, + })), + ) + const unitsUpserted = await upsertSyncUnits(qx, units) + + log.info({ repos: channels.length, syncNames, unitsUpserted }, 'seeding complete') + process.exit(0) + } catch (err) { + log.error(err, 'seeding failed') + process.exit(1) + } +}) diff --git a/services/libs/connectors/src/connectors/github/discover.ts b/services/libs/connectors/src/connectors/github/discover.ts index 5d86e3f4cc..53ebe6b6ce 100644 --- a/services/libs/connectors/src/connectors/github/discover.ts +++ b/services/libs/connectors/src/connectors/github/discover.ts @@ -11,6 +11,21 @@ interface InstallationRepo { html_url: string } +export async function resolveRepoChannel( + token: string, + owner: string, + name: string, +): Promise { + const response = await axios.get(`https://api.github.com/repos/${owner}/${name}`, { + headers: { + Authorization: `Bearer ${token}`, + Accept: 'application/vnd.github+json', + }, + timeout: GITHUB_REQUEST_TIMEOUT_MS, + }) + return { channelId: response.data.node_id, channelName: response.data.html_url } +} + export async function discoverRepos(credential: Credential): Promise { const installationId = requireInstallationId() const { token } = await mintInstallationToken(credential, installationId) From 1219e4211814723ac66d951da5ad5654acba65f3 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 28 Aug 2026 12:33:46 +0100 Subject: [PATCH 35/69] chore: add connectors worker builder definition Signed-off-by: Mouad BANI --- scripts/builders/connectors-worker.env | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 scripts/builders/connectors-worker.env diff --git a/scripts/builders/connectors-worker.env b/scripts/builders/connectors-worker.env new file mode 100644 index 0000000000..e2858df2a7 --- /dev/null +++ b/scripts/builders/connectors-worker.env @@ -0,0 +1,4 @@ +DOCKERFILE="./services/docker/Dockerfile.connectors_worker" +CONTEXT="../" +REPO="sjc.ocir.io/axbydjxa5zuh/connectors-worker" +SERVICES="connectors-worker" From 266690b6a5f4adfff8feae075967532f3ce20790 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 28 Aug 2026 13:15:42 +0100 Subject: [PATCH 36/69] feat: resolve github app installation id with env and flag overrides Signed-off-by: Mouad BANI --- .../src/bin/seed-github-sync-units.ts | 38 +++++++++++------ .../src/connectors/github/appToken.ts | 41 +++++++++++++------ .../src/connectors/github/discover.ts | 4 +- 3 files changed, 56 insertions(+), 27 deletions(-) diff --git a/services/apps/connectors_worker/src/bin/seed-github-sync-units.ts b/services/apps/connectors_worker/src/bin/seed-github-sync-units.ts index 9d999e1356..04ff732d24 100644 --- a/services/apps/connectors_worker/src/bin/seed-github-sync-units.ts +++ b/services/apps/connectors_worker/src/bin/seed-github-sync-units.ts @@ -4,7 +4,7 @@ import { getCredential } from '@crowd/connectors' import { githubConnector } from '@crowd/connectors/src/connectors/github' import { mintInstallationToken, - requireInstallationId, + resolveInstallationId, } from '@crowd/connectors/src/connectors/github/appToken' import { resolveRepoChannel } from '@crowd/connectors/src/connectors/github/discover' import type { SyncUnitUpsert } from '@crowd/data-access-layer/src/connectors' @@ -24,11 +24,21 @@ interface RepoRef { function usage(): never { log.error( - 'Usage: seed-github-sync-units --integration-id [more repos ...]', + 'Usage: seed-github-sync-units --integration-id [--installation-id ] [more repos ...]', ) process.exit(1) } +function takeFlag(argv: string[], flag: string): string | undefined { + const flagIndex = argv.indexOf(flag) + if (flagIndex === -1) { + return undefined + } + const value = argv[flagIndex + 1] + argv.splice(flagIndex, 2) + return value +} + function parseRepoArg(arg: string): RepoRef { const path = arg.replace(/^https:\/\/github\.com\//, '').replace(/\/+$/, '') const parts = path.split('/') @@ -38,17 +48,18 @@ function parseRepoArg(arg: string): RepoRef { return { owner: parts[0], name: parts[1] } } -function parseArgs(argv: string[]): { integrationId: string; repos: RepoRef[] } { - const flagIndex = argv.indexOf('--integration-id') - if (flagIndex === -1 || !argv[flagIndex + 1]) { - usage() - } - const integrationId = argv[flagIndex + 1] - const repoArgs = argv.filter((_, i) => i !== flagIndex && i !== flagIndex + 1) - if (repoArgs.length === 0) { +function parseArgs(rawArgv: string[]): { + integrationId: string + installationId?: string + repos: RepoRef[] +} { + const argv = rawArgv.filter((arg) => arg !== '--') + const integrationId = takeFlag(argv, '--integration-id') + const installationId = takeFlag(argv, '--installation-id') + if (!integrationId || argv.length === 0) { usage() } - return { integrationId, repos: repoArgs.map(parseRepoArg) } + return { integrationId, installationId, repos: argv.map(parseRepoArg) } } async function getSegmentContext( @@ -90,13 +101,14 @@ async function getSegmentContext( setImmediate(async () => { try { - const { integrationId, repos } = parseArgs(process.argv.slice(2)) + const args = parseArgs(process.argv.slice(2)) + const { integrationId, repos } = args const db = await getDbConnection(WRITE_DB_CONFIG()) const qx = pgpQx(db) const credential = await getCredential(qx, integrationId) - const installationId = requireInstallationId() + const installationId = args.installationId ?? (await resolveInstallationId(credential)) const { token, expiresAt } = await mintInstallationToken(credential, installationId) log.info({ installationId, expiresAt }, 'github app auth verified, installation token minted') diff --git a/services/libs/connectors/src/connectors/github/appToken.ts b/services/libs/connectors/src/connectors/github/appToken.ts index f1d0d7340c..5906662530 100644 --- a/services/libs/connectors/src/connectors/github/appToken.ts +++ b/services/libs/connectors/src/connectors/github/appToken.ts @@ -8,24 +8,26 @@ const GITHUB_API_VERSION = '2022-11-28' export const GITHUB_REQUEST_TIMEOUT_MS = 30_000 -export async function mintInstallationToken( - credential: Credential, - installationId: string, -): Promise<{ token: string; expiresAt: string }> { +function mintAppJwt(credential: Credential): string { const now = Math.floor(Date.now() / 1000) - const appJwt = jwt.sign( + return jwt.sign( { iat: now - 60, exp: now + 600, iss: credential.data.appId }, credential.data.privateKey, { algorithm: 'RS256' }, ) +} +export async function mintInstallationToken( + credential: Credential, + installationId: string, +): Promise<{ token: string; expiresAt: string }> { // POC only: minting cannot go through ConnectorHttp — it feeds the pool the client draws from const response = await axios.post( `https://api.github.com/app/installations/${installationId}/access_tokens`, {}, { headers: { - Authorization: `Bearer ${appJwt}`, + Authorization: `Bearer ${mintAppJwt(credential)}`, Accept: 'application/vnd.github+json', 'X-GitHub-Api-Version': GITHUB_API_VERSION, }, @@ -36,16 +38,31 @@ export async function mintInstallationToken( return { token: response.data.token, expiresAt: response.data.expires_at } } -export function requireInstallationId(): string { - const installationId = process.env.CROWD_GITHUB_INSTALLATION_ID - if (!installationId) { - throw new Error('missing CROWD_GITHUB_INSTALLATION_ID environment variable') +// TODO(CM-1372): POC-only resolution; store the installation id per integration after the POC +export async function resolveInstallationId(credential: Credential): Promise { + const fromEnv = process.env.CROWD_GITHUB_INSTALLATION_ID + if (fromEnv) { + return fromEnv + } + + const response = await axios.get('https://api.github.com/app/installations', { + headers: { + Authorization: `Bearer ${mintAppJwt(credential)}`, + Accept: 'application/vnd.github+json', + 'X-GitHub-Api-Version': GITHUB_API_VERSION, + }, + timeout: GITHUB_REQUEST_TIMEOUT_MS, + }) + + const installations = response.data as { id: number }[] + if (installations.length === 0) { + throw new Error('github app has no installations') } - return installationId + return String(installations[0].id) } export async function seedGithubTokens(credential: Credential, pool: TokenPool): Promise { - const installationId = requireInstallationId() + const installationId = await resolveInstallationId(credential) const { token } = await mintInstallationToken(credential, installationId) await pool.seed(`install-${installationId}`, token) } diff --git a/services/libs/connectors/src/connectors/github/discover.ts b/services/libs/connectors/src/connectors/github/discover.ts index 53ebe6b6ce..8b20b1e2f9 100644 --- a/services/libs/connectors/src/connectors/github/discover.ts +++ b/services/libs/connectors/src/connectors/github/discover.ts @@ -2,7 +2,7 @@ import axios from 'axios' import type { Channel, Credential } from '../../types' -import { GITHUB_REQUEST_TIMEOUT_MS, mintInstallationToken, requireInstallationId } from './appToken' +import { GITHUB_REQUEST_TIMEOUT_MS, mintInstallationToken, resolveInstallationId } from './appToken' const PER_PAGE = 100 @@ -27,7 +27,7 @@ export async function resolveRepoChannel( } export async function discoverRepos(credential: Credential): Promise { - const installationId = requireInstallationId() + const installationId = await resolveInstallationId(credential) const { token } = await mintInstallationToken(credential, installationId) const channels: Channel[] = [] From 6e406aeab3f738b3d1da016bb6d55f9c4c57069e Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Mon, 31 Aug 2026 12:24:24 +0100 Subject: [PATCH 37/69] feat: add github connector syncs with full pagination and nango payload parity Signed-off-by: Mouad BANI --- .../src/connectors/github/graphql/fields.ts | 48 ++++ .../src/connectors/github/graphql/forks.ts | 65 +++++ .../src/connectors/github/graphql/issues.ts | 183 ++++++++++++++ .../github/graphql/pullRequestChildren.ts | 237 ++++++++++++++++++ .../connectors/github/graphql/pullRequests.ts | 213 ++++++++++++++++ .../connectors/src/connectors/github/index.ts | 17 +- .../src/connectors/github/mappers/commit.ts | 28 +++ .../src/connectors/github/mappers/fork.ts | 23 ++ .../src/connectors/github/mappers/issue.ts | 49 ++++ .../connectors/github/mappers/issueComment.ts | 21 ++ .../connectors/github/mappers/prComment.ts | 29 +++ .../connectors/github/mappers/pullRequest.ts | 159 ++++++++++++ .../github/mappers/reviewThreadComment.ts | 33 +++ .../src/connectors/github/paging.ts | 28 +++ .../src/connectors/github/prWalk.ts | 103 ++++++++ .../src/connectors/github/syncs/forks.ts | 48 ++++ .../connectors/github/syncs/issueComments.ts | 131 ++++++++++ .../src/connectors/github/syncs/issues.ts | 50 ++++ .../github/syncs/pullRequestComments.ts | 92 +++++++ .../github/syncs/pullRequestCommits.ts | 63 +++++ .../github/syncs/pullRequestReviewComments.ts | 164 ++++++++++++ .../connectors/github/syncs/pullRequests.ts | 74 ++++++ 22 files changed, 1857 insertions(+), 1 deletion(-) create mode 100644 services/libs/connectors/src/connectors/github/graphql/fields.ts create mode 100644 services/libs/connectors/src/connectors/github/graphql/forks.ts create mode 100644 services/libs/connectors/src/connectors/github/graphql/issues.ts create mode 100644 services/libs/connectors/src/connectors/github/graphql/pullRequestChildren.ts create mode 100644 services/libs/connectors/src/connectors/github/graphql/pullRequests.ts create mode 100644 services/libs/connectors/src/connectors/github/mappers/commit.ts create mode 100644 services/libs/connectors/src/connectors/github/mappers/fork.ts create mode 100644 services/libs/connectors/src/connectors/github/mappers/issue.ts create mode 100644 services/libs/connectors/src/connectors/github/mappers/issueComment.ts create mode 100644 services/libs/connectors/src/connectors/github/mappers/prComment.ts create mode 100644 services/libs/connectors/src/connectors/github/mappers/pullRequest.ts create mode 100644 services/libs/connectors/src/connectors/github/mappers/reviewThreadComment.ts create mode 100644 services/libs/connectors/src/connectors/github/paging.ts create mode 100644 services/libs/connectors/src/connectors/github/prWalk.ts create mode 100644 services/libs/connectors/src/connectors/github/syncs/forks.ts create mode 100644 services/libs/connectors/src/connectors/github/syncs/issueComments.ts create mode 100644 services/libs/connectors/src/connectors/github/syncs/issues.ts create mode 100644 services/libs/connectors/src/connectors/github/syncs/pullRequestComments.ts create mode 100644 services/libs/connectors/src/connectors/github/syncs/pullRequestCommits.ts create mode 100644 services/libs/connectors/src/connectors/github/syncs/pullRequestReviewComments.ts create mode 100644 services/libs/connectors/src/connectors/github/syncs/pullRequests.ts diff --git a/services/libs/connectors/src/connectors/github/graphql/fields.ts b/services/libs/connectors/src/connectors/github/graphql/fields.ts new file mode 100644 index 0000000000..6ee626154c --- /dev/null +++ b/services/libs/connectors/src/connectors/github/graphql/fields.ts @@ -0,0 +1,48 @@ +export const USER_FIELDS = ` + __typename + login + name + avatarUrl + isHireable + url + bio + company + location + email + websiteUrl + databaseId + organizations(first: 5) { + nodes { + __typename + databaseId + login + name + url + websiteUrl + description + avatarUrl + twitterUsername + location + } + } +` + +export const BOT_FIELDS = ` + __typename + login + avatarUrl + url + databaseId +` + +export const ORGANIZATION_FIELDS = ` + __typename + avatarUrl + databaseId + email + location + login + name + url + websiteUrl +` diff --git a/services/libs/connectors/src/connectors/github/graphql/forks.ts b/services/libs/connectors/src/connectors/github/graphql/forks.ts new file mode 100644 index 0000000000..5c700a5465 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/graphql/forks.ts @@ -0,0 +1,65 @@ +import type { GithubUserNode } from '../mappers/member' + +import { ORGANIZATION_FIELDS, USER_FIELDS } from './fields' + +export interface ForkNode { + id: string + name: string + createdAt: string + updatedAt: string + url: string + isFork: boolean + isInOrganization: boolean + parent: { + nameWithOwner: string + isFork: boolean + } | null + owner: GithubUserNode +} + +export interface ForksPage { + repository: { + forks: { + pageInfo: { + endCursor: string | null + hasNextPage: boolean + } + nodes: (ForkNode | null)[] + } + } +} + +export const FORKS_QUERY = ` + query ($owner: String!, $repo: String!, $first: Int!, $cursor: String) { + repository(owner: $owner, name: $repo) { + forks(first: $first, after: $cursor, orderBy: {field: CREATED_AT, direction: ASC}) { + pageInfo { + endCursor + hasNextPage + } + nodes { + id + name + createdAt + updatedAt + url + isFork + isInOrganization + parent { + nameWithOwner + isFork + } + owner { + login + ... on User { + ${USER_FIELDS} + } + ... on Organization { + ${ORGANIZATION_FIELDS} + } + } + } + } + } + } +` diff --git a/services/libs/connectors/src/connectors/github/graphql/issues.ts b/services/libs/connectors/src/connectors/github/graphql/issues.ts new file mode 100644 index 0000000000..087516979f --- /dev/null +++ b/services/libs/connectors/src/connectors/github/graphql/issues.ts @@ -0,0 +1,183 @@ +import type { GithubUserNode } from '../mappers/member' + +import { BOT_FIELDS, ORGANIZATION_FIELDS, USER_FIELDS } from './fields' + +export interface IssueTimelineNode { + __typename: string + actor?: GithubUserNode | null + createdAt?: string +} + +export interface IssueNode { + id: string + number: number + title: string + url: string + state: string + createdAt: string + updatedAt: string + bodyText: string + author: GithubUserNode | null + timelineItems: { + nodes: (IssueTimelineNode | null)[] + } +} + +export interface IssuesPage { + repository: { + issues: { + pageInfo: { + endCursor: string | null + hasNextPage: boolean + } + nodes: (IssueNode | null)[] + } + } +} + +export interface IssueCommentNode { + id: string + bodyText: string + url: string + createdAt: string + author: GithubUserNode | null +} + +export interface IssueCommentsConnection { + pageInfo: { + endCursor: string | null + hasNextPage: boolean + } + nodes: (IssueCommentNode | null)[] +} + +export interface IssueCommentsBatchPage { + nodes: ({ + id: string + number: number + comments: IssueCommentsConnection + } | null)[] +} + +export interface IssueCommentsPaginatedPage { + repository: { + issue: { + comments: IssueCommentsConnection + } | null + } +} + +const COMMENT_FIELDS = ` + id + bodyText + url + createdAt + author { + login + ... on User { + ${USER_FIELDS} + } + ... on Bot { + ${BOT_FIELDS} + } + ... on Organization { + ${ORGANIZATION_FIELDS} + } + } +` + +export const ISSUE_COMMENTS_QUERY = ` + query ($ids: [ID!]!, $first: Int!) { + nodes(ids: $ids) { + ... on Issue { + id + number + comments(first: $first) { + pageInfo { + endCursor + hasNextPage + } + nodes { + ${COMMENT_FIELDS} + } + } + } + } + } +` + +export const ISSUE_COMMENTS_PAGINATED_QUERY = ` + query ($owner: String!, $repo: String!, $issueNumber: Int!, $first: Int!, $cursor: String) { + repository(owner: $owner, name: $repo) { + issue(number: $issueNumber) { + comments(first: $first, after: $cursor) { + pageInfo { + endCursor + hasNextPage + } + nodes { + ${COMMENT_FIELDS} + } + } + } + } + } +` + +export const ISSUES_QUERY = ` + query ($owner: String!, $repo: String!, $first: Int!, $cursor: String, $since: DateTime) { + repository(owner: $owner, name: $repo) { + issues( + first: $first + after: $cursor + orderBy: {field: UPDATED_AT, direction: ASC} + states: [OPEN, CLOSED] + filterBy: {since: $since} + ) { + pageInfo { + endCursor + hasNextPage + } + nodes { + id + number + title + url + state + createdAt + updatedAt + bodyText + author { + login + ... on User { + ${USER_FIELDS} + } + ... on Bot { + ${BOT_FIELDS} + } + ... on Organization { + ${ORGANIZATION_FIELDS} + } + } + timelineItems(first: 10, itemTypes: [CLOSED_EVENT]) { + nodes { + __typename + ... on ClosedEvent { + createdAt + actor { + login + ... on User { + ${USER_FIELDS} + } + ... on Bot { + ${BOT_FIELDS} + } + } + } + } + } + } + } + } + } +` diff --git a/services/libs/connectors/src/connectors/github/graphql/pullRequestChildren.ts b/services/libs/connectors/src/connectors/github/graphql/pullRequestChildren.ts new file mode 100644 index 0000000000..8085e3f88a --- /dev/null +++ b/services/libs/connectors/src/connectors/github/graphql/pullRequestChildren.ts @@ -0,0 +1,237 @@ +import type { GithubUserNode } from '../mappers/member' + +import { BOT_FIELDS, ORGANIZATION_FIELDS, USER_FIELDS } from './fields' + +export interface PrCommentNode { + id: string + body: string + createdAt: string + url: string + author: GithubUserNode | null +} + +export interface PrCommentsConnection { + pageInfo: { + endCursor: string | null + hasNextPage: boolean + } + edges: ({ node: PrCommentNode | null } | null)[] +} + +export interface PrCommentsBatchPage { + nodes: ({ + id: string + number: number + comments: PrCommentsConnection + } | null)[] +} + +export interface ReviewThreadNode { + id: string + isResolved: boolean +} + +export interface ReviewThreadsBatchPage { + nodes: ({ + id: string + number: number + reviewThreads: { + pageInfo: { + endCursor: string | null + hasNextPage: boolean + } + edges: ({ node: ReviewThreadNode | null } | null)[] + } + } | null)[] +} + +export interface ThreadCommentNode { + id: string + body: string + createdAt: string + url: string + author: GithubUserNode | null +} + +export interface ThreadCommentsBatchPage { + nodes: ({ + id: string + isResolved: boolean + comments: { + pageInfo: { + endCursor: string | null + hasNextPage: boolean + } + edges: ({ node: ThreadCommentNode | null } | null)[] + } + } | null)[] +} + +export interface PrCommitNode { + commit: { + additions: number + deletions: number + parents: { totalCount: number } + id: string + oid: string + message: string + authoredDate: string + url: string + author: { + user: GithubUserNode | null + email: string | null + name: string | null + } | null + } +} + +export interface PrCommitsPage { + repository: { + pullRequest: { + commits: { + pageInfo: { + endCursor: string | null + hasNextPage: boolean + } + nodes: (PrCommitNode | null)[] + } + } | null + } +} + +export const COMMENTS_FOR_PRS_QUERY = ` + query ($ids: [ID!]!, $first: Int!, $after: String) { + nodes(ids: $ids) { + ... on PullRequest { + id + number + comments(first: $first, after: $after) { + pageInfo { + endCursor + hasNextPage + } + edges { + node { + id + body + createdAt + url + author { + __typename + login + ... on User { + ${USER_FIELDS} + } + ... on Bot { + ${BOT_FIELDS} + } + ... on Organization { + ${ORGANIZATION_FIELDS} + } + } + } + } + } + } + } + } +` + +export const REVIEW_THREADS_FOR_PRS_QUERY = ` + query ($ids: [ID!]!, $first: Int!, $after: String) { + nodes(ids: $ids) { + ... on PullRequest { + id + number + reviewThreads(first: $first, after: $after) { + pageInfo { + endCursor + hasNextPage + } + edges { + node { + id + isResolved + } + } + } + } + } + } +` + +export const COMMENTS_FOR_THREADS_QUERY = ` + query ($ids: [ID!]!, $first: Int!, $after: String) { + nodes(ids: $ids) { + ... on PullRequestReviewThread { + id + isResolved + comments(first: $first, after: $after) { + pageInfo { + endCursor + hasNextPage + } + edges { + node { + id + body + createdAt + url + author { + __typename + login + ... on User { + ${USER_FIELDS} + } + ... on Bot { + ${BOT_FIELDS} + } + ... on Organization { + ${ORGANIZATION_FIELDS} + } + } + } + } + } + } + } + } +` + +export const PR_COMMITS_QUERY = ` + query ($owner: String!, $repo: String!, $prNumber: Int!, $first: Int!, $cursor: String) { + repository(name: $repo, owner: $owner) { + pullRequest(number: $prNumber) { + commits(first: $first, after: $cursor) { + pageInfo { + endCursor + hasNextPage + } + nodes { + commit { + additions + deletions + parents { + totalCount + } + id + oid + message + authoredDate + url + author { + user { + ... on User { + ${USER_FIELDS} + } + } + email + name + } + } + } + } + } + } + } +` diff --git a/services/libs/connectors/src/connectors/github/graphql/pullRequests.ts b/services/libs/connectors/src/connectors/github/graphql/pullRequests.ts new file mode 100644 index 0000000000..5428c89f34 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/graphql/pullRequests.ts @@ -0,0 +1,213 @@ +import type { GithubUserNode } from '../mappers/member' + +import { BOT_FIELDS, ORGANIZATION_FIELDS, USER_FIELDS } from './fields' + +export interface PullRequestNode { + id: string + number: number + createdAt: string + updatedAt: string + url: string + title: string + body: string + state: string + authorAssociation: string + labels: { nodes: { name: string }[] | null } | null + additions: number + deletions: number + changedFiles: number + author: GithubUserNode | null +} + +export interface PullRequestsPage { + repository: { + pullRequests: { + pageInfo: { + endCursor: string | null + hasNextPage: boolean + } + nodes: (PullRequestNode | null)[] + } + } +} + +export interface PrTimelineItem { + __typename: string + id?: string + createdAt?: string + submittedAt?: string + state?: string + body?: string + actor?: GithubUserNode | null + author?: GithubUserNode | null + assignee?: GithubUserNode | null + requestedReviewer?: GithubUserNode | null +} + +export interface PrTimelineBatchPage { + nodes: ({ + id: string + timelineItems: { + nodes: (PrTimelineItem | null)[] + } + } | null)[] +} + +const ACTOR_FIELDS = ` + __typename + login + ... on User { + ${USER_FIELDS} + } + ... on Bot { + ${BOT_FIELDS} + } +` + +export const PULL_REQUESTS_QUERY = ` + query ($owner: String!, $repo: String!, $first: Int!, $cursor: String, $direction: OrderDirection!) { + repository(owner: $owner, name: $repo) { + pullRequests(first: $first, after: $cursor, orderBy: {field: UPDATED_AT, direction: $direction}) { + pageInfo { + endCursor + hasNextPage + } + nodes { + id + number + createdAt + updatedAt + url + title + body + state + authorAssociation + labels(first: $first) { + nodes { + name + } + } + additions + deletions + changedFiles + author { + __typename + login + ... on User { + ${USER_FIELDS} + } + ... on Bot { + ${BOT_FIELDS} + } + ... on Organization { + ${ORGANIZATION_FIELDS} + } + } + } + } + } + } +` + +export interface PrTimelinePage { + nodes: ({ + id: string + timelineItems: { + pageInfo: { + endCursor: string | null + hasNextPage: boolean + } + nodes: (PrTimelineItem | null)[] + } + } | null)[] +} + +export const PR_TIMELINE_QUERY = ` + query ($ids: [ID!]!, $first: Int!, $after: String) { + nodes(ids: $ids) { + ... on PullRequest { + id + timelineItems( + first: $first + after: $after + itemTypes: [PULL_REQUEST_REVIEW, MERGED_EVENT, ASSIGNED_EVENT, REVIEW_REQUESTED_EVENT, CLOSED_EVENT] + ) { + pageInfo { + endCursor + hasNextPage + } + nodes { + __typename + ... on ReviewRequestedEvent { + id + createdAt + actor { + ${ACTOR_FIELDS} + } + requestedReviewer { + __typename + ... on User { + login + ${USER_FIELDS} + } + ... on Bot { + login + ${BOT_FIELDS} + } + ... on Team { + name + id + } + } + } + ... on PullRequestReview { + id + state + submittedAt + body + author { + ${ACTOR_FIELDS} + } + } + ... on AssignedEvent { + id + createdAt + assignee { + __typename + ... on User { + login + ${USER_FIELDS} + } + ... on Bot { + login + ${BOT_FIELDS} + } + ... on Organization { + login + ${ORGANIZATION_FIELDS} + } + } + actor { + ${ACTOR_FIELDS} + } + } + ... on MergedEvent { + id + createdAt + actor { + ${ACTOR_FIELDS} + } + } + ... on ClosedEvent { + id + createdAt + actor { + ${ACTOR_FIELDS} + } + } + } + } + } + } + } +` diff --git a/services/libs/connectors/src/connectors/github/index.ts b/services/libs/connectors/src/connectors/github/index.ts index 64f8fc975b..dc6abea660 100644 --- a/services/libs/connectors/src/connectors/github/index.ts +++ b/services/libs/connectors/src/connectors/github/index.ts @@ -4,10 +4,25 @@ import { seedGithubTokens } from './appToken' import { probeGithubBudget } from './budget' import { discoverRepos } from './discover' import { interpretGithubResponse } from './interpret' +import { forksSync } from './syncs/forks' +import { issueCommentsSync } from './syncs/issueComments' +import { issuesSync } from './syncs/issues' +import { pullRequestCommentsSync } from './syncs/pullRequestComments' +import { pullRequestCommitsSync } from './syncs/pullRequestCommits' +import { pullRequestReviewCommentsSync } from './syncs/pullRequestReviewComments' +import { pullRequestsSync } from './syncs/pullRequests' export const githubConnector: Manifest = { platform: 'github', - syncs: [], + syncs: [ + forksSync, + issuesSync, + issueCommentsSync, + pullRequestsSync, + pullRequestCommentsSync, + pullRequestReviewCommentsSync, + pullRequestCommitsSync, + ], discover: discoverRepos, seedTokens: seedGithubTokens, probeBudget: probeGithubBudget, diff --git a/services/libs/connectors/src/connectors/github/mappers/commit.ts b/services/libs/connectors/src/connectors/github/mappers/commit.ts new file mode 100644 index 0000000000..188e3316ee --- /dev/null +++ b/services/libs/connectors/src/connectors/github/mappers/commit.ts @@ -0,0 +1,28 @@ +import { GITHUB_GRID, GithubActivityType } from '@crowd/integrations' + +import type { PrCommitNode } from '../graphql/pullRequestChildren' +import type { GithubActivity } from '../schemas' + +import { toMember } from './member' + +const DEFAULT_TIMESTAMP = '1970-01-01T00:00:00Z' + +export function toCommit(commit: PrCommitNode['commit'], prId: string): GithubActivity { + return { + type: GithubActivityType.AUTHORED_COMMIT, + timestamp: commit.authoredDate ?? DEFAULT_TIMESTAMP, + sourceId: commit.oid, + sourceParentId: prId, + score: GITHUB_GRID[GithubActivityType.AUTHORED_COMMIT].score, + body: commit.message, + url: commit.url ?? '', + attributes: { + insertions: commit.additions, + deletions: commit.deletions, + // nango quirk kept for exact-match: lines = additions - deletions, not the sum + lines: commit.additions - commit.deletions, + isMerge: commit.parents.totalCount > 1, + }, + member: toMember(commit.author?.user), + } +} diff --git a/services/libs/connectors/src/connectors/github/mappers/fork.ts b/services/libs/connectors/src/connectors/github/mappers/fork.ts new file mode 100644 index 0000000000..8bd9dd1760 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/mappers/fork.ts @@ -0,0 +1,23 @@ +import { GITHUB_GRID, GithubActivityType } from '@crowd/integrations' + +import type { ForkNode } from '../graphql/forks' +import type { GithubActivity } from '../schemas' + +import { toMember } from './member' + +export function toFork(node: ForkNode): GithubActivity { + return { + type: GithubActivityType.FORK, + timestamp: node.createdAt, + sourceId: node.id, + score: GITHUB_GRID[GithubActivityType.FORK].score, + attributes: node.parent?.isFork + ? { + isForkByOrg: node.isInOrganization, + directParent: node.parent.nameWithOwner, + isIndirectFork: String(node.parent.isFork), + } + : {}, + member: toMember(node.owner), + } +} diff --git a/services/libs/connectors/src/connectors/github/mappers/issue.ts b/services/libs/connectors/src/connectors/github/mappers/issue.ts new file mode 100644 index 0000000000..9fc7182705 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/mappers/issue.ts @@ -0,0 +1,49 @@ +import { GITHUB_GRID, GithubActivityType } from '@crowd/integrations' + +import type { IssueNode } from '../graphql/issues' +import type { GithubActivity } from '../schemas' + +import { toMember } from './member' + +const DEFAULT_TIMESTAMP = '1970-01-01T00:00:00Z' + +export function toIssueActivities(issue: IssueNode): GithubActivity[] { + const activities: GithubActivity[] = [ + { + type: GithubActivityType.ISSUE_OPENED, + timestamp: issue.createdAt ?? DEFAULT_TIMESTAMP, + sourceId: issue.id, + score: GITHUB_GRID[GithubActivityType.ISSUE_OPENED].score, + title: issue.title || '', + body: issue.bodyText || '', + url: issue.url, + attributes: { state: 'open', issueNumber: issue.number }, + member: toMember(issue.author), + }, + ] + + const closedEvent = issue.timelineItems.nodes.find((node) => node?.__typename === 'ClosedEvent') + if (issue.state === 'CLOSED' && closedEvent) { + const closedAt = closedEvent.createdAt ?? issue.updatedAt ?? DEFAULT_TIMESTAMP + activities.push({ + type: GithubActivityType.ISSUE_CLOSED, + timestamp: closedAt, + sourceId: `gen-CE_${issue.id}_${closedEvent.actor?.login ?? issue.author?.login}_${new Date( + closedAt, + ).toISOString()}`, + sourceParentId: issue.id, + score: GITHUB_GRID[GithubActivityType.ISSUE_CLOSED].score, + title: issue.title || '', + body: issue.bodyText || '', + url: issue.url, + attributes: { + state: 'closed', + issueNumber: issue.number, + ...(closedEvent.actor ? { closedBy: closedEvent.actor.login } : {}), + }, + member: toMember(closedEvent.actor ?? issue.author), + }) + } + + return activities +} diff --git a/services/libs/connectors/src/connectors/github/mappers/issueComment.ts b/services/libs/connectors/src/connectors/github/mappers/issueComment.ts new file mode 100644 index 0000000000..5919779a0e --- /dev/null +++ b/services/libs/connectors/src/connectors/github/mappers/issueComment.ts @@ -0,0 +1,21 @@ +import { GITHUB_GRID, GithubActivityType } from '@crowd/integrations' + +import type { IssueCommentNode, IssueNode } from '../graphql/issues' +import type { GithubActivity } from '../schemas' + +import { toMember } from './member' + +export function toIssueComment(comment: IssueCommentNode, issue: IssueNode): GithubActivity { + return { + type: GithubActivityType.ISSUE_COMMENT, + timestamp: comment.createdAt, + sourceId: comment.id, + sourceParentId: issue.id, + score: GITHUB_GRID[GithubActivityType.ISSUE_COMMENT].score, + title: issue.title || '', + body: comment.bodyText || '', + url: comment.url || issue.url, + attributes: { state: issue.state.toLowerCase() }, + member: toMember(comment.author), + } +} diff --git a/services/libs/connectors/src/connectors/github/mappers/prComment.ts b/services/libs/connectors/src/connectors/github/mappers/prComment.ts new file mode 100644 index 0000000000..e00d80c721 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/mappers/prComment.ts @@ -0,0 +1,29 @@ +import { GITHUB_GRID, GithubActivityType } from '@crowd/integrations' + +import type { PrCommentNode } from '../graphql/pullRequestChildren' +import type { PullRequestNode } from '../graphql/pullRequests' +import type { GithubActivity } from '../schemas' + +import { toMember } from './member' + +export function toPrComment(comment: PrCommentNode, pullRequest: PullRequestNode): GithubActivity { + return { + type: GithubActivityType.PULL_REQUEST_COMMENT, + timestamp: comment.createdAt, + sourceId: comment.id, + sourceParentId: pullRequest.id, + score: GITHUB_GRID[GithubActivityType.PULL_REQUEST_COMMENT].score, + title: pullRequest.title || '', + body: comment.body || '', + url: pullRequest.url, + attributes: { + state: pullRequest.state, + additions: pullRequest.additions, + deletions: pullRequest.deletions, + changedFiles: pullRequest.changedFiles, + authorAssociation: pullRequest.authorAssociation, + labels: pullRequest.labels?.nodes?.map((label) => label.name) || [], + }, + member: toMember(comment.author), + } +} diff --git a/services/libs/connectors/src/connectors/github/mappers/pullRequest.ts b/services/libs/connectors/src/connectors/github/mappers/pullRequest.ts new file mode 100644 index 0000000000..d6dad713fc --- /dev/null +++ b/services/libs/connectors/src/connectors/github/mappers/pullRequest.ts @@ -0,0 +1,159 @@ +import { GITHUB_GRID, GithubActivityType } from '@crowd/integrations' + +import type { PrTimelineItem, PullRequestNode } from '../graphql/pullRequests' +import type { GithubActivity } from '../schemas' + +import { toMember } from './member' + +const DEFAULT_TIMESTAMP = '1970-01-01T00:00:00Z' + +function prAttributes(pullRequest: PullRequestNode): Record { + return { + additions: pullRequest.additions, + deletions: pullRequest.deletions, + changedFiles: pullRequest.changedFiles, + authorAssociation: pullRequest.authorAssociation, + labels: pullRequest.labels?.nodes?.map((label) => label.name) ?? [], + } +} + +function toTimelineActivity( + item: PrTimelineItem, + pullRequest: PullRequestNode, +): GithubActivity | null { + switch (item.__typename) { + case 'AssignedEvent': { + if (!item.assignee || !item.id) { + return null + } + const timestamp = item.createdAt ?? DEFAULT_TIMESTAMP + return { + type: GithubActivityType.PULL_REQUEST_ASSIGNED, + timestamp, + sourceId: `gen-AE_${pullRequest.id}_${item.actor?.login}_${item.assignee?.login}_${new Date(timestamp).toISOString()}`, + sourceParentId: pullRequest.id, + score: GITHUB_GRID[GithubActivityType.PULL_REQUEST_ASSIGNED].score, + title: pullRequest.title ?? '', + body: pullRequest.body ?? '', + url: pullRequest.url, + attributes: { state: pullRequest.state.toLowerCase(), ...prAttributes(pullRequest) }, + member: toMember(item.actor), + objectMember: toMember(item.assignee), + } + } + case 'ReviewRequestedEvent': { + if (!item.requestedReviewer || !item.id) { + return null + } + const timestamp = item.createdAt ?? DEFAULT_TIMESTAMP + return { + type: GithubActivityType.PULL_REQUEST_REVIEW_REQUESTED, + timestamp, + sourceId: `gen-RRE_${pullRequest.id}_${item.actor?.login}_${item.requestedReviewer?.login}_${new Date(timestamp).toISOString()}`, + sourceParentId: pullRequest.id, + score: GITHUB_GRID[GithubActivityType.PULL_REQUEST_REVIEW_REQUESTED].score, + title: pullRequest.title ?? '', + body: pullRequest.body ?? '', + url: pullRequest.url, + attributes: { state: pullRequest.state.toLowerCase(), ...prAttributes(pullRequest) }, + member: toMember(item.actor), + objectMember: toMember(item.requestedReviewer), + } + } + case 'PullRequestReview': { + if (!item.id) { + return null + } + const timestamp = item.submittedAt ?? DEFAULT_TIMESTAMP + return { + type: GithubActivityType.PULL_REQUEST_REVIEWED, + timestamp, + sourceId: `gen-PRR_${pullRequest.id}_${item.author?.login}_${new Date(timestamp).toISOString()}`, + sourceParentId: pullRequest.id, + score: GITHUB_GRID[GithubActivityType.PULL_REQUEST_REVIEWED].score, + title: pullRequest.title ?? '', + body: item.body ?? pullRequest.body ?? '', + url: pullRequest.url, + attributes: { + reviewState: item.state, + state: pullRequest.state.toLowerCase(), + ...prAttributes(pullRequest), + }, + member: toMember(item.author), + } + } + case 'MergedEvent': { + if (!item.id) { + return null + } + const timestamp = item.createdAt ?? DEFAULT_TIMESTAMP + return { + type: GithubActivityType.PULL_REQUEST_MERGED, + timestamp, + sourceId: `gen-ME_${pullRequest.id}_${item.actor?.login}_${new Date(timestamp).toISOString()}`, + sourceParentId: pullRequest.id, + score: GITHUB_GRID[GithubActivityType.PULL_REQUEST_MERGED].score, + title: pullRequest.title ?? '', + body: pullRequest.body ?? '', + url: pullRequest.url, + attributes: { state: 'merged', ...prAttributes(pullRequest) }, + member: toMember(item.actor), + } + } + case 'ClosedEvent': { + if (!item.id) { + return null + } + const timestamp = item.createdAt ?? DEFAULT_TIMESTAMP + return { + type: GithubActivityType.PULL_REQUEST_CLOSED, + timestamp, + sourceId: `gen-CE_${pullRequest.id}_${item.actor?.login}_${new Date(timestamp).toISOString()}`, + sourceParentId: pullRequest.id, + score: GITHUB_GRID[GithubActivityType.PULL_REQUEST_CLOSED].score, + title: pullRequest.title ?? '', + body: pullRequest.body ?? '', + url: pullRequest.url, + attributes: { state: 'closed', ...prAttributes(pullRequest) }, + member: toMember(item.actor), + } + } + default: + return null + } +} + +export function toPullRequestActivities( + pullRequest: PullRequestNode, + timelineItems: (PrTimelineItem | null)[], +): GithubActivity[] { + if (!pullRequest || !pullRequest.id) { + return [] + } + + const activities: GithubActivity[] = [ + { + type: GithubActivityType.PULL_REQUEST_OPENED, + timestamp: pullRequest.createdAt ?? DEFAULT_TIMESTAMP, + sourceId: pullRequest.id, + score: GITHUB_GRID[GithubActivityType.PULL_REQUEST_OPENED].score, + title: pullRequest.title ?? '', + body: pullRequest.body ?? '', + url: pullRequest.url, + attributes: { state: 'open', ...prAttributes(pullRequest) }, + member: toMember(pullRequest.author), + }, + ] + + for (const item of timelineItems) { + if (!item) { + continue + } + const activity = toTimelineActivity(item, pullRequest) + if (activity) { + activities.push(activity) + } + } + + return activities +} diff --git a/services/libs/connectors/src/connectors/github/mappers/reviewThreadComment.ts b/services/libs/connectors/src/connectors/github/mappers/reviewThreadComment.ts new file mode 100644 index 0000000000..0df20bf395 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/mappers/reviewThreadComment.ts @@ -0,0 +1,33 @@ +import { GITHUB_GRID, GithubActivityType } from '@crowd/integrations' + +import type { ReviewThreadNode, ThreadCommentNode } from '../graphql/pullRequestChildren' +import type { PullRequestNode } from '../graphql/pullRequests' +import type { GithubActivity } from '../schemas' + +import { toMember } from './member' + +export function toReviewThreadComment( + comment: ThreadCommentNode, + thread: ReviewThreadNode, + pullRequest: PullRequestNode, +): GithubActivity { + return { + type: GithubActivityType.PULL_REQUEST_REVIEW_THREAD_COMMENT, + timestamp: comment.createdAt, + sourceId: comment.id, + sourceParentId: pullRequest.id, + score: GITHUB_GRID[GithubActivityType.PULL_REQUEST_REVIEW_THREAD_COMMENT].score, + title: pullRequest.title || '', + body: `[Thread ${thread.isResolved ? 'RESOLVED' : 'OPEN'}] ${comment.body || ''}`, + url: comment.url || pullRequest.url, + attributes: { + state: pullRequest.state, + additions: pullRequest.additions, + deletions: pullRequest.deletions, + changedFiles: pullRequest.changedFiles, + authorAssociation: pullRequest.authorAssociation, + labels: pullRequest.labels?.nodes?.map((label) => label.name) || [], + }, + member: toMember(comment.author), + } +} diff --git a/services/libs/connectors/src/connectors/github/paging.ts b/services/libs/connectors/src/connectors/github/paging.ts new file mode 100644 index 0000000000..7f55e500af --- /dev/null +++ b/services/libs/connectors/src/connectors/github/paging.ts @@ -0,0 +1,28 @@ +export interface GithubWatermark { + phase: 'backfill' | 'incremental' + since: string | null + cursor: string | null +} + +export const MAX_PAGES_PER_RUN = Number.POSITIVE_INFINITY +export const PAGE_SIZE = 100 + +export function readWatermark(raw: Record | null): GithubWatermark { + if (raw && (raw.phase === 'backfill' || raw.phase === 'incremental')) { + return { + phase: raw.phase, + since: typeof raw.since === 'string' ? raw.since : null, + cursor: typeof raw.cursor === 'string' ? raw.cursor : null, + } + } + return { phase: 'backfill', since: null, cursor: null } +} + +export function parseRepoChannel(channelName: string): { owner: string; repo: string } { + const path = channelName.replace(/^https:\/\/github\.com\//, '').replace(/\/+$/, '') + const parts = path.split('/') + if (parts.length !== 2 || !parts[0] || !parts[1]) { + throw new Error(`invalid github repo channel name: ${channelName}`) + } + return { owner: parts[0], repo: parts[1] } +} diff --git a/services/libs/connectors/src/connectors/github/prWalk.ts b/services/libs/connectors/src/connectors/github/prWalk.ts new file mode 100644 index 0000000000..c083e89fe1 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/prWalk.ts @@ -0,0 +1,103 @@ +import type { SyncContext } from '../../types' + +import { githubGraphql } from './gql' +import type { PullRequestNode, PullRequestsPage } from './graphql/pullRequests' +import { PULL_REQUESTS_QUERY } from './graphql/pullRequests' +import { MAX_PAGES_PER_RUN, parseRepoChannel, readWatermark } from './paging' + +export const PR_PAGE_SIZE = 50 + +export type PrPageHandler = (prs: PullRequestNode[], sinceDate: Date | null) => Promise + +async function runBackfill( + ctx: SyncContext, + owner: string, + repo: string, + processPrs: PrPageHandler, +): Promise { + const watermark = readWatermark(ctx.watermark) + let cursor = watermark.cursor + let since = watermark.since + + for (let page = 0; page < MAX_PAGES_PER_RUN; page++) { + const data = await githubGraphql(ctx.http, PULL_REQUESTS_QUERY, { + owner, + repo, + first: PR_PAGE_SIZE, + cursor, + direction: 'ASC', + }) + + const { pageInfo, nodes } = data.repository.pullRequests + const pullRequests = nodes.filter((node): node is PullRequestNode => node !== null) + + if (pullRequests.length > 0) { + await processPrs(pullRequests, null) + since = pullRequests[pullRequests.length - 1].updatedAt + } + + if (!pageInfo.hasNextPage) { + await ctx.commitWatermark({ phase: 'incremental', since, cursor: null }) + return + } + + cursor = pageInfo.endCursor + await ctx.commitWatermark({ phase: 'backfill', since, cursor }) + } +} + +async function runIncremental( + ctx: SyncContext, + owner: string, + repo: string, + since: string, + processPrs: PrPageHandler, +): Promise { + const sinceDate = new Date(since) + let cursor: string | null = null + let newSince: string | null = null + + for (let page = 0; page < MAX_PAGES_PER_RUN; page++) { + const data = await githubGraphql(ctx.http, PULL_REQUESTS_QUERY, { + owner, + repo, + first: PR_PAGE_SIZE, + cursor, + direction: 'DESC', + }) + + const { pageInfo, nodes } = data.repository.pullRequests + const pullRequests = nodes.filter((node): node is PullRequestNode => node !== null) + + if (newSince === null && pullRequests.length > 0) { + newSince = pullRequests[0].updatedAt + } + + const fresh = pullRequests.filter((pr) => new Date(pr.updatedAt) > sinceDate) + if (fresh.length > 0) { + await processPrs(fresh, sinceDate) + } + + const reachedSince = fresh.length < pullRequests.length + if (reachedSince || !pageInfo.hasNextPage) { + await ctx.commitWatermark({ phase: 'incremental', since: newSince ?? since, cursor: null }) + return + } + + cursor = pageInfo.endCursor + } +} + +export async function runDualPhasePrSync( + ctx: SyncContext, + processPrs: PrPageHandler, +): Promise { + const { owner, repo } = parseRepoChannel(ctx.channel.channelName) + const watermark = readWatermark(ctx.watermark) + + if (watermark.phase === 'incremental' && watermark.since) { + await runIncremental(ctx, owner, repo, watermark.since, processPrs) + return + } + await runBackfill(ctx, owner, repo, processPrs) +} diff --git a/services/libs/connectors/src/connectors/github/syncs/forks.ts b/services/libs/connectors/src/connectors/github/syncs/forks.ts new file mode 100644 index 0000000000..288a78aac6 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/syncs/forks.ts @@ -0,0 +1,48 @@ +import type { SyncContext, SyncDefinition } from '../../../types' +import { githubGraphql } from '../gql' +import type { ForkNode, ForksPage } from '../graphql/forks' +import { FORKS_QUERY } from '../graphql/forks' +import { toFork } from '../mappers/fork' +import { MAX_PAGES_PER_RUN, PAGE_SIZE, parseRepoChannel, readWatermark } from '../paging' +import { githubActivitySchema } from '../schemas' + +async function runForksSync(ctx: SyncContext): Promise { + const { owner, repo } = parseRepoChannel(ctx.channel.channelName) + const watermark = readWatermark(ctx.watermark) + + let since = watermark.since + let cursor = watermark.cursor + + for (let page = 0; page < MAX_PAGES_PER_RUN; page++) { + const data = await githubGraphql(ctx.http, FORKS_QUERY, { + owner, + repo, + first: PAGE_SIZE, + cursor, + }) + + const { pageInfo, nodes } = data.repository.forks + const forks = nodes + .filter((node): node is ForkNode => node !== null) + .filter((node) => !since || node.createdAt > since) + + if (forks.length > 0) { + await ctx.emit(forks.map(toFork)) + since = forks[forks.length - 1].createdAt + } + + cursor = pageInfo.hasNextPage ? pageInfo.endCursor : null + await ctx.commitWatermark({ phase: 'incremental', since, cursor }) + + if (!pageInfo.hasNextPage) { + return + } + } +} + +export const forksSync: SyncDefinition = { + name: 'forks', + cadenceMinutes: 360, + schema: githubActivitySchema, + run: runForksSync, +} diff --git a/services/libs/connectors/src/connectors/github/syncs/issueComments.ts b/services/libs/connectors/src/connectors/github/syncs/issueComments.ts new file mode 100644 index 0000000000..753b68cd98 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/syncs/issueComments.ts @@ -0,0 +1,131 @@ +import type { SyncContext, SyncDefinition } from '../../../types' +import { githubGraphql } from '../gql' +import type { + IssueCommentNode, + IssueCommentsBatchPage, + IssueCommentsPaginatedPage, + IssueNode, + IssuesPage, +} from '../graphql/issues' +import { + ISSUES_QUERY, + ISSUE_COMMENTS_PAGINATED_QUERY, + ISSUE_COMMENTS_QUERY, +} from '../graphql/issues' +import { toIssueComment } from '../mappers/issueComment' +import { MAX_PAGES_PER_RUN, PAGE_SIZE, parseRepoChannel, readWatermark } from '../paging' +import { githubActivitySchema } from '../schemas' + +const ISSUE_BATCH_SIZE = 15 +const COMMENTS_BATCH_PAGE_SIZE = 25 +// api is highly unreliable when paginating comments per issue, so keep the page small +// (nango workaround: nango-integrations/github/syncs/issue-comments.ts) +const COMMENTS_PAGINATED_PAGE_SIZE = 5 + +async function runIssueCommentsSync(ctx: SyncContext): Promise { + const { owner, repo } = parseRepoChannel(ctx.channel.channelName) + const watermark = readWatermark(ctx.watermark) + + const querySince = watermark.since + const sinceDate = querySince ? new Date(querySince) : null + let since = watermark.since + let cursor: string | null = null + + const emitComments = async ( + nodes: (IssueCommentNode | null)[], + issue: IssueNode, + ): Promise => { + const comments = nodes + .filter((node): node is IssueCommentNode => node !== null && Boolean(node.id)) + .filter((node) => !sinceDate || new Date(node.createdAt) >= sinceDate) + if (comments.length > 0) { + await ctx.emit(comments.map((comment) => toIssueComment(comment, issue))) + } + } + + const drainRemainingComments = async (issue: IssueNode, startCursor: string | null) => { + let commentsCursor = startCursor + let hasMore = true + while (hasMore) { + const data = await githubGraphql( + ctx.http, + ISSUE_COMMENTS_PAGINATED_QUERY, + { + owner, + repo, + issueNumber: issue.number, + first: COMMENTS_PAGINATED_PAGE_SIZE, + cursor: commentsCursor, + }, + ) + const comments = data.repository.issue?.comments + if (!comments?.nodes) { + return + } + await emitComments(comments.nodes, issue) + hasMore = comments.pageInfo.hasNextPage + commentsCursor = comments.pageInfo.endCursor + } + } + + for (let page = 0; page < MAX_PAGES_PER_RUN; page++) { + const data = await githubGraphql(ctx.http, ISSUES_QUERY, { + owner, + repo, + first: PAGE_SIZE, + cursor, + since: querySince, + }) + + const { pageInfo, nodes } = data.repository.issues + const issues = nodes.filter((node): node is IssueNode => node !== null) + + for (let i = 0; i < issues.length; i += ISSUE_BATCH_SIZE) { + const batch = issues.slice(i, i + ISSUE_BATCH_SIZE) + const batchData = await githubGraphql( + ctx.http, + ISSUE_COMMENTS_QUERY, + { + ids: batch.map((issue) => issue.id), + first: COMMENTS_BATCH_PAGE_SIZE, + }, + ) + + const toPaginate: { issue: IssueNode; commentsCursor: string | null }[] = [] + for (const node of batchData.nodes) { + if (!node?.comments?.nodes) { + continue + } + const issue = batch.find((candidate) => candidate.id === node.id) + if (!issue) { + continue + } + await emitComments(node.comments.nodes, issue) + if (node.comments.pageInfo.hasNextPage) { + toPaginate.push({ issue, commentsCursor: node.comments.pageInfo.endCursor }) + } + } + + for (const { issue, commentsCursor } of toPaginate) { + await drainRemainingComments(issue, commentsCursor) + } + } + + if (issues.length > 0) { + since = issues[issues.length - 1].updatedAt + } + cursor = pageInfo.hasNextPage ? pageInfo.endCursor : null + await ctx.commitWatermark({ phase: 'incremental', since, cursor: null }) + + if (!pageInfo.hasNextPage) { + return + } + } +} + +export const issueCommentsSync: SyncDefinition = { + name: 'issue-comments', + cadenceMinutes: 60, + schema: githubActivitySchema, + run: runIssueCommentsSync, +} diff --git a/services/libs/connectors/src/connectors/github/syncs/issues.ts b/services/libs/connectors/src/connectors/github/syncs/issues.ts new file mode 100644 index 0000000000..2f208f1ff1 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/syncs/issues.ts @@ -0,0 +1,50 @@ +import type { SyncContext, SyncDefinition } from '../../../types' +import { githubGraphql } from '../gql' +import type { IssueNode, IssuesPage } from '../graphql/issues' +import { ISSUES_QUERY } from '../graphql/issues' +import { toIssueActivities } from '../mappers/issue' +import { MAX_PAGES_PER_RUN, PAGE_SIZE, parseRepoChannel, readWatermark } from '../paging' +import { githubActivitySchema } from '../schemas' + +async function runIssuesSync(ctx: SyncContext): Promise { + const { owner, repo } = parseRepoChannel(ctx.channel.channelName) + const watermark = readWatermark(ctx.watermark) + + // filterBy.since is coupled to the cursor's result set: advancing it between + // pages would invalidate cursors, so the query keeps the run-start since. + const querySince = watermark.since + let since = watermark.since + let cursor: string | null = null + + for (let page = 0; page < MAX_PAGES_PER_RUN; page++) { + const data = await githubGraphql(ctx.http, ISSUES_QUERY, { + owner, + repo, + first: PAGE_SIZE, + cursor, + since: querySince, + }) + + const { pageInfo, nodes } = data.repository.issues + const issues = nodes.filter((node): node is IssueNode => node !== null) + + if (issues.length > 0) { + await ctx.emit(issues.flatMap(toIssueActivities)) + since = issues[issues.length - 1].updatedAt + } + + cursor = pageInfo.hasNextPage ? pageInfo.endCursor : null + await ctx.commitWatermark({ phase: 'incremental', since, cursor: null }) + + if (!pageInfo.hasNextPage) { + return + } + } +} + +export const issuesSync: SyncDefinition = { + name: 'issues', + cadenceMinutes: 60, + schema: githubActivitySchema, + run: runIssuesSync, +} diff --git a/services/libs/connectors/src/connectors/github/syncs/pullRequestComments.ts b/services/libs/connectors/src/connectors/github/syncs/pullRequestComments.ts new file mode 100644 index 0000000000..e0e7c9cdca --- /dev/null +++ b/services/libs/connectors/src/connectors/github/syncs/pullRequestComments.ts @@ -0,0 +1,92 @@ +import type { SyncContext, SyncDefinition } from '../../../types' +import { githubGraphql } from '../gql' +import type { + PrCommentNode, + PrCommentsBatchPage, + PrCommentsConnection, +} from '../graphql/pullRequestChildren' +import { COMMENTS_FOR_PRS_QUERY } from '../graphql/pullRequestChildren' +import type { PullRequestNode } from '../graphql/pullRequests' +import { toPrComment } from '../mappers/prComment' +import { runDualPhasePrSync } from '../prWalk' +import { githubActivitySchema } from '../schemas' + +const PR_BATCH_SIZE = 50 +const COMMENTS_PAGE_SIZE = 50 + +async function runPullRequestCommentsSync(ctx: SyncContext): Promise { + const emitComments = async ( + connection: PrCommentsConnection, + pullRequest: PullRequestNode, + sinceDate: Date | null, + ): Promise => { + const comments = connection.edges + .map((edge) => edge?.node) + .filter((node): node is PrCommentNode => Boolean(node?.id)) + .filter((node) => !sinceDate || new Date(node.createdAt) >= sinceDate) + if (comments.length > 0) { + await ctx.emit(comments.map((comment) => toPrComment(comment, pullRequest))) + } + } + + const drainRemainingComments = async ( + prId: string, + pullRequest: PullRequestNode, + startCursor: string | null, + sinceDate: Date | null, + ): Promise => { + let cursor = startCursor + let hasMore = true + while (hasMore) { + const data = await githubGraphql(ctx.http, COMMENTS_FOR_PRS_QUERY, { + ids: [prId], + first: COMMENTS_PAGE_SIZE, + after: cursor, + }) + const comments = data.nodes[0]?.comments + if (!comments?.edges) { + return + } + await emitComments(comments, pullRequest, sinceDate) + hasMore = comments.pageInfo.hasNextPage + cursor = comments.pageInfo.endCursor + } + } + + await runDualPhasePrSync(ctx, async (prs, sinceDate) => { + for (let i = 0; i < prs.length; i += PR_BATCH_SIZE) { + const batch = prs.slice(i, i + PR_BATCH_SIZE) + const batchData = await githubGraphql(ctx.http, COMMENTS_FOR_PRS_QUERY, { + ids: batch.map((pr) => pr.id), + first: COMMENTS_PAGE_SIZE, + after: null, + }) + + for (const node of batchData.nodes) { + if (!node?.comments?.edges) { + continue + } + const pullRequest = batch.find((candidate) => candidate.id === node.id) + if (!pullRequest) { + continue + } + await emitComments(node.comments, pullRequest, sinceDate) + if (node.comments.pageInfo.hasNextPage) { + await drainRemainingComments( + node.id, + pullRequest, + node.comments.pageInfo.endCursor, + sinceDate, + ) + } + } + } + }) +} + +export const pullRequestCommentsSync: SyncDefinition = { + name: 'pull-request-comments', + cadenceMinutes: 60, + schema: githubActivitySchema, + run: runPullRequestCommentsSync, +} diff --git a/services/libs/connectors/src/connectors/github/syncs/pullRequestCommits.ts b/services/libs/connectors/src/connectors/github/syncs/pullRequestCommits.ts new file mode 100644 index 0000000000..6ce2f3d831 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/syncs/pullRequestCommits.ts @@ -0,0 +1,63 @@ +import type { SyncContext, SyncDefinition } from '../../../types' +import { githubGraphql } from '../gql' +import type { PrCommitNode, PrCommitsPage } from '../graphql/pullRequestChildren' +import { PR_COMMITS_QUERY } from '../graphql/pullRequestChildren' +import { toCommit } from '../mappers/commit' +import { parseRepoChannel } from '../paging' +import { runDualPhasePrSync } from '../prWalk' +import { githubActivitySchema } from '../schemas' + +const COMMITS_PAGE_SIZE = 50 + +async function runPullRequestCommitsSync(ctx: SyncContext): Promise { + const { owner, repo } = parseRepoChannel(ctx.channel.channelName) + + await runDualPhasePrSync(ctx, async (prs, sinceDate) => { + for (const pullRequest of prs) { + let cursor: string | null = null + let hasMore = true + + while (hasMore) { + const data = await githubGraphql(ctx.http, PR_COMMITS_QUERY, { + owner, + repo, + prNumber: pullRequest.number, + first: COMMITS_PAGE_SIZE, + cursor, + }) + + const commits = data.repository.pullRequest?.commits + if (!commits) { + break + } + + let reachedSince = false + const fresh: PrCommitNode['commit'][] = [] + for (const node of commits.nodes) { + if (!node?.commit) { + continue + } + if (sinceDate && new Date(node.commit.authoredDate) < sinceDate) { + reachedSince = true + break + } + fresh.push(node.commit) + } + + if (fresh.length > 0) { + await ctx.emit(fresh.map((commit) => toCommit(commit, pullRequest.id))) + } + + hasMore = !reachedSince && commits.pageInfo.hasNextPage + cursor = commits.pageInfo.endCursor + } + } + }) +} + +export const pullRequestCommitsSync: SyncDefinition = { + name: 'pull-request-commits', + cadenceMinutes: 120, + schema: githubActivitySchema, + run: runPullRequestCommitsSync, +} diff --git a/services/libs/connectors/src/connectors/github/syncs/pullRequestReviewComments.ts b/services/libs/connectors/src/connectors/github/syncs/pullRequestReviewComments.ts new file mode 100644 index 0000000000..f60387cb2d --- /dev/null +++ b/services/libs/connectors/src/connectors/github/syncs/pullRequestReviewComments.ts @@ -0,0 +1,164 @@ +import type { SyncContext, SyncDefinition } from '../../../types' +import { githubGraphql } from '../gql' +import type { + ReviewThreadNode, + ReviewThreadsBatchPage, + ThreadCommentNode, + ThreadCommentsBatchPage, +} from '../graphql/pullRequestChildren' +import { + COMMENTS_FOR_THREADS_QUERY, + REVIEW_THREADS_FOR_PRS_QUERY, +} from '../graphql/pullRequestChildren' +import type { PullRequestNode } from '../graphql/pullRequests' +import { toReviewThreadComment } from '../mappers/reviewThreadComment' +import { runDualPhasePrSync } from '../prWalk' +import { githubActivitySchema } from '../schemas' + +const PR_BATCH_SIZE = 50 +const THREADS_PAGE_SIZE = 50 +const THREAD_BATCH_SIZE = 100 +const COMMENTS_PAGE_SIZE = 50 + +function collectThreadIds( + prByThreadId: Map, + edges: ({ node: ReviewThreadNode | null } | null)[], + pullRequest: PullRequestNode, +): void { + for (const edge of edges) { + if (edge?.node?.id) { + prByThreadId.set(edge.node.id, pullRequest) + } + } +} + +async function drainRemainingThreads( + ctx: SyncContext, + prByThreadId: Map, + pullRequest: PullRequestNode, + startCursor: string | null, +): Promise { + let cursor = startCursor + let hasMore = true + while (hasMore) { + const data = await githubGraphql( + ctx.http, + REVIEW_THREADS_FOR_PRS_QUERY, + { ids: [pullRequest.id], first: THREADS_PAGE_SIZE, after: cursor }, + ) + const reviewThreads = data.nodes[0]?.reviewThreads + if (!reviewThreads?.edges) { + return + } + collectThreadIds(prByThreadId, reviewThreads.edges, pullRequest) + hasMore = reviewThreads.pageInfo.hasNextPage + cursor = reviewThreads.pageInfo.endCursor + } +} + +async function drainRemainingThreadComments( + ctx: SyncContext, + threadId: string, + startCursor: string | null, +): Promise { + const comments: ThreadCommentNode[] = [] + let cursor = startCursor + let hasMore = true + while (hasMore) { + const data = await githubGraphql( + ctx.http, + COMMENTS_FOR_THREADS_QUERY, + { ids: [threadId], first: COMMENTS_PAGE_SIZE, after: cursor }, + ) + const connection = data.nodes[0]?.comments + if (!connection?.edges) { + break + } + comments.push( + ...connection.edges + .map((edge) => edge?.node) + .filter((node): node is ThreadCommentNode => Boolean(node?.id)), + ) + hasMore = connection.pageInfo.hasNextPage + cursor = connection.pageInfo.endCursor + } + return comments +} + +async function runPullRequestReviewCommentsSync(ctx: SyncContext): Promise { + await runDualPhasePrSync(ctx, async (prs) => { + for (let i = 0; i < prs.length; i += PR_BATCH_SIZE) { + const batch = prs.slice(i, i + PR_BATCH_SIZE) + const threadsData = await githubGraphql( + ctx.http, + REVIEW_THREADS_FOR_PRS_QUERY, + { ids: batch.map((pr) => pr.id), first: THREADS_PAGE_SIZE, after: null }, + ) + + const prByThreadId = new Map() + for (const node of threadsData.nodes) { + if (!node?.reviewThreads?.edges) { + continue + } + const pullRequest = batch.find((candidate) => candidate.id === node.id) + if (!pullRequest) { + continue + } + collectThreadIds(prByThreadId, node.reviewThreads.edges, pullRequest) + if (node.reviewThreads.pageInfo.hasNextPage) { + await drainRemainingThreads( + ctx, + prByThreadId, + pullRequest, + node.reviewThreads.pageInfo.endCursor, + ) + } + } + + const threadIds = [...prByThreadId.keys()] + for (let j = 0; j < threadIds.length; j += THREAD_BATCH_SIZE) { + const threadBatch = threadIds.slice(j, j + THREAD_BATCH_SIZE) + const commentsData = await githubGraphql( + ctx.http, + COMMENTS_FOR_THREADS_QUERY, + { ids: threadBatch, first: COMMENTS_PAGE_SIZE, after: null }, + ) + + for (const thread of commentsData.nodes) { + if (!thread?.comments?.edges) { + continue + } + const pullRequest = prByThreadId.get(thread.id) + if (!pullRequest) { + continue + } + const threadNode: ReviewThreadNode = { id: thread.id, isResolved: thread.isResolved } + const comments = thread.comments.edges + .map((edge) => edge?.node) + .filter((node): node is ThreadCommentNode => Boolean(node?.id)) + if (thread.comments.pageInfo.hasNextPage) { + comments.push( + ...(await drainRemainingThreadComments( + ctx, + thread.id, + thread.comments.pageInfo.endCursor, + )), + ) + } + if (comments.length > 0) { + await ctx.emit( + comments.map((comment) => toReviewThreadComment(comment, threadNode, pullRequest)), + ) + } + } + } + } + }) +} + +export const pullRequestReviewCommentsSync: SyncDefinition = { + name: 'pull-request-review-comments', + cadenceMinutes: 120, + schema: githubActivitySchema, + run: runPullRequestReviewCommentsSync, +} diff --git a/services/libs/connectors/src/connectors/github/syncs/pullRequests.ts b/services/libs/connectors/src/connectors/github/syncs/pullRequests.ts new file mode 100644 index 0000000000..4c98ea07d9 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/syncs/pullRequests.ts @@ -0,0 +1,74 @@ +import type { SyncContext, SyncDefinition } from '../../../types' +import { githubGraphql } from '../gql' +import type { PrTimelineItem, PrTimelinePage, PullRequestNode } from '../graphql/pullRequests' +import { PR_TIMELINE_QUERY } from '../graphql/pullRequests' +import { toPullRequestActivities } from '../mappers/pullRequest' +import { runDualPhasePrSync } from '../prWalk' +import { githubActivitySchema } from '../schemas' + +const TIMELINE_BATCH_SIZE = 50 + +async function drainRemainingTimeline( + ctx: SyncContext, + prId: string, + startCursor: string | null, +): Promise<(PrTimelineItem | null)[]> { + const items: (PrTimelineItem | null)[] = [] + let cursor = startCursor + let hasMore = true + while (hasMore) { + const data = await githubGraphql(ctx.http, PR_TIMELINE_QUERY, { + ids: [prId], + first: TIMELINE_BATCH_SIZE, + after: cursor, + }) + const timeline = data.nodes[0]?.timelineItems + if (!timeline) { + break + } + items.push(...timeline.nodes) + hasMore = timeline.pageInfo.hasNextPage + cursor = timeline.pageInfo.endCursor + } + return items +} + +async function emitPullRequests(ctx: SyncContext, pullRequests: PullRequestNode[]): Promise { + for (let i = 0; i < pullRequests.length; i += TIMELINE_BATCH_SIZE) { + const batch = pullRequests.slice(i, i + TIMELINE_BATCH_SIZE) + const timelineData = await githubGraphql(ctx.http, PR_TIMELINE_QUERY, { + ids: batch.map((pr) => pr.id), + first: TIMELINE_BATCH_SIZE, + after: null, + }) + + const timelinesByPrId = new Map() + for (const node of timelineData.nodes) { + if (!node?.id) { + continue + } + const items = [...(node.timelineItems?.nodes ?? [])] + if (node.timelineItems?.pageInfo.hasNextPage) { + items.push( + ...(await drainRemainingTimeline(ctx, node.id, node.timelineItems.pageInfo.endCursor)), + ) + } + timelinesByPrId.set(node.id, items) + } + + await ctx.emit( + batch.flatMap((pr) => toPullRequestActivities(pr, timelinesByPrId.get(pr.id) ?? [])), + ) + } +} + +async function runPullRequestsSync(ctx: SyncContext): Promise { + await runDualPhasePrSync(ctx, (prs) => emitPullRequests(ctx, prs)) +} + +export const pullRequestsSync: SyncDefinition = { + name: 'pull-requests', + cadenceMinutes: 60, + schema: githubActivitySchema, + run: runPullRequestsSync, +} From bd43a598e1e39ba2a4a4dd8c1eaa40afe81d95ae Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Tue, 1 Sep 2026 09:41:54 +0100 Subject: [PATCH 38/69] fix: park github syncs on body-level and secondary rate limits with error class stamping Signed-off-by: Mouad BANI --- .../src/activities/syncRunActivities.ts | 5 +++-- .../src/connectors/github/interpret.ts | 15 ++++++++----- .../src/connectors/syncUnits.ts | 22 +++++++++++++++++++ 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/services/apps/connectors_worker/src/activities/syncRunActivities.ts b/services/apps/connectors_worker/src/activities/syncRunActivities.ts index 94971533ec..2be605e7b3 100644 --- a/services/apps/connectors_worker/src/activities/syncRunActivities.ts +++ b/services/apps/connectors_worker/src/activities/syncRunActivities.ts @@ -12,10 +12,10 @@ import { import type { Emitter, SyncContext } from '@crowd/connectors' import { getUnitById, + parkUnit, recordRunFailure, recordRunPartial, recordRunSuccess, - rescheduleUnit, } from '@crowd/data-access-layer/src/connectors' import { fetchIntegrationById } from '@crowd/data-access-layer/src/integrations' import IntegrationStreamRepository from '@crowd/data-access-layer/src/old/apps/integration_stream_worker/integrationStream.repo' @@ -120,9 +120,10 @@ export async function executeSync(unitId: string): Promise { unitId, { watermark: committedWatermark, emittedCount: emitter.emittedCount() }, resumeAt, + err.errorClass, ) } else { - await rescheduleUnit(qx, unitId, resumeAt) + await parkUnit(qx, unitId, resumeAt, err.errorClass) } log.info({ resumeAt }, 'sync run rate-limit parked') return diff --git a/services/libs/connectors/src/connectors/github/interpret.ts b/services/libs/connectors/src/connectors/github/interpret.ts index 40c0c7b4eb..ff51c7dd90 100644 --- a/services/libs/connectors/src/connectors/github/interpret.ts +++ b/services/libs/connectors/src/connectors/github/interpret.ts @@ -1,17 +1,22 @@ import type { ResponseInterpreter } from '../../http/client' import { RateLimitError } from '../../http/errors' -interface GraphqlErrorEnvelope { +interface GithubErrorBody { errors?: { type?: string }[] + message?: string } export const interpretGithubResponse: ResponseInterpreter = (response) => { - if (response.status !== 200) { + const body = response.data as GithubErrorBody | null + if (response.status === 200) { + // GitHub docs say RATE_LIMITED, but installation tokens return RATE_LIMIT + if (body?.errors?.some((e) => e.type === 'RATE_LIMITED' || e.type === 'RATE_LIMIT')) { + return new RateLimitError('github graphql rate limited') + } return null } - const body = response.data as GraphqlErrorEnvelope | null - if (body?.errors?.some((e) => e.type === 'RATE_LIMITED')) { - return new RateLimitError('github graphql rate limited') + if (response.status === 403 && body?.message?.toLowerCase().includes('secondary rate limit')) { + return new RateLimitError('github secondary rate limited') } return null } diff --git a/services/libs/data-access-layer/src/connectors/syncUnits.ts b/services/libs/data-access-layer/src/connectors/syncUnits.ts index 3faaec5fe7..873f15f8e8 100644 --- a/services/libs/data-access-layer/src/connectors/syncUnits.ts +++ b/services/libs/data-access-layer/src/connectors/syncUnits.ts @@ -85,6 +85,7 @@ export async function recordRunSuccess( "lastRunAt" = now(), "lastSuccessAt" = now(), "consecutiveFailures" = 0, + "lastErrorClass" = NULL, "updatedAt" = now() WHERE id = $(id)`, { id, watermark: JSON.stringify(data.watermark), emittedCount: data.emittedCount }, @@ -96,6 +97,7 @@ export async function recordRunPartial( id: string, progress: ISyncRunSuccess, resumeAt: Date, + errorClass: string, ): Promise { await qx.result( `UPDATE integration.sync_units @@ -103,6 +105,7 @@ export async function recordRunPartial( "emittedCount" = $(emittedCount), "nextRunAt" = $(resumeAt), "lastRunAt" = now(), + "lastErrorClass" = $(errorClass), "lockedAt" = NULL, "updatedAt" = now() WHERE id = $(id)`, @@ -111,10 +114,29 @@ export async function recordRunPartial( watermark: JSON.stringify(progress.watermark), emittedCount: progress.emittedCount, resumeAt, + errorClass, }, ) } +export async function parkUnit( + qx: QueryExecutor, + id: string, + resumeAt: Date, + errorClass: string, +): Promise { + await qx.result( + `UPDATE integration.sync_units + SET "nextRunAt" = $(resumeAt), + "lastRunAt" = now(), + "lastErrorClass" = $(errorClass), + "lockedAt" = NULL, + "updatedAt" = now() + WHERE id = $(id)`, + { id, resumeAt, errorClass }, + ) +} + export async function recordRunFailure( qx: QueryExecutor, id: string, From 1187b0e314997c09d2c915a2807861fa9bceb282 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Tue, 1 Sep 2026 09:43:18 +0100 Subject: [PATCH 39/69] fix: fetch pr children per item to avoid github batch node drops Signed-off-by: Mouad BANI --- services/libs/connectors/src/concurrency.ts | 16 ++ .../src/connectors/github/paging.ts | 3 + .../github/syncs/pullRequestComments.ts | 105 +++++-------- .../github/syncs/pullRequestReviewComments.ts | 144 +++++------------- .../connectors/github/syncs/pullRequests.ts | 57 +++---- services/libs/connectors/src/index.ts | 1 + 6 files changed, 116 insertions(+), 210 deletions(-) create mode 100644 services/libs/connectors/src/concurrency.ts diff --git a/services/libs/connectors/src/concurrency.ts b/services/libs/connectors/src/concurrency.ts new file mode 100644 index 0000000000..dcafecaefb --- /dev/null +++ b/services/libs/connectors/src/concurrency.ts @@ -0,0 +1,16 @@ +export async function mapWithConcurrency( + items: T[], + limit: number, + fn: (item: T) => Promise, +): Promise { + const results = new Array(items.length) + let nextIndex = 0 + const workers = Array.from({ length: Math.min(limit, items.length) }, async () => { + while (nextIndex < items.length) { + const index = nextIndex++ + results[index] = await fn(items[index]) + } + }) + await Promise.all(workers) + return results +} diff --git a/services/libs/connectors/src/connectors/github/paging.ts b/services/libs/connectors/src/connectors/github/paging.ts index 7f55e500af..983706b8b1 100644 --- a/services/libs/connectors/src/connectors/github/paging.ts +++ b/services/libs/connectors/src/connectors/github/paging.ts @@ -6,6 +6,9 @@ export interface GithubWatermark { export const MAX_PAGES_PER_RUN = Number.POSITIVE_INFINITY export const PAGE_SIZE = 100 +// GitHub GraphQL silently omits timeline/connection items from heavy nodes(ids:) batches +// (no error, pageInfo claims completeness) — fetch per item, bounded by this concurrency. +export const ITEM_FETCH_CONCURRENCY = 5 export function readWatermark(raw: Record | null): GithubWatermark { if (raw && (raw.phase === 'backfill' || raw.phase === 'incremental')) { diff --git a/services/libs/connectors/src/connectors/github/syncs/pullRequestComments.ts b/services/libs/connectors/src/connectors/github/syncs/pullRequestComments.ts index e0e7c9cdca..be0758f55c 100644 --- a/services/libs/connectors/src/connectors/github/syncs/pullRequestComments.ts +++ b/services/libs/connectors/src/connectors/github/syncs/pullRequestComments.ts @@ -1,85 +1,50 @@ +import { mapWithConcurrency } from '../../../concurrency' import type { SyncContext, SyncDefinition } from '../../../types' import { githubGraphql } from '../gql' -import type { - PrCommentNode, - PrCommentsBatchPage, - PrCommentsConnection, -} from '../graphql/pullRequestChildren' +import type { PrCommentNode, PrCommentsBatchPage } from '../graphql/pullRequestChildren' import { COMMENTS_FOR_PRS_QUERY } from '../graphql/pullRequestChildren' -import type { PullRequestNode } from '../graphql/pullRequests' import { toPrComment } from '../mappers/prComment' +import { ITEM_FETCH_CONCURRENCY } from '../paging' import { runDualPhasePrSync } from '../prWalk' import { githubActivitySchema } from '../schemas' -const PR_BATCH_SIZE = 50 const COMMENTS_PAGE_SIZE = 50 -async function runPullRequestCommentsSync(ctx: SyncContext): Promise { - const emitComments = async ( - connection: PrCommentsConnection, - pullRequest: PullRequestNode, - sinceDate: Date | null, - ): Promise => { - const comments = connection.edges - .map((edge) => edge?.node) - .filter((node): node is PrCommentNode => Boolean(node?.id)) - .filter((node) => !sinceDate || new Date(node.createdAt) >= sinceDate) - if (comments.length > 0) { - await ctx.emit(comments.map((comment) => toPrComment(comment, pullRequest))) - } - } - - const drainRemainingComments = async ( - prId: string, - pullRequest: PullRequestNode, - startCursor: string | null, - sinceDate: Date | null, - ): Promise => { - let cursor = startCursor - let hasMore = true - while (hasMore) { - const data = await githubGraphql(ctx.http, COMMENTS_FOR_PRS_QUERY, { - ids: [prId], - first: COMMENTS_PAGE_SIZE, - after: cursor, - }) - const comments = data.nodes[0]?.comments - if (!comments?.edges) { - return - } - await emitComments(comments, pullRequest, sinceDate) - hasMore = comments.pageInfo.hasNextPage - cursor = comments.pageInfo.endCursor +async function fetchComments(ctx: SyncContext, prId: string): Promise { + const comments: PrCommentNode[] = [] + let cursor: string | null = null + do { + const data = await githubGraphql(ctx.http, COMMENTS_FOR_PRS_QUERY, { + ids: [prId], + first: COMMENTS_PAGE_SIZE, + after: cursor, + }) + const connection = data.nodes[0]?.comments + if (!connection?.edges) { + break } - } + comments.push( + ...connection.edges + .map((edge) => edge?.node) + .filter((node): node is PrCommentNode => Boolean(node?.id)), + ) + cursor = connection.pageInfo.hasNextPage ? connection.pageInfo.endCursor : null + } while (cursor) + return comments +} +async function runPullRequestCommentsSync(ctx: SyncContext): Promise { await runDualPhasePrSync(ctx, async (prs, sinceDate) => { - for (let i = 0; i < prs.length; i += PR_BATCH_SIZE) { - const batch = prs.slice(i, i + PR_BATCH_SIZE) - const batchData = await githubGraphql(ctx.http, COMMENTS_FOR_PRS_QUERY, { - ids: batch.map((pr) => pr.id), - first: COMMENTS_PAGE_SIZE, - after: null, - }) - - for (const node of batchData.nodes) { - if (!node?.comments?.edges) { - continue - } - const pullRequest = batch.find((candidate) => candidate.id === node.id) - if (!pullRequest) { - continue - } - await emitComments(node.comments, pullRequest, sinceDate) - if (node.comments.pageInfo.hasNextPage) { - await drainRemainingComments( - node.id, - pullRequest, - node.comments.pageInfo.endCursor, - sinceDate, - ) - } - } + const commentsPerPr = await mapWithConcurrency(prs, ITEM_FETCH_CONCURRENCY, (pr) => + fetchComments(ctx, pr.id), + ) + const activities = prs.flatMap((pullRequest, index) => + commentsPerPr[index] + .filter((comment) => !sinceDate || new Date(comment.createdAt) >= sinceDate) + .map((comment) => toPrComment(comment, pullRequest)), + ) + if (activities.length > 0) { + await ctx.emit(activities) } }) } diff --git a/services/libs/connectors/src/connectors/github/syncs/pullRequestReviewComments.ts b/services/libs/connectors/src/connectors/github/syncs/pullRequestReviewComments.ts index f60387cb2d..0c75450ae8 100644 --- a/services/libs/connectors/src/connectors/github/syncs/pullRequestReviewComments.ts +++ b/services/libs/connectors/src/connectors/github/syncs/pullRequestReviewComments.ts @@ -1,3 +1,4 @@ +import { mapWithConcurrency } from '../../../concurrency' import type { SyncContext, SyncDefinition } from '../../../types' import { githubGraphql } from '../gql' import type { @@ -10,61 +11,44 @@ import { COMMENTS_FOR_THREADS_QUERY, REVIEW_THREADS_FOR_PRS_QUERY, } from '../graphql/pullRequestChildren' -import type { PullRequestNode } from '../graphql/pullRequests' import { toReviewThreadComment } from '../mappers/reviewThreadComment' +import { ITEM_FETCH_CONCURRENCY } from '../paging' import { runDualPhasePrSync } from '../prWalk' import { githubActivitySchema } from '../schemas' -const PR_BATCH_SIZE = 50 const THREADS_PAGE_SIZE = 50 -const THREAD_BATCH_SIZE = 100 const COMMENTS_PAGE_SIZE = 50 -function collectThreadIds( - prByThreadId: Map, - edges: ({ node: ReviewThreadNode | null } | null)[], - pullRequest: PullRequestNode, -): void { - for (const edge of edges) { - if (edge?.node?.id) { - prByThreadId.set(edge.node.id, pullRequest) - } - } -} - -async function drainRemainingThreads( - ctx: SyncContext, - prByThreadId: Map, - pullRequest: PullRequestNode, - startCursor: string | null, -): Promise { - let cursor = startCursor - let hasMore = true - while (hasMore) { +async function fetchThreads(ctx: SyncContext, prId: string): Promise { + const threads: ReviewThreadNode[] = [] + let cursor: string | null = null + do { const data = await githubGraphql( ctx.http, REVIEW_THREADS_FOR_PRS_QUERY, - { ids: [pullRequest.id], first: THREADS_PAGE_SIZE, after: cursor }, + { ids: [prId], first: THREADS_PAGE_SIZE, after: cursor }, ) - const reviewThreads = data.nodes[0]?.reviewThreads - if (!reviewThreads?.edges) { - return + const connection = data.nodes[0]?.reviewThreads + if (!connection?.edges) { + break } - collectThreadIds(prByThreadId, reviewThreads.edges, pullRequest) - hasMore = reviewThreads.pageInfo.hasNextPage - cursor = reviewThreads.pageInfo.endCursor - } + threads.push( + ...connection.edges + .map((edge) => edge?.node) + .filter((node): node is ReviewThreadNode => Boolean(node?.id)), + ) + cursor = connection.pageInfo.hasNextPage ? connection.pageInfo.endCursor : null + } while (cursor) + return threads } -async function drainRemainingThreadComments( +async function fetchThreadComments( ctx: SyncContext, threadId: string, - startCursor: string | null, ): Promise { const comments: ThreadCommentNode[] = [] - let cursor = startCursor - let hasMore = true - while (hasMore) { + let cursor: string | null = null + do { const data = await githubGraphql( ctx.http, COMMENTS_FOR_THREADS_QUERY, @@ -79,79 +63,31 @@ async function drainRemainingThreadComments( .map((edge) => edge?.node) .filter((node): node is ThreadCommentNode => Boolean(node?.id)), ) - hasMore = connection.pageInfo.hasNextPage - cursor = connection.pageInfo.endCursor - } + cursor = connection.pageInfo.hasNextPage ? connection.pageInfo.endCursor : null + } while (cursor) return comments } async function runPullRequestReviewCommentsSync(ctx: SyncContext): Promise { - await runDualPhasePrSync(ctx, async (prs) => { - for (let i = 0; i < prs.length; i += PR_BATCH_SIZE) { - const batch = prs.slice(i, i + PR_BATCH_SIZE) - const threadsData = await githubGraphql( - ctx.http, - REVIEW_THREADS_FOR_PRS_QUERY, - { ids: batch.map((pr) => pr.id), first: THREADS_PAGE_SIZE, after: null }, - ) - - const prByThreadId = new Map() - for (const node of threadsData.nodes) { - if (!node?.reviewThreads?.edges) { - continue - } - const pullRequest = batch.find((candidate) => candidate.id === node.id) - if (!pullRequest) { - continue - } - collectThreadIds(prByThreadId, node.reviewThreads.edges, pullRequest) - if (node.reviewThreads.pageInfo.hasNextPage) { - await drainRemainingThreads( - ctx, - prByThreadId, - pullRequest, - node.reviewThreads.pageInfo.endCursor, - ) - } - } + await runDualPhasePrSync(ctx, async (prs, sinceDate) => { + const threadsPerPr = await mapWithConcurrency(prs, ITEM_FETCH_CONCURRENCY, (pr) => + fetchThreads(ctx, pr.id), + ) + const threads = prs.flatMap((pullRequest, index) => + threadsPerPr[index].map((thread) => ({ thread, pullRequest })), + ) - const threadIds = [...prByThreadId.keys()] - for (let j = 0; j < threadIds.length; j += THREAD_BATCH_SIZE) { - const threadBatch = threadIds.slice(j, j + THREAD_BATCH_SIZE) - const commentsData = await githubGraphql( - ctx.http, - COMMENTS_FOR_THREADS_QUERY, - { ids: threadBatch, first: COMMENTS_PAGE_SIZE, after: null }, - ) + const commentsPerThread = await mapWithConcurrency(threads, ITEM_FETCH_CONCURRENCY, (entry) => + fetchThreadComments(ctx, entry.thread.id), + ) - for (const thread of commentsData.nodes) { - if (!thread?.comments?.edges) { - continue - } - const pullRequest = prByThreadId.get(thread.id) - if (!pullRequest) { - continue - } - const threadNode: ReviewThreadNode = { id: thread.id, isResolved: thread.isResolved } - const comments = thread.comments.edges - .map((edge) => edge?.node) - .filter((node): node is ThreadCommentNode => Boolean(node?.id)) - if (thread.comments.pageInfo.hasNextPage) { - comments.push( - ...(await drainRemainingThreadComments( - ctx, - thread.id, - thread.comments.pageInfo.endCursor, - )), - ) - } - if (comments.length > 0) { - await ctx.emit( - comments.map((comment) => toReviewThreadComment(comment, threadNode, pullRequest)), - ) - } - } - } + const activities = threads.flatMap(({ thread, pullRequest }, index) => + commentsPerThread[index] + .filter((comment) => !sinceDate || new Date(comment.createdAt) >= sinceDate) + .map((comment) => toReviewThreadComment(comment, thread, pullRequest)), + ) + if (activities.length > 0) { + await ctx.emit(activities) } }) } diff --git a/services/libs/connectors/src/connectors/github/syncs/pullRequests.ts b/services/libs/connectors/src/connectors/github/syncs/pullRequests.ts index 4c98ea07d9..9e73fd99da 100644 --- a/services/libs/connectors/src/connectors/github/syncs/pullRequests.ts +++ b/services/libs/connectors/src/connectors/github/syncs/pullRequests.ts @@ -1,25 +1,22 @@ +import { mapWithConcurrency } from '../../../concurrency' import type { SyncContext, SyncDefinition } from '../../../types' import { githubGraphql } from '../gql' import type { PrTimelineItem, PrTimelinePage, PullRequestNode } from '../graphql/pullRequests' import { PR_TIMELINE_QUERY } from '../graphql/pullRequests' import { toPullRequestActivities } from '../mappers/pullRequest' +import { ITEM_FETCH_CONCURRENCY } from '../paging' import { runDualPhasePrSync } from '../prWalk' import { githubActivitySchema } from '../schemas' -const TIMELINE_BATCH_SIZE = 50 +const TIMELINE_PAGE_SIZE = 50 -async function drainRemainingTimeline( - ctx: SyncContext, - prId: string, - startCursor: string | null, -): Promise<(PrTimelineItem | null)[]> { +async function fetchTimeline(ctx: SyncContext, prId: string): Promise<(PrTimelineItem | null)[]> { const items: (PrTimelineItem | null)[] = [] - let cursor = startCursor - let hasMore = true - while (hasMore) { + let cursor: string | null = null + do { const data = await githubGraphql(ctx.http, PR_TIMELINE_QUERY, { ids: [prId], - first: TIMELINE_BATCH_SIZE, + first: TIMELINE_PAGE_SIZE, after: cursor, }) const timeline = data.nodes[0]?.timelineItems @@ -27,39 +24,27 @@ async function drainRemainingTimeline( break } items.push(...timeline.nodes) - hasMore = timeline.pageInfo.hasNextPage - cursor = timeline.pageInfo.endCursor - } + cursor = timeline.pageInfo.hasNextPage ? timeline.pageInfo.endCursor : null + } while (cursor) return items } async function emitPullRequests(ctx: SyncContext, pullRequests: PullRequestNode[]): Promise { - for (let i = 0; i < pullRequests.length; i += TIMELINE_BATCH_SIZE) { - const batch = pullRequests.slice(i, i + TIMELINE_BATCH_SIZE) - const timelineData = await githubGraphql(ctx.http, PR_TIMELINE_QUERY, { - ids: batch.map((pr) => pr.id), - first: TIMELINE_BATCH_SIZE, - after: null, - }) + const timelines = await mapWithConcurrency(pullRequests, ITEM_FETCH_CONCURRENCY, (pr) => + fetchTimeline(ctx, pr.id), + ) - const timelinesByPrId = new Map() - for (const node of timelineData.nodes) { - if (!node?.id) { - continue - } - const items = [...(node.timelineItems?.nodes ?? [])] - if (node.timelineItems?.pageInfo.hasNextPage) { - items.push( - ...(await drainRemainingTimeline(ctx, node.id, node.timelineItems.pageInfo.endCursor)), - ) - } - timelinesByPrId.set(node.id, items) + pullRequests.forEach((pr, index) => { + const hasMergedEvent = timelines[index].some((item) => item?.__typename === 'MergedEvent') + if (pr.state === 'MERGED' && !hasMergedEvent) { + ctx.log.warn( + { prId: pr.id, prNumber: pr.number, itemCount: timelines[index].length }, + 'merged pr timeline missing MergedEvent', + ) } + }) - await ctx.emit( - batch.flatMap((pr) => toPullRequestActivities(pr, timelinesByPrId.get(pr.id) ?? [])), - ) - } + await ctx.emit(pullRequests.flatMap((pr, index) => toPullRequestActivities(pr, timelines[index]))) } async function runPullRequestsSync(ctx: SyncContext): Promise { diff --git a/services/libs/connectors/src/index.ts b/services/libs/connectors/src/index.ts index 0e212766d4..520162b5c2 100644 --- a/services/libs/connectors/src/index.ts +++ b/services/libs/connectors/src/index.ts @@ -1,3 +1,4 @@ +export * from './concurrency' export * from './credentials' export * from './emit' export * from './http/client' From 3b69675e9814e4884aac65195441acb9af06d629 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Tue, 1 Sep 2026 09:43:49 +0100 Subject: [PATCH 40/69] fix: align github member payloads and event skips with nango implementation Signed-off-by: Mouad BANI --- .../src/connectors/github/mappers/member.ts | 43 +++++++++++-------- .../connectors/github/mappers/pullRequest.ts | 2 +- .../src/connectors/github/schemas.ts | 19 ++++---- .../src/connectors/github/syncs/forks.ts | 2 +- .../github/syncs/pullRequestCommits.ts | 21 +++------ 5 files changed, 46 insertions(+), 41 deletions(-) diff --git a/services/libs/connectors/src/connectors/github/mappers/member.ts b/services/libs/connectors/src/connectors/github/mappers/member.ts index cd8681781d..8546ce7299 100644 --- a/services/libs/connectors/src/connectors/github/mappers/member.ts +++ b/services/libs/connectors/src/connectors/github/mappers/member.ts @@ -42,28 +42,37 @@ const GHOST_MEMBER: GithubMember = { }, ], attributes: { - avatarUrl: 'https://avatars.githubusercontent.com/u/10137?v=4', - bio: "Hi, I'm @ghost! I take the place of user accounts that have been deleted.\n:ghost:\n", - company: '', - isBot: false, - isHireable: false, - location: 'Nothing to see here, move along.', - url: 'https://github.com/ghost', - websiteUrl: '', + url: { github: 'https://github.com/ghost' }, + avatarUrl: { github: 'https://avatars.githubusercontent.com/u/10137?v=4' }, + bio: { + github: + "Hi, I'm @ghost! I take the place of user accounts that have been deleted.\n:ghost:\n", + }, }, } function toAttributes(user: GithubUserNode): GithubMember['attributes'] { - return { - isHireable: user.isHireable ?? false, - url: `https://github.com/${user.login ?? ''}`, - bio: user.bio ?? '', - location: user.location ?? '', - avatarUrl: user.avatarUrl ?? '', - company: user.company ?? '', - isBot: user.__typename === 'Bot', - websiteUrl: user.websiteUrl ?? '', + if (user.__typename === 'Bot') { + return { + url: { github: user.url ?? '' }, + avatarUrl: { github: user.avatarUrl ?? '' }, + sourceId: { github: user.id?.toString() ?? '' }, + isBot: { github: true }, + } + } + + const attributes: GithubMember['attributes'] = { + isHireable: { github: user.isHireable ?? false }, + url: { github: user.url ?? `https://github.com/${user.login ?? ''}` }, + bio: { github: user.bio ?? '' }, + location: { github: user.location ?? '' }, + avatarUrl: { github: user.avatarUrl ?? '' }, + company: { github: user.company ?? '' }, + } + if (user.websiteUrl) { + attributes.websiteUrl = { github: user.websiteUrl } } + return attributes } function toOrganizations(user: GithubUserNode): GithubOrganization[] { diff --git a/services/libs/connectors/src/connectors/github/mappers/pullRequest.ts b/services/libs/connectors/src/connectors/github/mappers/pullRequest.ts index d6dad713fc..1c357d0938 100644 --- a/services/libs/connectors/src/connectors/github/mappers/pullRequest.ts +++ b/services/libs/connectors/src/connectors/github/mappers/pullRequest.ts @@ -42,7 +42,7 @@ function toTimelineActivity( } } case 'ReviewRequestedEvent': { - if (!item.requestedReviewer || !item.id) { + if (!item.requestedReviewer?.login || !item.id) { return null } const timestamp = item.createdAt ?? DEFAULT_TIMESTAMP diff --git a/services/libs/connectors/src/connectors/github/schemas.ts b/services/libs/connectors/src/connectors/github/schemas.ts index e8ad03f273..48429149c6 100644 --- a/services/libs/connectors/src/connectors/github/schemas.ts +++ b/services/libs/connectors/src/connectors/github/schemas.ts @@ -10,15 +10,18 @@ const identitySchema = z.object({ sourceId: z.string(), }) +const githubScoped = (value: T) => z.object({ github: value }) + const memberAttributesSchema = z.object({ - isHireable: z.boolean(), - url: z.string(), - bio: z.string(), - location: z.string(), - avatarUrl: z.string(), - company: z.string(), - isBot: z.boolean(), - websiteUrl: z.string().optional(), + isHireable: githubScoped(z.boolean()).optional(), + url: githubScoped(z.string()).optional(), + bio: githubScoped(z.string()).optional(), + location: githubScoped(z.string()).optional(), + avatarUrl: githubScoped(z.string()).optional(), + company: githubScoped(z.string()).optional(), + websiteUrl: githubScoped(z.string()).optional(), + sourceId: githubScoped(z.string()).optional(), + isBot: githubScoped(z.boolean()).optional(), }) const organizationIdentitySchema = z.object({ diff --git a/services/libs/connectors/src/connectors/github/syncs/forks.ts b/services/libs/connectors/src/connectors/github/syncs/forks.ts index 288a78aac6..312c5b4c96 100644 --- a/services/libs/connectors/src/connectors/github/syncs/forks.ts +++ b/services/libs/connectors/src/connectors/github/syncs/forks.ts @@ -31,7 +31,7 @@ async function runForksSync(ctx: SyncContext): Promise { since = forks[forks.length - 1].createdAt } - cursor = pageInfo.hasNextPage ? pageInfo.endCursor : null + cursor = pageInfo.endCursor ?? cursor await ctx.commitWatermark({ phase: 'incremental', since, cursor }) if (!pageInfo.hasNextPage) { diff --git a/services/libs/connectors/src/connectors/github/syncs/pullRequestCommits.ts b/services/libs/connectors/src/connectors/github/syncs/pullRequestCommits.ts index 6ce2f3d831..b5aae620ae 100644 --- a/services/libs/connectors/src/connectors/github/syncs/pullRequestCommits.ts +++ b/services/libs/connectors/src/connectors/github/syncs/pullRequestCommits.ts @@ -12,7 +12,7 @@ const COMMITS_PAGE_SIZE = 50 async function runPullRequestCommitsSync(ctx: SyncContext): Promise { const { owner, repo } = parseRepoChannel(ctx.channel.channelName) - await runDualPhasePrSync(ctx, async (prs, sinceDate) => { + await runDualPhasePrSync(ctx, async (prs) => { for (const pullRequest of prs) { let cursor: string | null = null let hasMore = true @@ -31,24 +31,17 @@ async function runPullRequestCommitsSync(ctx: SyncContext): Promise { break } - let reachedSince = false - const fresh: PrCommitNode['commit'][] = [] - for (const node of commits.nodes) { - if (!node?.commit) { - continue - } - if (sinceDate && new Date(node.commit.authoredDate) < sinceDate) { - reachedSince = true - break - } - fresh.push(node.commit) - } + const fresh = commits.nodes + .map((node) => node?.commit) + .filter((commit): commit is PrCommitNode['commit'] => + Boolean(commit?.author?.user?.login), + ) if (fresh.length > 0) { await ctx.emit(fresh.map((commit) => toCommit(commit, pullRequest.id))) } - hasMore = !reachedSince && commits.pageInfo.hasNextPage + hasMore = commits.pageInfo.hasNextPage cursor = commits.pageInfo.endCursor } } From 7dcd42d63366e51e591e970fa3f3ea7fe6b2c498 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Tue, 1 Sep 2026 09:44:16 +0100 Subject: [PATCH 41/69] perf: probe token budgets concurrently in dispatcher admission Signed-off-by: Mouad BANI --- .../src/activities/dispatcherActivities.ts | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/services/apps/connectors_worker/src/activities/dispatcherActivities.ts b/services/apps/connectors_worker/src/activities/dispatcherActivities.ts index df13d44b7a..d0ebedf676 100644 --- a/services/apps/connectors_worker/src/activities/dispatcherActivities.ts +++ b/services/apps/connectors_worker/src/activities/dispatcherActivities.ts @@ -1,4 +1,4 @@ -import { createTokenPool, findManifest, getSync } from '@crowd/connectors' +import { createTokenPool, findManifest, getSync, mapWithConcurrency } from '@crowd/connectors' import { claimDueUnits, rescheduleUnit } from '@crowd/data-access-layer/src/connectors' import type { IClaimedUnit } from '@crowd/data-access-layer/src/connectors' import { dbStoreQx } from '@crowd/data-access-layer/src/queryExecutor' @@ -14,26 +14,24 @@ const CADENCE_JITTER_RATIO = 0.1 const DEFAULT_RUN_ESTIMATE = 50 const DEFER_MIN_MS = 30_000 const DEFER_JITTER_MS = 60_000 +const BUDGET_PROBE_CONCURRENCY = 10 export async function claimDue(limit: number): Promise { return claimDueUnits(dbStoreQx(svc.postgres.writer), limit) } export async function admitByBudget(units: IClaimedUnit[]): Promise { - const admitted: IClaimedUnit[] = [] - const deferred: IClaimedUnit[] = [] - for (const unit of units) { + const headrooms = await mapWithConcurrency(units, BUDGET_PROBE_CONCURRENCY, (unit) => { const manifest = findManifest(unit.platform) const pool = createTokenPool(svc.redis, unit.platform, unit.integrationId, { probeBudget: manifest?.probeBudget, }) - if (await pool.hasHeadroom(DEFAULT_RUN_ESTIMATE)) { - admitted.push(unit) - } else { - deferred.push(unit) - } + return pool.hasHeadroom(DEFAULT_RUN_ESTIMATE) + }) + return { + admitted: units.filter((_, index) => headrooms[index]), + deferred: units.filter((_, index) => !headrooms[index]), } - return { admitted, deferred } } export async function deferUnit(unitId: string): Promise { From ed1c9402dbb0fe6549e3c964e87a6805dd81c707 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Tue, 1 Sep 2026 18:18:45 +0100 Subject: [PATCH 42/69] feat: add github discussions sync with nested reply traversal Signed-off-by: Mouad BANI --- .../connectors/github/graphql/discussions.ts | 185 ++++++++++++++++++ .../connectors/github/mappers/discussion.ts | 52 +++++ .../connectors/github/syncs/discussions.ts | 150 ++++++++++++++ 3 files changed, 387 insertions(+) create mode 100644 services/libs/connectors/src/connectors/github/graphql/discussions.ts create mode 100644 services/libs/connectors/src/connectors/github/mappers/discussion.ts create mode 100644 services/libs/connectors/src/connectors/github/syncs/discussions.ts diff --git a/services/libs/connectors/src/connectors/github/graphql/discussions.ts b/services/libs/connectors/src/connectors/github/graphql/discussions.ts new file mode 100644 index 0000000000..2dd6d12a87 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/graphql/discussions.ts @@ -0,0 +1,185 @@ +import type { GithubUserNode } from '../mappers/member' + +import { BOT_FIELDS, ORGANIZATION_FIELDS, USER_FIELDS } from './fields' + +export interface DiscussionCategory { + id: string + isAnswerable: boolean + name: string + slug: string + emoji: string + description: string | null +} + +export interface DiscussionNode { + id: string + number: number + title: string + bodyText: string + url: string + createdAt: string + updatedAt: string + isAnswered: boolean | null + author: GithubUserNode | null + category: DiscussionCategory + comments: { totalCount: number } +} + +export interface DiscussionsPage { + repository: { + discussions: { + pageInfo: { + endCursor: string | null + hasNextPage: boolean + } + nodes: (DiscussionNode | null)[] + } + } +} + +export interface DiscussionReplyNode { + id: string + bodyText: string + url: string + createdAt: string +} + +export interface DiscussionCommentNode extends DiscussionReplyNode { + replyTo: { id: string } | null + replies: RepliesConnection +} + +export interface RepliesConnection { + pageInfo: { + endCursor: string | null + hasNextPage: boolean + } + nodes: (DiscussionReplyNode | null)[] +} + +export interface DiscussionCommentsPage { + node: { + comments: { + pageInfo: { + endCursor: string | null + hasNextPage: boolean + } + nodes: (DiscussionCommentNode | null)[] + } + } | null +} + +export interface CommentRepliesPage { + node: { + replies: RepliesConnection + } | null +} + +const AUTHOR_FIELDS = ` + author { + login + ... on User { + ${USER_FIELDS} + } + ... on Bot { + ${BOT_FIELDS} + } + ... on Organization { + ${ORGANIZATION_FIELDS} + } + } +` + +const REPLY_FIELDS = ` + id + bodyText + url + createdAt +` + +export const DISCUSSIONS_QUERY = ` + query ($owner: String!, $repo: String!, $first: Int!, $cursor: String) { + repository(owner: $owner, name: $repo) { + discussions( + first: $first + after: $cursor + orderBy: {field: UPDATED_AT, direction: DESC} + ) { + pageInfo { + endCursor + hasNextPage + } + nodes { + id + number + title + bodyText + url + createdAt + updatedAt + isAnswered + ${AUTHOR_FIELDS} + category { + id + isAnswerable + name + slug + emoji + description + } + comments { + totalCount + } + } + } + } + } +` + +export const DISCUSSION_COMMENTS_QUERY = ` + query ($id: ID!, $first: Int!, $cursor: String, $repliesFirst: Int!) { + node(id: $id) { + ... on Discussion { + comments(first: $first, after: $cursor) { + pageInfo { + endCursor + hasNextPage + } + nodes { + ${REPLY_FIELDS} + replyTo { + id + } + replies(first: $repliesFirst) { + pageInfo { + endCursor + hasNextPage + } + nodes { + ${REPLY_FIELDS} + } + } + } + } + } + } + } +` + +export const COMMENT_REPLIES_QUERY = ` + query ($id: ID!, $first: Int!, $cursor: String) { + node(id: $id) { + ... on DiscussionComment { + replies(first: $first, after: $cursor) { + pageInfo { + endCursor + hasNextPage + } + nodes { + ${REPLY_FIELDS} + } + } + } + } + } +` diff --git a/services/libs/connectors/src/connectors/github/mappers/discussion.ts b/services/libs/connectors/src/connectors/github/mappers/discussion.ts new file mode 100644 index 0000000000..f23b6889c4 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/mappers/discussion.ts @@ -0,0 +1,52 @@ +import { GITHUB_GRID, GithubActivityType } from '@crowd/integrations' + +import type { DiscussionNode, DiscussionReplyNode } from '../graphql/discussions' +import type { GithubActivity } from '../schemas' + +import { toMember } from './member' + +const DEFAULT_TIMESTAMP = '1970-01-01T00:00:00Z' + +export function toDiscussionStartedActivity(discussion: DiscussionNode): GithubActivity { + return { + type: GithubActivityType.DISCUSSION_STARTED, + timestamp: discussion.createdAt ?? DEFAULT_TIMESTAMP, + sourceId: discussion.id, + score: GITHUB_GRID[GithubActivityType.DISCUSSION_STARTED].score, + title: discussion.title || '', + body: discussion.bodyText || '', + url: discussion.url, + attributes: { + category: { + id: String(discussion.category.id), + isAnswerable: String(discussion.category.isAnswerable), + name: discussion.category.name, + slug: discussion.category.slug, + emoji: discussion.category.emoji, + description: discussion.category.description, + }, + }, + member: toMember(discussion.author), + } +} + +export function toDiscussionCommentActivity( + discussion: DiscussionNode, + comment: DiscussionReplyNode, + isReply: boolean, +): GithubActivity { + return { + type: GithubActivityType.DISCUSSION_COMMENT, + timestamp: comment.createdAt ?? DEFAULT_TIMESTAMP, + sourceId: comment.id, + sourceParentId: discussion.id, + score: + discussion.isAnswered && !isReply + ? GITHUB_GRID[GithubActivityType.DISCUSSION_COMMENT].score + 2 + : GITHUB_GRID[GithubActivityType.DISCUSSION_COMMENT].score, + body: comment.bodyText || '', + url: comment.url, + attributes: { isAnswer: discussion.isAnswered ?? false }, + member: toMember(discussion.author), + } +} diff --git a/services/libs/connectors/src/connectors/github/syncs/discussions.ts b/services/libs/connectors/src/connectors/github/syncs/discussions.ts new file mode 100644 index 0000000000..b2d62885fc --- /dev/null +++ b/services/libs/connectors/src/connectors/github/syncs/discussions.ts @@ -0,0 +1,150 @@ +import { mapWithConcurrency } from '../../../concurrency' +import type { SyncContext, SyncDefinition } from '../../../types' +import { githubGraphql } from '../gql' +import type { + CommentRepliesPage, + DiscussionCommentNode, + DiscussionCommentsPage, + DiscussionNode, + DiscussionReplyNode, + DiscussionsPage, +} from '../graphql/discussions' +import { + COMMENT_REPLIES_QUERY, + DISCUSSIONS_QUERY, + DISCUSSION_COMMENTS_QUERY, +} from '../graphql/discussions' +import { toDiscussionCommentActivity, toDiscussionStartedActivity } from '../mappers/discussion' +import { + ITEM_FETCH_CONCURRENCY, + MAX_PAGES_PER_RUN, + PAGE_SIZE, + parseRepoChannel, + readWatermark, +} from '../paging' +import type { GithubActivity } from '../schemas' +import { githubActivitySchema } from '../schemas' + +const COMMENTS_PAGE_SIZE = 50 +const REPLIES_PAGE_SIZE = 100 + +async function runDiscussionsSync(ctx: SyncContext): Promise { + const { owner, repo } = parseRepoChannel(ctx.channel.channelName) + const watermark = readWatermark(ctx.watermark) + + const sinceDate = watermark.since ? new Date(watermark.since) : null + let newestUpdatedAt = watermark.since + let cursor: string | null = null + + const collectReplies = async ( + discussion: DiscussionNode, + comment: DiscussionCommentNode, + ): Promise => { + const activities = comment.replies.nodes + .filter((node): node is DiscussionReplyNode => node !== null) + .map((reply) => toDiscussionCommentActivity(discussion, reply, true)) + + let repliesCursor = comment.replies.pageInfo.hasNextPage + ? comment.replies.pageInfo.endCursor + : null + while (repliesCursor) { + const data = await githubGraphql(ctx.http, COMMENT_REPLIES_QUERY, { + id: comment.id, + first: REPLIES_PAGE_SIZE, + cursor: repliesCursor, + }) + const replies = data.node?.replies + if (!replies) { + break + } + activities.push( + ...replies.nodes + .filter((node): node is DiscussionReplyNode => node !== null) + .map((reply) => toDiscussionCommentActivity(discussion, reply, true)), + ) + repliesCursor = replies.pageInfo.hasNextPage ? replies.pageInfo.endCursor : null + } + + return activities + } + + const collectDiscussionActivities = async ( + discussion: DiscussionNode, + ): Promise => { + const activities: GithubActivity[] = [toDiscussionStartedActivity(discussion)] + + let commentsCursor: string | null = null + let hasMore = discussion.comments.totalCount > 0 + while (hasMore) { + const data = await githubGraphql( + ctx.http, + DISCUSSION_COMMENTS_QUERY, + { + id: discussion.id, + first: COMMENTS_PAGE_SIZE, + cursor: commentsCursor, + repliesFirst: REPLIES_PAGE_SIZE, + }, + ) + const comments = data.node?.comments + if (!comments) { + break + } + for (const comment of comments.nodes.filter( + (node): node is DiscussionCommentNode => node !== null, + )) { + activities.push(toDiscussionCommentActivity(discussion, comment, Boolean(comment.replyTo))) + activities.push(...(await collectReplies(discussion, comment))) + } + hasMore = comments.pageInfo.hasNextPage + commentsCursor = comments.pageInfo.endCursor + } + + return activities + } + + // discussions has no filterBy.since; walk UPDATED_AT DESC and stop at the watermark, + // committing only after the walk so a partial run cannot skip older updates. + for (let page = 0; page < MAX_PAGES_PER_RUN; page++) { + const data = await githubGraphql(ctx.http, DISCUSSIONS_QUERY, { + owner, + repo, + first: PAGE_SIZE, + cursor, + }) + + const { pageInfo, nodes } = data.repository.discussions + const discussions = nodes.filter((node): node is DiscussionNode => node !== null) + const fresh = sinceDate + ? discussions.filter((discussion) => new Date(discussion.updatedAt) >= sinceDate) + : discussions + + if (page === 0 && discussions.length > 0) { + newestUpdatedAt = discussions[0].updatedAt + } + + if (fresh.length > 0) { + const batches = await mapWithConcurrency( + fresh, + ITEM_FETCH_CONCURRENCY, + collectDiscussionActivities, + ) + await ctx.emit(batches.flat()) + } + + const reachedWatermark = fresh.length < discussions.length + cursor = pageInfo.hasNextPage ? pageInfo.endCursor : null + if (reachedWatermark || !pageInfo.hasNextPage) { + break + } + } + + await ctx.commitWatermark({ phase: 'incremental', since: newestUpdatedAt, cursor: null }) +} + +export const discussionsSync: SyncDefinition = { + name: 'discussions', + cadenceMinutes: 60, + schema: githubActivitySchema, + run: runDiscussionsSync, +} From e918730e501c392b41720b32820a7de6cb74464d Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Tue, 1 Sep 2026 18:20:41 +0100 Subject: [PATCH 43/69] feat: add github stars sync and register discussions and stars in manifest Signed-off-by: Mouad BANI --- .../src/connectors/github/graphql/stars.ts | 40 ++++++++++++++++ .../connectors/src/connectors/github/index.ts | 4 ++ .../src/connectors/github/mappers/star.ts | 26 ++++++++++ .../src/connectors/github/syncs/stars.ts | 48 +++++++++++++++++++ 4 files changed, 118 insertions(+) create mode 100644 services/libs/connectors/src/connectors/github/graphql/stars.ts create mode 100644 services/libs/connectors/src/connectors/github/mappers/star.ts create mode 100644 services/libs/connectors/src/connectors/github/syncs/stars.ts diff --git a/services/libs/connectors/src/connectors/github/graphql/stars.ts b/services/libs/connectors/src/connectors/github/graphql/stars.ts new file mode 100644 index 0000000000..fd0c2f0e56 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/graphql/stars.ts @@ -0,0 +1,40 @@ +import type { GithubUserNode } from '../mappers/member' + +import { USER_FIELDS } from './fields' + +export interface StargazerEdge { + starredAt: string + node: GithubUserNode | null +} + +export interface StargazersPage { + repository: { + stargazers: { + pageInfo: { + endCursor: string | null + hasNextPage: boolean + } + edges: (StargazerEdge | null)[] + } + } +} + +export const STARGAZERS_QUERY = ` + query ($owner: String!, $repo: String!, $first: Int!, $cursor: String) { + repository(owner: $owner, name: $repo) { + stargazers(first: $first, after: $cursor, orderBy: {field: STARRED_AT, direction: ASC}) { + pageInfo { + endCursor + hasNextPage + } + edges { + starredAt + node { + login + ${USER_FIELDS} + } + } + } + } + } +` diff --git a/services/libs/connectors/src/connectors/github/index.ts b/services/libs/connectors/src/connectors/github/index.ts index dc6abea660..16884e6566 100644 --- a/services/libs/connectors/src/connectors/github/index.ts +++ b/services/libs/connectors/src/connectors/github/index.ts @@ -4,6 +4,7 @@ import { seedGithubTokens } from './appToken' import { probeGithubBudget } from './budget' import { discoverRepos } from './discover' import { interpretGithubResponse } from './interpret' +import { discussionsSync } from './syncs/discussions' import { forksSync } from './syncs/forks' import { issueCommentsSync } from './syncs/issueComments' import { issuesSync } from './syncs/issues' @@ -11,10 +12,12 @@ import { pullRequestCommentsSync } from './syncs/pullRequestComments' import { pullRequestCommitsSync } from './syncs/pullRequestCommits' import { pullRequestReviewCommentsSync } from './syncs/pullRequestReviewComments' import { pullRequestsSync } from './syncs/pullRequests' +import { starsSync } from './syncs/stars' export const githubConnector: Manifest = { platform: 'github', syncs: [ + discussionsSync, forksSync, issuesSync, issueCommentsSync, @@ -22,6 +25,7 @@ export const githubConnector: Manifest = { pullRequestCommentsSync, pullRequestReviewCommentsSync, pullRequestCommitsSync, + starsSync, ], discover: discoverRepos, seedTokens: seedGithubTokens, diff --git a/services/libs/connectors/src/connectors/github/mappers/star.ts b/services/libs/connectors/src/connectors/github/mappers/star.ts new file mode 100644 index 0000000000..a44bcff682 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/mappers/star.ts @@ -0,0 +1,26 @@ +import { createHash } from 'crypto' + +import { GITHUB_GRID, GithubActivityType } from '@crowd/integrations' + +import type { StargazerEdge } from '../graphql/stars' +import type { GithubActivity } from '../schemas' + +import { toMember } from './member' + +const DEFAULT_TIMESTAMP = '1970-01-01T00:00:00Z' + +export function toStar(edge: StargazerEdge): GithubActivity { + const timestamp = edge.starredAt ?? DEFAULT_TIMESTAMP + const sourceIdData = `${edge.node?.login}-star-${Math.floor( + new Date(timestamp).getTime() / 1000, + )}-github` + + return { + type: GithubActivityType.STAR, + timestamp, + sourceId: `gen-${createHash('md5').update(sourceIdData).digest('hex')}`, + score: GITHUB_GRID[GithubActivityType.STAR].score, + attributes: {}, + member: toMember(edge.node), + } +} diff --git a/services/libs/connectors/src/connectors/github/syncs/stars.ts b/services/libs/connectors/src/connectors/github/syncs/stars.ts new file mode 100644 index 0000000000..e83b3d5423 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/syncs/stars.ts @@ -0,0 +1,48 @@ +import type { SyncContext, SyncDefinition } from '../../../types' +import { githubGraphql } from '../gql' +import type { StargazerEdge, StargazersPage } from '../graphql/stars' +import { STARGAZERS_QUERY } from '../graphql/stars' +import { toStar } from '../mappers/star' +import { MAX_PAGES_PER_RUN, PAGE_SIZE, parseRepoChannel, readWatermark } from '../paging' +import { githubActivitySchema } from '../schemas' + +async function runStarsSync(ctx: SyncContext): Promise { + const { owner, repo } = parseRepoChannel(ctx.channel.channelName) + const watermark = readWatermark(ctx.watermark) + + let since = watermark.since + let cursor = watermark.cursor + + for (let page = 0; page < MAX_PAGES_PER_RUN; page++) { + const data = await githubGraphql(ctx.http, STARGAZERS_QUERY, { + owner, + repo, + first: PAGE_SIZE, + cursor, + }) + + const { pageInfo, edges } = data.repository.stargazers + const stargazers = edges + .filter((edge): edge is StargazerEdge => edge !== null) + .filter((edge) => !since || edge.starredAt > since) + + if (stargazers.length > 0) { + await ctx.emit(stargazers.map(toStar)) + since = stargazers[stargazers.length - 1].starredAt + } + + cursor = pageInfo.endCursor ?? cursor + await ctx.commitWatermark({ phase: 'incremental', since, cursor }) + + if (!pageInfo.hasNextPage) { + return + } + } +} + +export const starsSync: SyncDefinition = { + name: 'stars', + cadenceMinutes: 1440, + schema: githubActivitySchema, + run: runStarsSync, +} From cdcd3c8e476f86998bde6cca576d08866fc68b36 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Wed, 2 Sep 2026 11:13:28 +0100 Subject: [PATCH 44/69] fix: classify github app token minting errors by status Signed-off-by: Mouad BANI --- .../src/connectors/github/appToken.ts | 81 +++++++++++++------ 1 file changed, 57 insertions(+), 24 deletions(-) diff --git a/services/libs/connectors/src/connectors/github/appToken.ts b/services/libs/connectors/src/connectors/github/appToken.ts index 5906662530..c16c826be5 100644 --- a/services/libs/connectors/src/connectors/github/appToken.ts +++ b/services/libs/connectors/src/connectors/github/appToken.ts @@ -1,6 +1,12 @@ import axios from 'axios' import * as jwt from 'jsonwebtoken' +import { + ConnectorError, + ProviderAuthError, + ProviderContractError, + ProviderUnavailableError, +} from '../../http/errors' import type { TokenPool } from '../../pool/tokenPool' import type { Credential } from '../../types' @@ -17,25 +23,48 @@ function mintAppJwt(credential: Credential): string { ) } +function classifyAppApiError(err: unknown, operation: string): ConnectorError | null { + if (!axios.isAxiosError(err)) { + return null + } + const status = err.response?.status + const body = err.response?.data as { message?: string } | undefined + const detail = body?.message ?? err.message + const options = { status, cause: err } + if (status === 401) { + return new ProviderAuthError(`github app ${operation} unauthorized: ${detail}`, options) + } + if (status === undefined || status >= 500) { + return new ProviderUnavailableError(`github app ${operation} failed: ${detail}`, options) + } + return new ProviderContractError( + `github app ${operation} returned status ${status}: ${detail}`, + options, + ) +} + export async function mintInstallationToken( credential: Credential, installationId: string, ): Promise<{ token: string; expiresAt: string }> { - // POC only: minting cannot go through ConnectorHttp — it feeds the pool the client draws from - const response = await axios.post( - `https://api.github.com/app/installations/${installationId}/access_tokens`, - {}, - { - headers: { - Authorization: `Bearer ${mintAppJwt(credential)}`, - Accept: 'application/vnd.github+json', - 'X-GitHub-Api-Version': GITHUB_API_VERSION, + try { + // POC only: minting cannot go through ConnectorHttp — it feeds the pool the client draws from + const response = await axios.post( + `https://api.github.com/app/installations/${installationId}/access_tokens`, + {}, + { + headers: { + Authorization: `Bearer ${mintAppJwt(credential)}`, + Accept: 'application/vnd.github+json', + 'X-GitHub-Api-Version': GITHUB_API_VERSION, + }, + timeout: GITHUB_REQUEST_TIMEOUT_MS, }, - timeout: GITHUB_REQUEST_TIMEOUT_MS, - }, - ) - - return { token: response.data.token, expiresAt: response.data.expires_at } + ) + return { token: response.data.token, expiresAt: response.data.expires_at } + } catch (err) { + throw classifyAppApiError(err, 'installation token minting') ?? err + } } // TODO(CM-1372): POC-only resolution; store the installation id per integration after the POC @@ -45,16 +74,20 @@ export async function resolveInstallationId(credential: Credential): Promise Date: Wed, 2 Sep 2026 11:29:34 +0100 Subject: [PATCH 45/69] fix: classify github 403s by cause instead of defaulting to auth Signed-off-by: Mouad BANI --- .../src/connectors/github/interpret.ts | 32 ++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/services/libs/connectors/src/connectors/github/interpret.ts b/services/libs/connectors/src/connectors/github/interpret.ts index ff51c7dd90..143ad942d4 100644 --- a/services/libs/connectors/src/connectors/github/interpret.ts +++ b/services/libs/connectors/src/connectors/github/interpret.ts @@ -1,22 +1,46 @@ import type { ResponseInterpreter } from '../../http/client' -import { RateLimitError } from '../../http/errors' +import { ProviderAuthError, ProviderContractError, RateLimitError } from '../../http/errors' interface GithubErrorBody { errors?: { type?: string }[] message?: string } +// Legacy prod parity (baseQuery.ts): GitHub's "temporarily blocked" 403 is retryable +const RETRYABLE_403_MESSAGES = [ + 'secondary rate limit', + 'although you appear to have the correct authorization credentials', +] + +function hasRateLimitHeaders(headers: Record): boolean { + return headers['retry-after'] !== undefined || headers['x-ratelimit-remaining'] === '0' +} + export const interpretGithubResponse: ResponseInterpreter = (response) => { const body = response.data as GithubErrorBody | null if (response.status === 200) { // GitHub docs say RATE_LIMITED, but installation tokens return RATE_LIMIT - if (body?.errors?.some((e) => e.type === 'RATE_LIMITED' || e.type === 'RATE_LIMIT')) { + if (body?.errors?.some((e) => e.type?.includes('RATE_LIMIT'))) { return new RateLimitError('github graphql rate limited') } return null } - if (response.status === 403 && body?.message?.toLowerCase().includes('secondary rate limit')) { - return new RateLimitError('github secondary rate limited') + if (response.status === 401) { + return new ProviderAuthError(`github unauthorized: ${body?.message ?? 'no message'}`, { + status: 401, + }) + } + if (response.status === 403) { + const message = body?.message?.toLowerCase() ?? '' + if (RETRYABLE_403_MESSAGES.some((m) => message.includes(m))) { + return new RateLimitError(`github rate limited: ${body?.message}`) + } + if (hasRateLimitHeaders(response.headers)) { + return null + } + return new ProviderContractError(`github forbidden: ${body?.message ?? 'no message'}`, { + status: 403, + }) } return null } From 531e71f7643a539ffab0314f2612a34d511fa254 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Wed, 2 Sep 2026 11:41:25 +0100 Subject: [PATCH 46/69] fix: park unavailable runs and dead-letter sync units on auth failures only Signed-off-by: Mouad BANI --- .../src/activities/syncRunActivities.ts | 17 +++++++++++++---- .../src/connectors/syncUnits.ts | 5 +++-- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/services/apps/connectors_worker/src/activities/syncRunActivities.ts b/services/apps/connectors_worker/src/activities/syncRunActivities.ts index 2be605e7b3..23295963c4 100644 --- a/services/apps/connectors_worker/src/activities/syncRunActivities.ts +++ b/services/apps/connectors_worker/src/activities/syncRunActivities.ts @@ -27,6 +27,9 @@ import { svc } from '../main' const DEAD_LETTER_AFTER = 5 const HEARTBEAT_INTERVAL_MS = 10_000 const RATE_LIMIT_FALLBACK_MS = 60_000 +const UNAVAILABLE_PARK_MS = 300_000 + +const PARKED_ERROR_CLASSES = ['provider.rate_limit', 'provider.unavailable'] as const export async function executeSync(unitId: string): Promise { const qx = dbStoreQx(svc.postgres.writer) @@ -112,8 +115,13 @@ export async function executeSync(unitId: string): Promise { }) log.info({ emittedCount: emitter.emittedCount() }, 'sync run succeeded') } catch (err) { - if (err instanceof ConnectorError && err.errorClass === 'provider.rate_limit') { - const resumeAt = err.options?.resumeAt ?? new Date(Date.now() + RATE_LIMIT_FALLBACK_MS) + if ( + err instanceof ConnectorError && + (PARKED_ERROR_CLASSES as readonly string[]).includes(err.errorClass) + ) { + const fallbackMs = + err.errorClass === 'provider.rate_limit' ? RATE_LIMIT_FALLBACK_MS : UNAVAILABLE_PARK_MS + const resumeAt = err.options?.resumeAt ?? new Date(Date.now() + fallbackMs) if (emitter && committedWatermark) { await recordRunPartial( qx, @@ -125,12 +133,13 @@ export async function executeSync(unitId: string): Promise { } else { await parkUnit(qx, unitId, resumeAt, err.errorClass) } - log.info({ resumeAt }, 'sync run rate-limit parked') + log.info({ resumeAt, errorClass: err.errorClass }, 'sync run parked') return } const errorClass = err instanceof ConnectorError ? err.errorClass : 'unknown' + const deadLetterAfter = errorClass === 'provider.auth' ? DEAD_LETTER_AFTER : null log.error(err, 'sync run failed') - await recordRunFailure(qx, unitId, errorClass, DEAD_LETTER_AFTER) + await recordRunFailure(qx, unitId, errorClass, deadLetterAfter) throw err } finally { clearInterval(heartbeat) diff --git a/services/libs/data-access-layer/src/connectors/syncUnits.ts b/services/libs/data-access-layer/src/connectors/syncUnits.ts index 873f15f8e8..9c73d70ae6 100644 --- a/services/libs/data-access-layer/src/connectors/syncUnits.ts +++ b/services/libs/data-access-layer/src/connectors/syncUnits.ts @@ -141,14 +141,15 @@ export async function recordRunFailure( qx: QueryExecutor, id: string, errorClass: string, - deadLetterAfter: number, + deadLetterAfter: number | null, ): Promise { await qx.result( `UPDATE integration.sync_units SET "consecutiveFailures" = "consecutiveFailures" + 1, "lastErrorClass" = $(errorClass), "lastRunAt" = now(), - status = CASE WHEN "consecutiveFailures" + 1 >= $(deadLetterAfter) + status = CASE WHEN $(deadLetterAfter)::int IS NOT NULL + AND "consecutiveFailures" + 1 >= $(deadLetterAfter) THEN 'dead_letter' ELSE status END, "updatedAt" = now() WHERE id = $(id)`, From d530dce40a4ee850a0265f47eca93470c9706710 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Wed, 2 Sep 2026 11:52:53 +0100 Subject: [PATCH 47/69] feat: encode platform, sync and channel in sync-run workflow ids Signed-off-by: Mouad BANI --- .../src/activities/dispatcherActivities.ts | 25 ++++++++++++++++++- .../src/connectors/syncUnits.ts | 2 +- .../data-access-layer/src/connectors/types.ts | 5 +++- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/services/apps/connectors_worker/src/activities/dispatcherActivities.ts b/services/apps/connectors_worker/src/activities/dispatcherActivities.ts index d0ebedf676..b465c9ff32 100644 --- a/services/apps/connectors_worker/src/activities/dispatcherActivities.ts +++ b/services/apps/connectors_worker/src/activities/dispatcherActivities.ts @@ -15,6 +15,7 @@ const DEFAULT_RUN_ESTIMATE = 50 const DEFER_MIN_MS = 30_000 const DEFER_JITTER_MS = 60_000 const BUDGET_PROBE_CONCURRENCY = 10 +const CHANNEL_LABEL_MAX_LENGTH = 80 export async function claimDue(limit: number): Promise { return claimDueUnits(dbStoreQx(svc.postgres.writer), limit) @@ -39,14 +40,36 @@ export async function deferUnit(unitId: string): Promise { await rescheduleUnit(dbStoreQx(svc.postgres.writer), unitId, new Date(Date.now() + delayMs)) } +function channelLabel(channelName: string): string { + const label = channelName + .replace(/^[a-z][a-z0-9+.-]*:\/\/[^/]+\//i, '') + .replace(/^\/+|\/+$/g, '') + .replace(/\s+/g, '-') + .slice(0, CHANNEL_LABEL_MAX_LENGTH) + + return label || 'unknown-channel' +} + +export function syncRunWorkflowId(unit: IClaimedUnit): string { + return `sync-run/${unit.platform}/${unit.syncName}/${channelLabel(unit.channelName)}/${unit.id}` +} + export async function startRun(unit: IClaimedUnit): Promise { try { await svc.temporal.workflow.start('syncRun', { taskQueue: TASK_QUEUE, - workflowId: `sync-run/${unit.id}`, + workflowId: syncRunWorkflowId(unit), workflowIdReusePolicy: WorkflowIdReusePolicy.ALLOW_DUPLICATE, workflowIdConflictPolicy: WorkflowIdConflictPolicy.FAIL, args: [unit.id], + memo: { + unitId: unit.id, + integrationId: unit.integrationId, + platform: unit.platform, + syncName: unit.syncName, + channelId: unit.channelId, + channelName: unit.channelName, + }, }) return 'started' } catch (err) { diff --git a/services/libs/data-access-layer/src/connectors/syncUnits.ts b/services/libs/data-access-layer/src/connectors/syncUnits.ts index 9c73d70ae6..3fc455b58c 100644 --- a/services/libs/data-access-layer/src/connectors/syncUnits.ts +++ b/services/libs/data-access-layer/src/connectors/syncUnits.ts @@ -55,7 +55,7 @@ export async function claimDueUnits(qx: QueryExecutor, limit: number): Promise -export type IClaimedUnit = Pick +export type IClaimedUnit = Pick< + ISyncUnit, + 'id' | 'integrationId' | 'platform' | 'syncName' | 'channelId' | 'channelName' +> export interface ISyncRunSuccess { watermark: Record From e07421d1c96642cd49efd7573699be2bac6f7725 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Wed, 2 Sep 2026 12:31:25 +0100 Subject: [PATCH 48/69] feat: fast-track incomplete sync runs and bound runs by wall clock Signed-off-by: Mouad BANI --- .../apps/connectors_worker/src/activities.ts | 4 +-- .../src/activities/dispatcherActivities.ts | 28 ++++++++----------- .../src/activities/syncRunActivities.ts | 25 +++++++++++++---- .../apps/connectors_worker/src/runLimits.ts | 7 +++++ .../apps/connectors_worker/src/scheduling.ts | 12 ++++++++ .../src/workflows/dispatcher.ts | 2 +- .../src/workflows/syncRun.ts | 3 +- .../src/connectors/github/paging.ts | 1 - .../src/connectors/github/prWalk.ts | 27 ++++++++++-------- .../connectors/github/syncs/discussions.ts | 26 +++++++++-------- .../src/connectors/github/syncs/forks.ts | 12 ++++---- .../connectors/github/syncs/issueComments.ts | 12 ++++---- .../src/connectors/github/syncs/issues.ts | 12 ++++---- .../github/syncs/pullRequestComments.ts | 6 ++-- .../github/syncs/pullRequestCommits.ts | 6 ++-- .../github/syncs/pullRequestReviewComments.ts | 6 ++-- .../connectors/github/syncs/pullRequests.ts | 6 ++-- .../src/connectors/github/syncs/stars.ts | 12 ++++---- .../connectors/src/testing/dummyConnector.ts | 1 + services/libs/connectors/src/types.ts | 7 ++++- .../src/connectors/syncUnits.ts | 25 ++++++++++++++++- 21 files changed, 155 insertions(+), 85 deletions(-) create mode 100644 services/apps/connectors_worker/src/runLimits.ts create mode 100644 services/apps/connectors_worker/src/scheduling.ts diff --git a/services/apps/connectors_worker/src/activities.ts b/services/apps/connectors_worker/src/activities.ts index 054ba53dff..df1cfdbd59 100644 --- a/services/apps/connectors_worker/src/activities.ts +++ b/services/apps/connectors_worker/src/activities.ts @@ -2,10 +2,10 @@ import { admitByBudget, claimDue, deferUnit, - reschedule, + guardLease, startRun, touchHeartbeat, } from './activities/dispatcherActivities' import { executeSync } from './activities/syncRunActivities' -export { admitByBudget, claimDue, deferUnit, executeSync, reschedule, startRun, touchHeartbeat } +export { admitByBudget, claimDue, deferUnit, executeSync, guardLease, startRun, touchHeartbeat } diff --git a/services/apps/connectors_worker/src/activities/dispatcherActivities.ts b/services/apps/connectors_worker/src/activities/dispatcherActivities.ts index b465c9ff32..50ba471fca 100644 --- a/services/apps/connectors_worker/src/activities/dispatcherActivities.ts +++ b/services/apps/connectors_worker/src/activities/dispatcherActivities.ts @@ -1,19 +1,22 @@ -import { createTokenPool, findManifest, getSync, mapWithConcurrency } from '@crowd/connectors' -import { claimDueUnits, rescheduleUnit } from '@crowd/data-access-layer/src/connectors' +import { createTokenPool, findManifest, mapWithConcurrency } from '@crowd/connectors' +import { + claimDueUnits, + guardUnitLease, + rescheduleUnit, +} from '@crowd/data-access-layer/src/connectors' import type { IClaimedUnit } from '@crowd/data-access-layer/src/connectors' import { dbStoreQx } from '@crowd/data-access-layer/src/queryExecutor' import { RedisCache } from '@crowd/redis' import { WorkflowIdConflictPolicy, WorkflowIdReusePolicy } from '@crowd/temporal' import { svc } from '../main' +import { LEASE_GUARD_MS } from '../runLimits' +import { shortDeferRunAt } from '../scheduling' import type { IAdmissionResult, StartRunResult } from '../types' const TASK_QUEUE = 'connectors' const HEARTBEAT_TTL_SECONDS = 300 -const CADENCE_JITTER_RATIO = 0.1 const DEFAULT_RUN_ESTIMATE = 50 -const DEFER_MIN_MS = 30_000 -const DEFER_JITTER_MS = 60_000 const BUDGET_PROBE_CONCURRENCY = 10 const CHANNEL_LABEL_MAX_LENGTH = 80 @@ -36,8 +39,7 @@ export async function admitByBudget(units: IClaimedUnit[]): Promise { - const delayMs = DEFER_MIN_MS + Math.random() * DEFER_JITTER_MS - await rescheduleUnit(dbStoreQx(svc.postgres.writer), unitId, new Date(Date.now() + delayMs)) + await rescheduleUnit(dbStoreQx(svc.postgres.writer), unitId, shortDeferRunAt()) } function channelLabel(channelName: string): string { @@ -80,16 +82,8 @@ export async function startRun(unit: IClaimedUnit): Promise { } } -export async function reschedule( - unitId: string, - platform: string, - syncName: string, -): Promise { - const { cadenceMinutes } = getSync(platform, syncName) - const jitterMinutes = cadenceMinutes * CADENCE_JITTER_RATIO * (Math.random() * 2 - 1) - const nextRunAt = new Date(Date.now() + (cadenceMinutes + jitterMinutes) * 60_000) - - await rescheduleUnit(dbStoreQx(svc.postgres.writer), unitId, nextRunAt) +export async function guardLease(unitId: string): Promise { + await guardUnitLease(dbStoreQx(svc.postgres.writer), unitId, LEASE_GUARD_MS) } export async function touchHeartbeat(): Promise { diff --git a/services/apps/connectors_worker/src/activities/syncRunActivities.ts b/services/apps/connectors_worker/src/activities/syncRunActivities.ts index 23295963c4..250d4287a2 100644 --- a/services/apps/connectors_worker/src/activities/syncRunActivities.ts +++ b/services/apps/connectors_worker/src/activities/syncRunActivities.ts @@ -23,6 +23,8 @@ import { dbStoreQx } from '@crowd/data-access-layer/src/queryExecutor' import { getChildLogger } from '@crowd/logging' import { svc } from '../main' +import { RUN_BUDGET_MS } from '../runLimits' +import { cadenceRunAt, shortDeferRunAt } from '../scheduling' const DEAD_LETTER_AFTER = 5 const HEARTBEAT_INTERVAL_MS = 10_000 @@ -58,6 +60,7 @@ export async function executeSync(unitId: string): Promise { let emitter: Emitter | null = null let committedWatermark = unit.watermark + const runDeadline = Date.now() + RUN_BUDGET_MS try { const integration = await fetchIntegrationById(qx, unit.integrationId) @@ -103,17 +106,27 @@ export async function executeSync(unitId: string): Promise { commitWatermark: async (watermark) => { committedWatermark = watermark }, + hasRunBudget: () => Date.now() < runDeadline, http, log, } - await sync.run(ctx) + const outcome = await sync.run(ctx) + const nextRunAt = outcome.complete ? cadenceRunAt(sync.cadenceMinutes) : shortDeferRunAt() - await recordRunSuccess(qx, unitId, { - watermark: committedWatermark ?? {}, - emittedCount: emitter.emittedCount(), - }) - log.info({ emittedCount: emitter.emittedCount() }, 'sync run succeeded') + await recordRunSuccess( + qx, + unitId, + { + watermark: committedWatermark ?? {}, + emittedCount: emitter.emittedCount(), + }, + nextRunAt, + ) + log.info( + { emittedCount: emitter.emittedCount(), complete: outcome.complete, nextRunAt }, + 'sync run succeeded', + ) } catch (err) { if ( err instanceof ConnectorError && diff --git a/services/apps/connectors_worker/src/runLimits.ts b/services/apps/connectors_worker/src/runLimits.ts new file mode 100644 index 0000000000..505c536937 --- /dev/null +++ b/services/apps/connectors_worker/src/runLimits.ts @@ -0,0 +1,7 @@ +const MINUTE_MS = 60_000 + +export const RUN_START_TO_CLOSE_TIMEOUT_MS = 60 * MINUTE_MS + +export const RUN_BUDGET_MS = RUN_START_TO_CLOSE_TIMEOUT_MS - 10 * MINUTE_MS + +export const LEASE_GUARD_MS = RUN_START_TO_CLOSE_TIMEOUT_MS + 5 * MINUTE_MS diff --git a/services/apps/connectors_worker/src/scheduling.ts b/services/apps/connectors_worker/src/scheduling.ts new file mode 100644 index 0000000000..b0af66943d --- /dev/null +++ b/services/apps/connectors_worker/src/scheduling.ts @@ -0,0 +1,12 @@ +const CADENCE_JITTER_RATIO = 0.1 +const SHORT_DEFER_MIN_MS = 30_000 +const SHORT_DEFER_JITTER_MS = 60_000 + +export function cadenceRunAt(cadenceMinutes: number): Date { + const jitterMinutes = cadenceMinutes * CADENCE_JITTER_RATIO * (Math.random() * 2 - 1) + return new Date(Date.now() + (cadenceMinutes + jitterMinutes) * 60_000) +} + +export function shortDeferRunAt(): Date { + return new Date(Date.now() + SHORT_DEFER_MIN_MS + Math.random() * SHORT_DEFER_JITTER_MS) +} diff --git a/services/apps/connectors_worker/src/workflows/dispatcher.ts b/services/apps/connectors_worker/src/workflows/dispatcher.ts index f3d800e79c..3724d551a1 100644 --- a/services/apps/connectors_worker/src/workflows/dispatcher.ts +++ b/services/apps/connectors_worker/src/workflows/dispatcher.ts @@ -19,7 +19,7 @@ export async function dispatcher(): Promise { for (const unit of admitted) { try { await activity.startRun(unit) - await activity.reschedule(unit.id, unit.platform, unit.syncName) + await activity.guardLease(unit.id) } catch (err) { log.error('failed to dispatch sync unit', { unitId: unit.id, err }) } diff --git a/services/apps/connectors_worker/src/workflows/syncRun.ts b/services/apps/connectors_worker/src/workflows/syncRun.ts index 40f2637e4e..4132c68566 100644 --- a/services/apps/connectors_worker/src/workflows/syncRun.ts +++ b/services/apps/connectors_worker/src/workflows/syncRun.ts @@ -1,9 +1,10 @@ import { proxyActivities } from '@temporalio/workflow' import type * as activities from '../activities/syncRunActivities' +import { RUN_START_TO_CLOSE_TIMEOUT_MS } from '../runLimits' const activity = proxyActivities({ - startToCloseTimeout: '30 minutes', + startToCloseTimeout: RUN_START_TO_CLOSE_TIMEOUT_MS, heartbeatTimeout: '1 minute', retry: { maximumAttempts: 1 }, }) diff --git a/services/libs/connectors/src/connectors/github/paging.ts b/services/libs/connectors/src/connectors/github/paging.ts index 983706b8b1..2f479d418d 100644 --- a/services/libs/connectors/src/connectors/github/paging.ts +++ b/services/libs/connectors/src/connectors/github/paging.ts @@ -4,7 +4,6 @@ export interface GithubWatermark { cursor: string | null } -export const MAX_PAGES_PER_RUN = Number.POSITIVE_INFINITY export const PAGE_SIZE = 100 // GitHub GraphQL silently omits timeline/connection items from heavy nodes(ids:) batches // (no error, pageInfo claims completeness) — fetch per item, bounded by this concurrency. diff --git a/services/libs/connectors/src/connectors/github/prWalk.ts b/services/libs/connectors/src/connectors/github/prWalk.ts index c083e89fe1..81568177bb 100644 --- a/services/libs/connectors/src/connectors/github/prWalk.ts +++ b/services/libs/connectors/src/connectors/github/prWalk.ts @@ -1,9 +1,9 @@ -import type { SyncContext } from '../../types' +import type { SyncContext, SyncOutcome } from '../../types' import { githubGraphql } from './gql' import type { PullRequestNode, PullRequestsPage } from './graphql/pullRequests' import { PULL_REQUESTS_QUERY } from './graphql/pullRequests' -import { MAX_PAGES_PER_RUN, parseRepoChannel, readWatermark } from './paging' +import { parseRepoChannel, readWatermark } from './paging' export const PR_PAGE_SIZE = 50 @@ -14,12 +14,12 @@ async function runBackfill( owner: string, repo: string, processPrs: PrPageHandler, -): Promise { +): Promise { const watermark = readWatermark(ctx.watermark) let cursor = watermark.cursor let since = watermark.since - for (let page = 0; page < MAX_PAGES_PER_RUN; page++) { + while (ctx.hasRunBudget()) { const data = await githubGraphql(ctx.http, PULL_REQUESTS_QUERY, { owner, repo, @@ -38,12 +38,14 @@ async function runBackfill( if (!pageInfo.hasNextPage) { await ctx.commitWatermark({ phase: 'incremental', since, cursor: null }) - return + return { complete: true } } cursor = pageInfo.endCursor await ctx.commitWatermark({ phase: 'backfill', since, cursor }) } + + return { complete: false } } async function runIncremental( @@ -52,12 +54,12 @@ async function runIncremental( repo: string, since: string, processPrs: PrPageHandler, -): Promise { +): Promise { const sinceDate = new Date(since) let cursor: string | null = null let newSince: string | null = null - for (let page = 0; page < MAX_PAGES_PER_RUN; page++) { + while (ctx.hasRunBudget()) { const data = await githubGraphql(ctx.http, PULL_REQUESTS_QUERY, { owner, repo, @@ -81,23 +83,24 @@ async function runIncremental( const reachedSince = fresh.length < pullRequests.length if (reachedSince || !pageInfo.hasNextPage) { await ctx.commitWatermark({ phase: 'incremental', since: newSince ?? since, cursor: null }) - return + return { complete: true } } cursor = pageInfo.endCursor } + + return { complete: false } } export async function runDualPhasePrSync( ctx: SyncContext, processPrs: PrPageHandler, -): Promise { +): Promise { const { owner, repo } = parseRepoChannel(ctx.channel.channelName) const watermark = readWatermark(ctx.watermark) if (watermark.phase === 'incremental' && watermark.since) { - await runIncremental(ctx, owner, repo, watermark.since, processPrs) - return + return runIncremental(ctx, owner, repo, watermark.since, processPrs) } - await runBackfill(ctx, owner, repo, processPrs) + return runBackfill(ctx, owner, repo, processPrs) } diff --git a/services/libs/connectors/src/connectors/github/syncs/discussions.ts b/services/libs/connectors/src/connectors/github/syncs/discussions.ts index b2d62885fc..8ea545b843 100644 --- a/services/libs/connectors/src/connectors/github/syncs/discussions.ts +++ b/services/libs/connectors/src/connectors/github/syncs/discussions.ts @@ -1,5 +1,5 @@ import { mapWithConcurrency } from '../../../concurrency' -import type { SyncContext, SyncDefinition } from '../../../types' +import type { SyncContext, SyncDefinition, SyncOutcome } from '../../../types' import { githubGraphql } from '../gql' import type { CommentRepliesPage, @@ -15,20 +15,14 @@ import { DISCUSSION_COMMENTS_QUERY, } from '../graphql/discussions' import { toDiscussionCommentActivity, toDiscussionStartedActivity } from '../mappers/discussion' -import { - ITEM_FETCH_CONCURRENCY, - MAX_PAGES_PER_RUN, - PAGE_SIZE, - parseRepoChannel, - readWatermark, -} from '../paging' +import { ITEM_FETCH_CONCURRENCY, PAGE_SIZE, parseRepoChannel, readWatermark } from '../paging' import type { GithubActivity } from '../schemas' import { githubActivitySchema } from '../schemas' const COMMENTS_PAGE_SIZE = 50 const REPLIES_PAGE_SIZE = 100 -async function runDiscussionsSync(ctx: SyncContext): Promise { +async function runDiscussionsSync(ctx: SyncContext): Promise { const { owner, repo } = parseRepoChannel(ctx.channel.channelName) const watermark = readWatermark(ctx.watermark) @@ -105,7 +99,10 @@ async function runDiscussionsSync(ctx: SyncContext): Promise { // discussions has no filterBy.since; walk UPDATED_AT DESC and stop at the watermark, // committing only after the walk so a partial run cannot skip older updates. - for (let page = 0; page < MAX_PAGES_PER_RUN; page++) { + let firstPage = true + let walked = false + + while (ctx.hasRunBudget()) { const data = await githubGraphql(ctx.http, DISCUSSIONS_QUERY, { owner, repo, @@ -119,9 +116,10 @@ async function runDiscussionsSync(ctx: SyncContext): Promise { ? discussions.filter((discussion) => new Date(discussion.updatedAt) >= sinceDate) : discussions - if (page === 0 && discussions.length > 0) { + if (firstPage && discussions.length > 0) { newestUpdatedAt = discussions[0].updatedAt } + firstPage = false if (fresh.length > 0) { const batches = await mapWithConcurrency( @@ -135,11 +133,17 @@ async function runDiscussionsSync(ctx: SyncContext): Promise { const reachedWatermark = fresh.length < discussions.length cursor = pageInfo.hasNextPage ? pageInfo.endCursor : null if (reachedWatermark || !pageInfo.hasNextPage) { + walked = true break } } + if (!walked) { + return { complete: false } + } + await ctx.commitWatermark({ phase: 'incremental', since: newestUpdatedAt, cursor: null }) + return { complete: true } } export const discussionsSync: SyncDefinition = { diff --git a/services/libs/connectors/src/connectors/github/syncs/forks.ts b/services/libs/connectors/src/connectors/github/syncs/forks.ts index 312c5b4c96..a213e08f7d 100644 --- a/services/libs/connectors/src/connectors/github/syncs/forks.ts +++ b/services/libs/connectors/src/connectors/github/syncs/forks.ts @@ -1,19 +1,19 @@ -import type { SyncContext, SyncDefinition } from '../../../types' +import type { SyncContext, SyncDefinition, SyncOutcome } from '../../../types' import { githubGraphql } from '../gql' import type { ForkNode, ForksPage } from '../graphql/forks' import { FORKS_QUERY } from '../graphql/forks' import { toFork } from '../mappers/fork' -import { MAX_PAGES_PER_RUN, PAGE_SIZE, parseRepoChannel, readWatermark } from '../paging' +import { PAGE_SIZE, parseRepoChannel, readWatermark } from '../paging' import { githubActivitySchema } from '../schemas' -async function runForksSync(ctx: SyncContext): Promise { +async function runForksSync(ctx: SyncContext): Promise { const { owner, repo } = parseRepoChannel(ctx.channel.channelName) const watermark = readWatermark(ctx.watermark) let since = watermark.since let cursor = watermark.cursor - for (let page = 0; page < MAX_PAGES_PER_RUN; page++) { + while (ctx.hasRunBudget()) { const data = await githubGraphql(ctx.http, FORKS_QUERY, { owner, repo, @@ -35,9 +35,11 @@ async function runForksSync(ctx: SyncContext): Promise { await ctx.commitWatermark({ phase: 'incremental', since, cursor }) if (!pageInfo.hasNextPage) { - return + return { complete: true } } } + + return { complete: false } } export const forksSync: SyncDefinition = { diff --git a/services/libs/connectors/src/connectors/github/syncs/issueComments.ts b/services/libs/connectors/src/connectors/github/syncs/issueComments.ts index 753b68cd98..9eb62ab049 100644 --- a/services/libs/connectors/src/connectors/github/syncs/issueComments.ts +++ b/services/libs/connectors/src/connectors/github/syncs/issueComments.ts @@ -1,4 +1,4 @@ -import type { SyncContext, SyncDefinition } from '../../../types' +import type { SyncContext, SyncDefinition, SyncOutcome } from '../../../types' import { githubGraphql } from '../gql' import type { IssueCommentNode, @@ -13,7 +13,7 @@ import { ISSUE_COMMENTS_QUERY, } from '../graphql/issues' import { toIssueComment } from '../mappers/issueComment' -import { MAX_PAGES_PER_RUN, PAGE_SIZE, parseRepoChannel, readWatermark } from '../paging' +import { PAGE_SIZE, parseRepoChannel, readWatermark } from '../paging' import { githubActivitySchema } from '../schemas' const ISSUE_BATCH_SIZE = 15 @@ -22,7 +22,7 @@ const COMMENTS_BATCH_PAGE_SIZE = 25 // (nango workaround: nango-integrations/github/syncs/issue-comments.ts) const COMMENTS_PAGINATED_PAGE_SIZE = 5 -async function runIssueCommentsSync(ctx: SyncContext): Promise { +async function runIssueCommentsSync(ctx: SyncContext): Promise { const { owner, repo } = parseRepoChannel(ctx.channel.channelName) const watermark = readWatermark(ctx.watermark) @@ -68,7 +68,7 @@ async function runIssueCommentsSync(ctx: SyncContext): Promise { } } - for (let page = 0; page < MAX_PAGES_PER_RUN; page++) { + while (ctx.hasRunBudget()) { const data = await githubGraphql(ctx.http, ISSUES_QUERY, { owner, repo, @@ -118,9 +118,11 @@ async function runIssueCommentsSync(ctx: SyncContext): Promise { await ctx.commitWatermark({ phase: 'incremental', since, cursor: null }) if (!pageInfo.hasNextPage) { - return + return { complete: true } } } + + return { complete: false } } export const issueCommentsSync: SyncDefinition = { diff --git a/services/libs/connectors/src/connectors/github/syncs/issues.ts b/services/libs/connectors/src/connectors/github/syncs/issues.ts index 2f208f1ff1..d53c0d50dc 100644 --- a/services/libs/connectors/src/connectors/github/syncs/issues.ts +++ b/services/libs/connectors/src/connectors/github/syncs/issues.ts @@ -1,12 +1,12 @@ -import type { SyncContext, SyncDefinition } from '../../../types' +import type { SyncContext, SyncDefinition, SyncOutcome } from '../../../types' import { githubGraphql } from '../gql' import type { IssueNode, IssuesPage } from '../graphql/issues' import { ISSUES_QUERY } from '../graphql/issues' import { toIssueActivities } from '../mappers/issue' -import { MAX_PAGES_PER_RUN, PAGE_SIZE, parseRepoChannel, readWatermark } from '../paging' +import { PAGE_SIZE, parseRepoChannel, readWatermark } from '../paging' import { githubActivitySchema } from '../schemas' -async function runIssuesSync(ctx: SyncContext): Promise { +async function runIssuesSync(ctx: SyncContext): Promise { const { owner, repo } = parseRepoChannel(ctx.channel.channelName) const watermark = readWatermark(ctx.watermark) @@ -16,7 +16,7 @@ async function runIssuesSync(ctx: SyncContext): Promise { let since = watermark.since let cursor: string | null = null - for (let page = 0; page < MAX_PAGES_PER_RUN; page++) { + while (ctx.hasRunBudget()) { const data = await githubGraphql(ctx.http, ISSUES_QUERY, { owner, repo, @@ -37,9 +37,11 @@ async function runIssuesSync(ctx: SyncContext): Promise { await ctx.commitWatermark({ phase: 'incremental', since, cursor: null }) if (!pageInfo.hasNextPage) { - return + return { complete: true } } } + + return { complete: false } } export const issuesSync: SyncDefinition = { diff --git a/services/libs/connectors/src/connectors/github/syncs/pullRequestComments.ts b/services/libs/connectors/src/connectors/github/syncs/pullRequestComments.ts index be0758f55c..a888979b35 100644 --- a/services/libs/connectors/src/connectors/github/syncs/pullRequestComments.ts +++ b/services/libs/connectors/src/connectors/github/syncs/pullRequestComments.ts @@ -1,5 +1,5 @@ import { mapWithConcurrency } from '../../../concurrency' -import type { SyncContext, SyncDefinition } from '../../../types' +import type { SyncContext, SyncDefinition, SyncOutcome } from '../../../types' import { githubGraphql } from '../gql' import type { PrCommentNode, PrCommentsBatchPage } from '../graphql/pullRequestChildren' import { COMMENTS_FOR_PRS_QUERY } from '../graphql/pullRequestChildren' @@ -33,8 +33,8 @@ async function fetchComments(ctx: SyncContext, prId: string): Promise { - await runDualPhasePrSync(ctx, async (prs, sinceDate) => { +async function runPullRequestCommentsSync(ctx: SyncContext): Promise { + return runDualPhasePrSync(ctx, async (prs, sinceDate) => { const commentsPerPr = await mapWithConcurrency(prs, ITEM_FETCH_CONCURRENCY, (pr) => fetchComments(ctx, pr.id), ) diff --git a/services/libs/connectors/src/connectors/github/syncs/pullRequestCommits.ts b/services/libs/connectors/src/connectors/github/syncs/pullRequestCommits.ts index b5aae620ae..7fc0b1417b 100644 --- a/services/libs/connectors/src/connectors/github/syncs/pullRequestCommits.ts +++ b/services/libs/connectors/src/connectors/github/syncs/pullRequestCommits.ts @@ -1,4 +1,4 @@ -import type { SyncContext, SyncDefinition } from '../../../types' +import type { SyncContext, SyncDefinition, SyncOutcome } from '../../../types' import { githubGraphql } from '../gql' import type { PrCommitNode, PrCommitsPage } from '../graphql/pullRequestChildren' import { PR_COMMITS_QUERY } from '../graphql/pullRequestChildren' @@ -9,10 +9,10 @@ import { githubActivitySchema } from '../schemas' const COMMITS_PAGE_SIZE = 50 -async function runPullRequestCommitsSync(ctx: SyncContext): Promise { +async function runPullRequestCommitsSync(ctx: SyncContext): Promise { const { owner, repo } = parseRepoChannel(ctx.channel.channelName) - await runDualPhasePrSync(ctx, async (prs) => { + return runDualPhasePrSync(ctx, async (prs) => { for (const pullRequest of prs) { let cursor: string | null = null let hasMore = true diff --git a/services/libs/connectors/src/connectors/github/syncs/pullRequestReviewComments.ts b/services/libs/connectors/src/connectors/github/syncs/pullRequestReviewComments.ts index 0c75450ae8..937293f397 100644 --- a/services/libs/connectors/src/connectors/github/syncs/pullRequestReviewComments.ts +++ b/services/libs/connectors/src/connectors/github/syncs/pullRequestReviewComments.ts @@ -1,5 +1,5 @@ import { mapWithConcurrency } from '../../../concurrency' -import type { SyncContext, SyncDefinition } from '../../../types' +import type { SyncContext, SyncDefinition, SyncOutcome } from '../../../types' import { githubGraphql } from '../gql' import type { ReviewThreadNode, @@ -68,8 +68,8 @@ async function fetchThreadComments( return comments } -async function runPullRequestReviewCommentsSync(ctx: SyncContext): Promise { - await runDualPhasePrSync(ctx, async (prs, sinceDate) => { +async function runPullRequestReviewCommentsSync(ctx: SyncContext): Promise { + return runDualPhasePrSync(ctx, async (prs, sinceDate) => { const threadsPerPr = await mapWithConcurrency(prs, ITEM_FETCH_CONCURRENCY, (pr) => fetchThreads(ctx, pr.id), ) diff --git a/services/libs/connectors/src/connectors/github/syncs/pullRequests.ts b/services/libs/connectors/src/connectors/github/syncs/pullRequests.ts index 9e73fd99da..b5cc4a4333 100644 --- a/services/libs/connectors/src/connectors/github/syncs/pullRequests.ts +++ b/services/libs/connectors/src/connectors/github/syncs/pullRequests.ts @@ -1,5 +1,5 @@ import { mapWithConcurrency } from '../../../concurrency' -import type { SyncContext, SyncDefinition } from '../../../types' +import type { SyncContext, SyncDefinition, SyncOutcome } from '../../../types' import { githubGraphql } from '../gql' import type { PrTimelineItem, PrTimelinePage, PullRequestNode } from '../graphql/pullRequests' import { PR_TIMELINE_QUERY } from '../graphql/pullRequests' @@ -47,8 +47,8 @@ async function emitPullRequests(ctx: SyncContext, pullRequests: PullRequestNode[ await ctx.emit(pullRequests.flatMap((pr, index) => toPullRequestActivities(pr, timelines[index]))) } -async function runPullRequestsSync(ctx: SyncContext): Promise { - await runDualPhasePrSync(ctx, (prs) => emitPullRequests(ctx, prs)) +async function runPullRequestsSync(ctx: SyncContext): Promise { + return runDualPhasePrSync(ctx, (prs) => emitPullRequests(ctx, prs)) } export const pullRequestsSync: SyncDefinition = { diff --git a/services/libs/connectors/src/connectors/github/syncs/stars.ts b/services/libs/connectors/src/connectors/github/syncs/stars.ts index e83b3d5423..4b5940ed5d 100644 --- a/services/libs/connectors/src/connectors/github/syncs/stars.ts +++ b/services/libs/connectors/src/connectors/github/syncs/stars.ts @@ -1,19 +1,19 @@ -import type { SyncContext, SyncDefinition } from '../../../types' +import type { SyncContext, SyncDefinition, SyncOutcome } from '../../../types' import { githubGraphql } from '../gql' import type { StargazerEdge, StargazersPage } from '../graphql/stars' import { STARGAZERS_QUERY } from '../graphql/stars' import { toStar } from '../mappers/star' -import { MAX_PAGES_PER_RUN, PAGE_SIZE, parseRepoChannel, readWatermark } from '../paging' +import { PAGE_SIZE, parseRepoChannel, readWatermark } from '../paging' import { githubActivitySchema } from '../schemas' -async function runStarsSync(ctx: SyncContext): Promise { +async function runStarsSync(ctx: SyncContext): Promise { const { owner, repo } = parseRepoChannel(ctx.channel.channelName) const watermark = readWatermark(ctx.watermark) let since = watermark.since let cursor = watermark.cursor - for (let page = 0; page < MAX_PAGES_PER_RUN; page++) { + while (ctx.hasRunBudget()) { const data = await githubGraphql(ctx.http, STARGAZERS_QUERY, { owner, repo, @@ -35,9 +35,11 @@ async function runStarsSync(ctx: SyncContext): Promise { await ctx.commitWatermark({ phase: 'incremental', since, cursor }) if (!pageInfo.hasNextPage) { - return + return { complete: true } } } + + return { complete: false } } export const starsSync: SyncDefinition = { diff --git a/services/libs/connectors/src/testing/dummyConnector.ts b/services/libs/connectors/src/testing/dummyConnector.ts index d67d26a676..93ef346ec9 100644 --- a/services/libs/connectors/src/testing/dummyConnector.ts +++ b/services/libs/connectors/src/testing/dummyConnector.ts @@ -14,6 +14,7 @@ export const dummyConnector: Manifest = { run: async (ctx: SyncContext) => { await ctx.emit(Array.from({ length: TICK_COUNT }, (_, index) => ({ tick: index }))) await ctx.commitWatermark({ since: new Date().toISOString() }) + return { complete: true } }, }, ], diff --git a/services/libs/connectors/src/types.ts b/services/libs/connectors/src/types.ts index ada7994574..0354b786aa 100644 --- a/services/libs/connectors/src/types.ts +++ b/services/libs/connectors/src/types.ts @@ -28,15 +28,20 @@ export interface SyncContext { watermark: Record | null emit: (records: unknown[]) => Promise commitWatermark: (watermark: Record) => Promise + hasRunBudget: () => boolean http: ConnectorHttp log: Logger } +export interface SyncOutcome { + complete: boolean +} + export interface SyncDefinition { name: string cadenceMinutes: number schema: ZodType> - run: (ctx: SyncContext) => Promise + run: (ctx: SyncContext) => Promise } export interface Manifest { diff --git a/services/libs/data-access-layer/src/connectors/syncUnits.ts b/services/libs/data-access-layer/src/connectors/syncUnits.ts index 3fc455b58c..2033e0eb55 100644 --- a/services/libs/data-access-layer/src/connectors/syncUnits.ts +++ b/services/libs/data-access-layer/src/connectors/syncUnits.ts @@ -73,22 +73,44 @@ export async function rescheduleUnit( ) } +export async function guardUnitLease( + qx: QueryExecutor, + id: string, + guardMs: number, +): Promise { + await qx.result( + `UPDATE integration.sync_units + SET "nextRunAt" = GREATEST("nextRunAt", now() + $(guardMs) * interval '1 millisecond'), + "updatedAt" = now() + WHERE id = $(id)`, + { id, guardMs }, + ) +} + export async function recordRunSuccess( qx: QueryExecutor, id: string, data: ISyncRunSuccess, + nextRunAt: Date, ): Promise { await qx.result( `UPDATE integration.sync_units SET watermark = $(watermark)::jsonb, "emittedCount" = $(emittedCount), + "nextRunAt" = $(nextRunAt), "lastRunAt" = now(), "lastSuccessAt" = now(), "consecutiveFailures" = 0, "lastErrorClass" = NULL, + "lockedAt" = NULL, "updatedAt" = now() WHERE id = $(id)`, - { id, watermark: JSON.stringify(data.watermark), emittedCount: data.emittedCount }, + { + id, + watermark: JSON.stringify(data.watermark), + emittedCount: data.emittedCount, + nextRunAt, + }, ) } @@ -148,6 +170,7 @@ export async function recordRunFailure( SET "consecutiveFailures" = "consecutiveFailures" + 1, "lastErrorClass" = $(errorClass), "lastRunAt" = now(), + "lockedAt" = NULL, status = CASE WHEN $(deadLetterAfter)::int IS NOT NULL AND "consecutiveFailures" + 1 >= $(deadLetterAfter) THEN 'dead_letter' ELSE status END, From accd74653bfafe16e4e553155b5145415ef320ab Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Wed, 2 Sep 2026 12:39:03 +0100 Subject: [PATCH 49/69] fix: back off failed sync runs instead of waiting out the lease guard Signed-off-by: Mouad BANI --- .../src/activities/syncRunActivities.ts | 12 +++++++++--- .../apps/connectors_worker/src/scheduling.ts | 18 +++++++++++++++--- services/libs/connectors/src/registry.ts | 4 ++++ .../src/connectors/syncUnits.ts | 4 +++- 4 files changed, 31 insertions(+), 7 deletions(-) diff --git a/services/apps/connectors_worker/src/activities/syncRunActivities.ts b/services/apps/connectors_worker/src/activities/syncRunActivities.ts index 250d4287a2..c983604c4d 100644 --- a/services/apps/connectors_worker/src/activities/syncRunActivities.ts +++ b/services/apps/connectors_worker/src/activities/syncRunActivities.ts @@ -5,6 +5,7 @@ import { createEmit, createHttpClient, createTokenPool, + findSync, getCredential, getManifest, getSync, @@ -24,7 +25,7 @@ import { getChildLogger } from '@crowd/logging' import { svc } from '../main' import { RUN_BUDGET_MS } from '../runLimits' -import { cadenceRunAt, shortDeferRunAt } from '../scheduling' +import { cadenceRunAt, failureRunAt, shortDeferRunAt } from '../scheduling' const DEAD_LETTER_AFTER = 5 const HEARTBEAT_INTERVAL_MS = 10_000 @@ -151,8 +152,13 @@ export async function executeSync(unitId: string): Promise { } const errorClass = err instanceof ConnectorError ? err.errorClass : 'unknown' const deadLetterAfter = errorClass === 'provider.auth' ? DEAD_LETTER_AFTER : null - log.error(err, 'sync run failed') - await recordRunFailure(qx, unitId, errorClass, deadLetterAfter) + const consecutiveFailures = unit.consecutiveFailures + 1 + const nextRunAt = failureRunAt( + consecutiveFailures, + findSync(unit.platform, unit.syncName)?.cadenceMinutes ?? null, + ) + log.error(err, { errorClass, consecutiveFailures, nextRunAt }, 'sync run failed') + await recordRunFailure(qx, unitId, errorClass, deadLetterAfter, nextRunAt) throw err } finally { clearInterval(heartbeat) diff --git a/services/apps/connectors_worker/src/scheduling.ts b/services/apps/connectors_worker/src/scheduling.ts index b0af66943d..6d742d16ae 100644 --- a/services/apps/connectors_worker/src/scheduling.ts +++ b/services/apps/connectors_worker/src/scheduling.ts @@ -1,12 +1,24 @@ -const CADENCE_JITTER_RATIO = 0.1 +const JITTER_RATIO = 0.1 const SHORT_DEFER_MIN_MS = 30_000 const SHORT_DEFER_JITTER_MS = 60_000 +const FAILURE_BACKOFF_BASE_MS = 60_000 +const FALLBACK_FAILURE_CAP_MS = 60 * 60_000 + +function withJitter(delayMs: number): number { + return delayMs + delayMs * JITTER_RATIO * (Math.random() * 2 - 1) +} export function cadenceRunAt(cadenceMinutes: number): Date { - const jitterMinutes = cadenceMinutes * CADENCE_JITTER_RATIO * (Math.random() * 2 - 1) - return new Date(Date.now() + (cadenceMinutes + jitterMinutes) * 60_000) + return new Date(Date.now() + withJitter(cadenceMinutes * 60_000)) } export function shortDeferRunAt(): Date { return new Date(Date.now() + SHORT_DEFER_MIN_MS + Math.random() * SHORT_DEFER_JITTER_MS) } + +export function failureRunAt(consecutiveFailures: number, cadenceMinutes: number | null): Date { + const capMs = cadenceMinutes === null ? FALLBACK_FAILURE_CAP_MS : cadenceMinutes * 60_000 + const backoffMs = FAILURE_BACKOFF_BASE_MS * 2 ** (Math.max(1, consecutiveFailures) - 1) + + return new Date(Date.now() + withJitter(Math.min(backoffMs, capMs))) +} diff --git a/services/libs/connectors/src/registry.ts b/services/libs/connectors/src/registry.ts index ecd06e3314..d88ddb68db 100644 --- a/services/libs/connectors/src/registry.ts +++ b/services/libs/connectors/src/registry.ts @@ -18,6 +18,10 @@ export function getManifest(platform: string): Manifest { return manifest } +export function findSync(platform: string, syncName: string): SyncDefinition | undefined { + return findManifest(platform)?.syncs.find((s) => s.name === syncName) +} + export function getSync(platform: string, syncName: string): SyncDefinition { const sync = getManifest(platform).syncs.find((s) => s.name === syncName) if (!sync) { diff --git a/services/libs/data-access-layer/src/connectors/syncUnits.ts b/services/libs/data-access-layer/src/connectors/syncUnits.ts index 2033e0eb55..4ace0ffcd8 100644 --- a/services/libs/data-access-layer/src/connectors/syncUnits.ts +++ b/services/libs/data-access-layer/src/connectors/syncUnits.ts @@ -164,19 +164,21 @@ export async function recordRunFailure( id: string, errorClass: string, deadLetterAfter: number | null, + nextRunAt: Date, ): Promise { await qx.result( `UPDATE integration.sync_units SET "consecutiveFailures" = "consecutiveFailures" + 1, "lastErrorClass" = $(errorClass), "lastRunAt" = now(), + "nextRunAt" = $(nextRunAt), "lockedAt" = NULL, status = CASE WHEN $(deadLetterAfter)::int IS NOT NULL AND "consecutiveFailures" + 1 >= $(deadLetterAfter) THEN 'dead_letter' ELSE status END, "updatedAt" = now() WHERE id = $(id)`, - { id, errorClass, deadLetterAfter }, + { id, errorClass, deadLetterAfter, nextRunAt }, ) } From ad2d086019371840d489fe52f226ed89cc914d0e Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Wed, 2 Sep 2026 13:01:16 +0100 Subject: [PATCH 50/69] fix: preserve installation park state across token re-mints Signed-off-by: Mouad BANI --- services/libs/connectors/src/pool/tokenPool.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/libs/connectors/src/pool/tokenPool.ts b/services/libs/connectors/src/pool/tokenPool.ts index f1852e8d87..b120a148e6 100644 --- a/services/libs/connectors/src/pool/tokenPool.ts +++ b/services/libs/connectors/src/pool/tokenPool.ts @@ -196,7 +196,7 @@ export function createTokenPool( const states = await readStates() const healthy = [...states.entries()].filter(([, state]) => isHealthy(state, nowMs)) if (healthy.length === 0) { - return true + return states.size === 0 } let pooledRemaining = 0 for (const [id, state] of healthy) { @@ -224,7 +224,7 @@ export function createTokenPool( async seed(tokenId: string, value: string): Promise { const json = await redis.hGet(tokensKey, tokenId) const state = json ? (JSON.parse(json) as ITokenState) : null - const next = state && state.value === value ? { ...state, value } : { value } + const next = state ? { ...state, value } : { value } await redis.hSet(tokensKey, tokenId, JSON.stringify(next)) await redis.zAdd(lruKey, { score: 0, value: tokenId }, { NX: true }) }, From cb814c1a560fd865585272fbb79fb5d5784d7955 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Wed, 2 Sep 2026 16:09:49 +0100 Subject: [PATCH 51/69] fix: ask temporal whether a sync is running instead of guarding the lease Signed-off-by: Mouad BANI --- .../apps/connectors_worker/src/activities.ts | 3 +-- .../src/activities/dispatcherActivities.ts | 24 ++++++++++++------- .../apps/connectors_worker/src/runLimits.ts | 2 -- .../apps/connectors_worker/src/scheduling.ts | 7 ++++++ .../src/workflows/dispatcher.ts | 1 - .../src/connectors/syncUnits.ts | 14 ----------- 6 files changed, 23 insertions(+), 28 deletions(-) diff --git a/services/apps/connectors_worker/src/activities.ts b/services/apps/connectors_worker/src/activities.ts index df1cfdbd59..b78114ee2e 100644 --- a/services/apps/connectors_worker/src/activities.ts +++ b/services/apps/connectors_worker/src/activities.ts @@ -2,10 +2,9 @@ import { admitByBudget, claimDue, deferUnit, - guardLease, startRun, touchHeartbeat, } from './activities/dispatcherActivities' import { executeSync } from './activities/syncRunActivities' -export { admitByBudget, claimDue, deferUnit, executeSync, guardLease, startRun, touchHeartbeat } +export { admitByBudget, claimDue, deferUnit, executeSync, startRun, touchHeartbeat } diff --git a/services/apps/connectors_worker/src/activities/dispatcherActivities.ts b/services/apps/connectors_worker/src/activities/dispatcherActivities.ts index 50ba471fca..a6a8da63f5 100644 --- a/services/apps/connectors_worker/src/activities/dispatcherActivities.ts +++ b/services/apps/connectors_worker/src/activities/dispatcherActivities.ts @@ -1,17 +1,12 @@ import { createTokenPool, findManifest, mapWithConcurrency } from '@crowd/connectors' -import { - claimDueUnits, - guardUnitLease, - rescheduleUnit, -} from '@crowd/data-access-layer/src/connectors' +import { claimDueUnits, rescheduleUnit } from '@crowd/data-access-layer/src/connectors' import type { IClaimedUnit } from '@crowd/data-access-layer/src/connectors' import { dbStoreQx } from '@crowd/data-access-layer/src/queryExecutor' import { RedisCache } from '@crowd/redis' import { WorkflowIdConflictPolicy, WorkflowIdReusePolicy } from '@crowd/temporal' import { svc } from '../main' -import { LEASE_GUARD_MS } from '../runLimits' -import { shortDeferRunAt } from '../scheduling' +import { runningProbeRunAt, shortDeferRunAt } from '../scheduling' import type { IAdmissionResult, StartRunResult } from '../types' const TASK_QUEUE = 'connectors' @@ -76,14 +71,25 @@ export async function startRun(unit: IClaimedUnit): Promise { return 'started' } catch (err) { if (err instanceof Error && err.name === 'WorkflowExecutionAlreadyStartedError') { + await backOffRunningUnit(unit) return 'alreadyRunning' } throw err } } -export async function guardLease(unitId: string): Promise { - await guardUnitLease(dbStoreQx(svc.postgres.writer), unitId, LEASE_GUARD_MS) +async function backOffRunningUnit(unit: IClaimedUnit): Promise { + const nextRunAt = await runningSince(unit) + .then(runningProbeRunAt) + .catch(() => shortDeferRunAt()) + + await rescheduleUnit(dbStoreQx(svc.postgres.writer), unit.id, nextRunAt) +} + +async function runningSince(unit: IClaimedUnit): Promise { + const description = await svc.temporal.workflow.getHandle(syncRunWorkflowId(unit)).describe() + + return description.startTime } export async function touchHeartbeat(): Promise { diff --git a/services/apps/connectors_worker/src/runLimits.ts b/services/apps/connectors_worker/src/runLimits.ts index 505c536937..65db1da714 100644 --- a/services/apps/connectors_worker/src/runLimits.ts +++ b/services/apps/connectors_worker/src/runLimits.ts @@ -3,5 +3,3 @@ const MINUTE_MS = 60_000 export const RUN_START_TO_CLOSE_TIMEOUT_MS = 60 * MINUTE_MS export const RUN_BUDGET_MS = RUN_START_TO_CLOSE_TIMEOUT_MS - 10 * MINUTE_MS - -export const LEASE_GUARD_MS = RUN_START_TO_CLOSE_TIMEOUT_MS + 5 * MINUTE_MS diff --git a/services/apps/connectors_worker/src/scheduling.ts b/services/apps/connectors_worker/src/scheduling.ts index 6d742d16ae..ee5c886e07 100644 --- a/services/apps/connectors_worker/src/scheduling.ts +++ b/services/apps/connectors_worker/src/scheduling.ts @@ -2,6 +2,7 @@ const JITTER_RATIO = 0.1 const SHORT_DEFER_MIN_MS = 30_000 const SHORT_DEFER_JITTER_MS = 60_000 const FAILURE_BACKOFF_BASE_MS = 60_000 +const RUNNING_PROBE_CAP_MS = 5 * 60_000 const FALLBACK_FAILURE_CAP_MS = 60 * 60_000 function withJitter(delayMs: number): number { @@ -16,6 +17,12 @@ export function shortDeferRunAt(): Date { return new Date(Date.now() + SHORT_DEFER_MIN_MS + Math.random() * SHORT_DEFER_JITTER_MS) } +export function runningProbeRunAt(startedAt: Date): Date { + const elapsedMs = Math.max(SHORT_DEFER_MIN_MS, Date.now() - startedAt.getTime()) + + return new Date(Date.now() + withJitter(Math.min(elapsedMs, RUNNING_PROBE_CAP_MS))) +} + export function failureRunAt(consecutiveFailures: number, cadenceMinutes: number | null): Date { const capMs = cadenceMinutes === null ? FALLBACK_FAILURE_CAP_MS : cadenceMinutes * 60_000 const backoffMs = FAILURE_BACKOFF_BASE_MS * 2 ** (Math.max(1, consecutiveFailures) - 1) diff --git a/services/apps/connectors_worker/src/workflows/dispatcher.ts b/services/apps/connectors_worker/src/workflows/dispatcher.ts index 3724d551a1..6b07013401 100644 --- a/services/apps/connectors_worker/src/workflows/dispatcher.ts +++ b/services/apps/connectors_worker/src/workflows/dispatcher.ts @@ -19,7 +19,6 @@ export async function dispatcher(): Promise { for (const unit of admitted) { try { await activity.startRun(unit) - await activity.guardLease(unit.id) } catch (err) { log.error('failed to dispatch sync unit', { unitId: unit.id, err }) } diff --git a/services/libs/data-access-layer/src/connectors/syncUnits.ts b/services/libs/data-access-layer/src/connectors/syncUnits.ts index 4ace0ffcd8..01e99193f5 100644 --- a/services/libs/data-access-layer/src/connectors/syncUnits.ts +++ b/services/libs/data-access-layer/src/connectors/syncUnits.ts @@ -73,20 +73,6 @@ export async function rescheduleUnit( ) } -export async function guardUnitLease( - qx: QueryExecutor, - id: string, - guardMs: number, -): Promise { - await qx.result( - `UPDATE integration.sync_units - SET "nextRunAt" = GREATEST("nextRunAt", now() + $(guardMs) * interval '1 millisecond'), - "updatedAt" = now() - WHERE id = $(id)`, - { id, guardMs }, - ) -} - export async function recordRunSuccess( qx: QueryExecutor, id: string, From 9f7713f83d90c14921805e3000f980736b617d9a Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Wed, 2 Sep 2026 16:28:01 +0100 Subject: [PATCH 52/69] feat: persist last error message and run completeness on sync units Signed-off-by: Mouad BANI --- ...62681__addSyncUnitObservabilityColumns.sql | 3 ++ .../src/activities/syncRunActivities.ts | 7 +++- .../src/connectors/syncUnits.ts | 41 +++++++++++++++++-- .../data-access-layer/src/connectors/types.ts | 8 +++- 4 files changed, 52 insertions(+), 7 deletions(-) create mode 100644 backend/src/database/migrations/V1788362681__addSyncUnitObservabilityColumns.sql diff --git a/backend/src/database/migrations/V1788362681__addSyncUnitObservabilityColumns.sql b/backend/src/database/migrations/V1788362681__addSyncUnitObservabilityColumns.sql new file mode 100644 index 0000000000..81466cd2db --- /dev/null +++ b/backend/src/database/migrations/V1788362681__addSyncUnitObservabilityColumns.sql @@ -0,0 +1,3 @@ +ALTER TABLE integration.sync_units + ADD COLUMN "lastErrorMessage" TEXT, + ADD COLUMN "lastRunComplete" BOOLEAN; diff --git a/services/apps/connectors_worker/src/activities/syncRunActivities.ts b/services/apps/connectors_worker/src/activities/syncRunActivities.ts index c983604c4d..7d99971b2b 100644 --- a/services/apps/connectors_worker/src/activities/syncRunActivities.ts +++ b/services/apps/connectors_worker/src/activities/syncRunActivities.ts @@ -121,6 +121,7 @@ export async function executeSync(unitId: string): Promise { { watermark: committedWatermark ?? {}, emittedCount: emitter.emittedCount(), + complete: outcome.complete, }, nextRunAt, ) @@ -143,9 +144,10 @@ export async function executeSync(unitId: string): Promise { { watermark: committedWatermark, emittedCount: emitter.emittedCount() }, resumeAt, err.errorClass, + err.message, ) } else { - await parkUnit(qx, unitId, resumeAt, err.errorClass) + await parkUnit(qx, unitId, resumeAt, err.errorClass, err.message) } log.info({ resumeAt, errorClass: err.errorClass }, 'sync run parked') return @@ -157,8 +159,9 @@ export async function executeSync(unitId: string): Promise { consecutiveFailures, findSync(unit.platform, unit.syncName)?.cadenceMinutes ?? null, ) + const errorMessage = err instanceof Error ? err.message : String(err) log.error(err, { errorClass, consecutiveFailures, nextRunAt }, 'sync run failed') - await recordRunFailure(qx, unitId, errorClass, deadLetterAfter, nextRunAt) + await recordRunFailure(qx, unitId, errorClass, errorMessage, deadLetterAfter, nextRunAt) throw err } finally { clearInterval(heartbeat) diff --git a/services/libs/data-access-layer/src/connectors/syncUnits.ts b/services/libs/data-access-layer/src/connectors/syncUnits.ts index 01e99193f5..22f9634335 100644 --- a/services/libs/data-access-layer/src/connectors/syncUnits.ts +++ b/services/libs/data-access-layer/src/connectors/syncUnits.ts @@ -1,10 +1,24 @@ import type { QueryExecutor } from '../queryExecutor' -import type { IClaimedUnit, ISyncRunSuccess, ISyncUnit, SyncUnitUpsert } from './types' +import type { + IClaimedUnit, + ISyncRunProgress, + ISyncRunSuccess, + ISyncUnit, + SyncUnitUpsert, +} from './types' const MIN_INITIAL_DELAY_SECONDS = 10 const MAX_INITIAL_DELAY_SECONDS = 900 const CLAIM_LEASE_MINUTES = 5 +const ERROR_MESSAGE_MAX_LENGTH = 500 + +function truncateErrorMessage(message: string | null): string | null { + if (!message) { + return null + } + return message.slice(0, ERROR_MESSAGE_MAX_LENGTH) +} export async function upsertSyncUnits(qx: QueryExecutor, units: SyncUnitUpsert[]): Promise { if (units.length === 0) { @@ -88,6 +102,8 @@ export async function recordRunSuccess( "lastSuccessAt" = now(), "consecutiveFailures" = 0, "lastErrorClass" = NULL, + "lastErrorMessage" = NULL, + "lastRunComplete" = $(complete), "lockedAt" = NULL, "updatedAt" = now() WHERE id = $(id)`, @@ -95,6 +111,7 @@ export async function recordRunSuccess( id, watermark: JSON.stringify(data.watermark), emittedCount: data.emittedCount, + complete: data.complete, nextRunAt, }, ) @@ -103,9 +120,10 @@ export async function recordRunSuccess( export async function recordRunPartial( qx: QueryExecutor, id: string, - progress: ISyncRunSuccess, + progress: ISyncRunProgress, resumeAt: Date, errorClass: string, + errorMessage: string | null, ): Promise { await qx.result( `UPDATE integration.sync_units @@ -114,6 +132,8 @@ export async function recordRunPartial( "nextRunAt" = $(resumeAt), "lastRunAt" = now(), "lastErrorClass" = $(errorClass), + "lastErrorMessage" = $(errorMessage), + "lastRunComplete" = false, "lockedAt" = NULL, "updatedAt" = now() WHERE id = $(id)`, @@ -123,6 +143,7 @@ export async function recordRunPartial( emittedCount: progress.emittedCount, resumeAt, errorClass, + errorMessage: truncateErrorMessage(errorMessage), }, ) } @@ -132,16 +153,19 @@ export async function parkUnit( id: string, resumeAt: Date, errorClass: string, + errorMessage: string | null, ): Promise { await qx.result( `UPDATE integration.sync_units SET "nextRunAt" = $(resumeAt), "lastRunAt" = now(), "lastErrorClass" = $(errorClass), + "lastErrorMessage" = $(errorMessage), + "lastRunComplete" = false, "lockedAt" = NULL, "updatedAt" = now() WHERE id = $(id)`, - { id, resumeAt, errorClass }, + { id, resumeAt, errorClass, errorMessage: truncateErrorMessage(errorMessage) }, ) } @@ -149,6 +173,7 @@ export async function recordRunFailure( qx: QueryExecutor, id: string, errorClass: string, + errorMessage: string | null, deadLetterAfter: number | null, nextRunAt: Date, ): Promise { @@ -156,6 +181,8 @@ export async function recordRunFailure( `UPDATE integration.sync_units SET "consecutiveFailures" = "consecutiveFailures" + 1, "lastErrorClass" = $(errorClass), + "lastErrorMessage" = $(errorMessage), + "lastRunComplete" = false, "lastRunAt" = now(), "nextRunAt" = $(nextRunAt), "lockedAt" = NULL, @@ -164,7 +191,13 @@ export async function recordRunFailure( THEN 'dead_letter' ELSE status END, "updatedAt" = now() WHERE id = $(id)`, - { id, errorClass, deadLetterAfter, nextRunAt }, + { + id, + errorClass, + errorMessage: truncateErrorMessage(errorMessage), + deadLetterAfter, + nextRunAt, + }, ) } diff --git a/services/libs/data-access-layer/src/connectors/types.ts b/services/libs/data-access-layer/src/connectors/types.ts index 2d2cb1af37..483e8b0bd3 100644 --- a/services/libs/data-access-layer/src/connectors/types.ts +++ b/services/libs/data-access-layer/src/connectors/types.ts @@ -14,6 +14,8 @@ export interface ISyncUnit { lastSuccessAt: string | null consecutiveFailures: number lastErrorClass: string | null + lastErrorMessage: string | null + lastRunComplete: boolean | null watermark: Record | null emittedCount: number | null } @@ -28,7 +30,11 @@ export type IClaimedUnit = Pick< 'id' | 'integrationId' | 'platform' | 'syncName' | 'channelId' | 'channelName' > -export interface ISyncRunSuccess { +export interface ISyncRunProgress { watermark: Record emittedCount: number } + +export interface ISyncRunSuccess extends ISyncRunProgress { + complete: boolean +} From ef0f76a89b101c62f62c0ab2ffd9108a48a4ee5b Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Wed, 2 Sep 2026 16:29:21 +0100 Subject: [PATCH 53/69] feat: count provider requests in the connector http client Signed-off-by: Mouad BANI --- services/libs/connectors/src/http/client.ts | 23 +++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/services/libs/connectors/src/http/client.ts b/services/libs/connectors/src/http/client.ts index 35aa18f690..b31fba7fbc 100644 --- a/services/libs/connectors/src/http/client.ts +++ b/services/libs/connectors/src/http/client.ts @@ -36,20 +36,34 @@ export interface HttpClientDeps { export interface ConnectorHttp { request(config: AxiosRequestConfig): Promise + requestCount(): number } +type CountingHttpClientDeps = HttpClientDeps & { countRequest: () => void } + const MAX_ATTEMPTS = 3 const BACKOFF_BASE_MS = 1000 const RATE_LIMIT_FALLBACK_MS = 60_000 const DEFAULT_TIMEOUT_MS = 60_000 export function createHttpClient(deps: HttpClientDeps): ConnectorHttp { + let requests = 0 + const countingDeps: CountingHttpClientDeps = { + ...deps, + countRequest: () => { + requests += 1 + }, + } return { - request: (config: AxiosRequestConfig) => requestWithRetry(deps, config), + request: (config: AxiosRequestConfig) => requestWithRetry(countingDeps, config), + requestCount: () => requests, } } -async function requestWithRetry(deps: HttpClientDeps, config: AxiosRequestConfig): Promise { +async function requestWithRetry( + deps: CountingHttpClientDeps, + config: AxiosRequestConfig, +): Promise { let lastError: ConnectorError = new ProviderUnavailableError() for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { try { @@ -70,7 +84,7 @@ async function requestWithRetry(deps: HttpClientDeps, config: AxiosRequestCon } async function attemptRequest( - deps: HttpClientDeps, + deps: CountingHttpClientDeps, config: AxiosRequestConfig, allowTokenRotation: boolean, ): Promise { @@ -111,7 +125,7 @@ async function attemptRequest( } async function send( - deps: HttpClientDeps, + deps: CountingHttpClientDeps, config: AxiosRequestConfig, token: IPooledToken, ): Promise> { @@ -121,6 +135,7 @@ async function send( ...applyToken(config, token), validateStatus: () => true, } + deps.countRequest() try { return await axios.request(authenticatedConfig) } catch (err) { From 2942d7b3d721b931fca7b19543cb5bf9a5761dd2 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Wed, 2 Sep 2026 16:31:13 +0100 Subject: [PATCH 54/69] feat: emit structured run-summary log lines from sync runs Signed-off-by: Mouad BANI --- .../src/activities/syncRunActivities.ts | 53 ++++++++++++++++--- 1 file changed, 46 insertions(+), 7 deletions(-) diff --git a/services/apps/connectors_worker/src/activities/syncRunActivities.ts b/services/apps/connectors_worker/src/activities/syncRunActivities.ts index 7d99971b2b..ab7838731e 100644 --- a/services/apps/connectors_worker/src/activities/syncRunActivities.ts +++ b/services/apps/connectors_worker/src/activities/syncRunActivities.ts @@ -10,7 +10,7 @@ import { getManifest, getSync, } from '@crowd/connectors' -import type { Emitter, SyncContext } from '@crowd/connectors' +import type { ConnectorHttp, Emitter, SyncContext } from '@crowd/connectors' import { getUnitById, parkUnit, @@ -44,10 +44,13 @@ export async function executeSync(unitId: string): Promise { const activityContext = Context.current() const log = getChildLogger('syncRun', svc.log, { + workflowId: activityContext.info.workflowExecution.workflowId, runId: activityContext.info.workflowExecution.runId, unitId: unit.id, + integrationId: unit.integrationId, platform: unit.platform, syncName: unit.syncName, + channelId: unit.channelId, channelName: unit.channelName, }) @@ -60,8 +63,24 @@ export async function executeSync(unitId: string): Promise { }, HEARTBEAT_INTERVAL_MS) let emitter: Emitter | null = null + let http: ConnectorHttp | null = null let committedWatermark = unit.watermark - const runDeadline = Date.now() + RUN_BUDGET_MS + const startedAt = Date.now() + const runDeadline = startedAt + RUN_BUDGET_MS + + const runSummary = (fields: Record) => ({ + event: 'sync_run_summary', + durationMs: Date.now() - startedAt, + emittedCount: emitter?.emittedCount() ?? 0, + requestCount: http?.requestCount() ?? 0, + complete: null as boolean | null, + ...fields, + }) + + log.info( + { event: 'sync_run_started', consecutiveFailures: unit.consecutiveFailures }, + 'sync run started', + ) try { const integration = await fetchIntegrationById(qx, unit.integrationId) @@ -80,7 +99,7 @@ export async function executeSync(unitId: string): Promise { const credential = await getCredential(qx, unit.integrationId) await manifest.seedTokens(credential, pool) } - const http = createHttpClient({ + http = createHttpClient({ acquireToken: pool.acquire, parkToken: pool.park, quarantineToken: pool.quarantine, @@ -126,8 +145,8 @@ export async function executeSync(unitId: string): Promise { nextRunAt, ) log.info( - { emittedCount: emitter.emittedCount(), complete: outcome.complete, nextRunAt }, - 'sync run succeeded', + runSummary({ outcome: 'success', complete: outcome.complete, nextRunAt }), + 'sync run summary', ) } catch (err) { if ( @@ -137,6 +156,7 @@ export async function executeSync(unitId: string): Promise { const fallbackMs = err.errorClass === 'provider.rate_limit' ? RATE_LIMIT_FALLBACK_MS : UNAVAILABLE_PARK_MS const resumeAt = err.options?.resumeAt ?? new Date(Date.now() + fallbackMs) + const progressCommitted = Boolean(emitter && committedWatermark) if (emitter && committedWatermark) { await recordRunPartial( qx, @@ -149,7 +169,16 @@ export async function executeSync(unitId: string): Promise { } else { await parkUnit(qx, unitId, resumeAt, err.errorClass, err.message) } - log.info({ resumeAt, errorClass: err.errorClass }, 'sync run parked') + log.info( + runSummary({ + outcome: 'parked', + errorClass: err.errorClass, + errorMessage: err.message, + nextRunAt: resumeAt, + progressCommitted, + }), + 'sync run summary', + ) return } const errorClass = err instanceof ConnectorError ? err.errorClass : 'unknown' @@ -160,7 +189,17 @@ export async function executeSync(unitId: string): Promise { findSync(unit.platform, unit.syncName)?.cadenceMinutes ?? null, ) const errorMessage = err instanceof Error ? err.message : String(err) - log.error(err, { errorClass, consecutiveFailures, nextRunAt }, 'sync run failed') + log.error( + runSummary({ + outcome: 'failed', + errorClass, + errorMessage, + consecutiveFailures, + nextRunAt, + err, + }), + 'sync run summary', + ) await recordRunFailure(qx, unitId, errorClass, errorMessage, deadLetterAfter, nextRunAt) throw err } finally { From 91b39f8b9e3d1ff50a166f992e775f6ab1fed0a9 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Wed, 2 Sep 2026 16:32:42 +0100 Subject: [PATCH 55/69] feat: log per-tick dispatch summaries from the connectors dispatcher Signed-off-by: Mouad BANI --- .../src/activities/dispatcherActivities.ts | 16 +++++++++++-- services/apps/connectors_worker/src/types.ts | 10 ++++++++ .../src/workflows/dispatcher.ts | 23 ++++++++++++++++++- 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/services/apps/connectors_worker/src/activities/dispatcherActivities.ts b/services/apps/connectors_worker/src/activities/dispatcherActivities.ts index a6a8da63f5..7efa17f270 100644 --- a/services/apps/connectors_worker/src/activities/dispatcherActivities.ts +++ b/services/apps/connectors_worker/src/activities/dispatcherActivities.ts @@ -7,7 +7,7 @@ import { WorkflowIdConflictPolicy, WorkflowIdReusePolicy } from '@crowd/temporal import { svc } from '../main' import { runningProbeRunAt, shortDeferRunAt } from '../scheduling' -import type { IAdmissionResult, StartRunResult } from '../types' +import type { IAdmissionResult, IDispatchCounts, StartRunResult } from '../types' const TASK_QUEUE = 'connectors' const HEARTBEAT_TTL_SECONDS = 300 @@ -27,10 +27,17 @@ export async function admitByBudget(units: IClaimedUnit[]): Promise headrooms[index]), deferred: units.filter((_, index) => !headrooms[index]), } + if (result.deferred.length > 0) { + svc.log.info( + { unitIds: result.deferred.map((unit) => unit.id) }, + 'units deferred: no token headroom', + ) + } + return result } export async function deferUnit(unitId: string): Promise { @@ -84,6 +91,7 @@ async function backOffRunningUnit(unit: IClaimedUnit): Promise { .catch(() => shortDeferRunAt()) await rescheduleUnit(dbStoreQx(svc.postgres.writer), unit.id, nextRunAt) + svc.log.info({ unitId: unit.id, nextRunAt }, 'unit already running, rescheduled probe') } async function runningSince(unit: IClaimedUnit): Promise { @@ -92,6 +100,10 @@ async function runningSince(unit: IClaimedUnit): Promise { return description.startTime } +export async function logDispatchSummary(counts: IDispatchCounts): Promise { + svc.log.info({ event: 'dispatcher_tick', ...counts }, 'dispatch summary') +} + export async function touchHeartbeat(): Promise { const cache = new RedisCache('connectors', svc.redis, svc.log) await cache.set('dispatcherHeartbeat', new Date().toISOString(), HEARTBEAT_TTL_SECONDS) diff --git a/services/apps/connectors_worker/src/types.ts b/services/apps/connectors_worker/src/types.ts index 31f1c992c0..f16800666c 100644 --- a/services/apps/connectors_worker/src/types.ts +++ b/services/apps/connectors_worker/src/types.ts @@ -6,3 +6,13 @@ export interface IAdmissionResult { admitted: IClaimedUnit[] deferred: IClaimedUnit[] } + +export interface IDispatchCounts { + claimed: number + admitted: number + deferred: number + started: number + alreadyRunning: number + failed: number + durationMs: number +} diff --git a/services/apps/connectors_worker/src/workflows/dispatcher.ts b/services/apps/connectors_worker/src/workflows/dispatcher.ts index 6b07013401..7cd25d05f2 100644 --- a/services/apps/connectors_worker/src/workflows/dispatcher.ts +++ b/services/apps/connectors_worker/src/workflows/dispatcher.ts @@ -10,16 +10,27 @@ const activity = proxyActivities({ const CLAIM_LIMIT = 100 export async function dispatcher(): Promise { + const startedAt = Date.now() + await activity.touchHeartbeat() const units = await activity.claimDue(CLAIM_LIMIT) const { admitted, deferred } = await activity.admitByBudget(units) + let started = 0 + let alreadyRunning = 0 + let failed = 0 for (const unit of admitted) { try { - await activity.startRun(unit) + const result = await activity.startRun(unit) + if (result === 'started') { + started += 1 + } else { + alreadyRunning += 1 + } } catch (err) { + failed += 1 log.error('failed to dispatch sync unit', { unitId: unit.id, err }) } } @@ -27,4 +38,14 @@ export async function dispatcher(): Promise { for (const unit of deferred) { await activity.deferUnit(unit.id) } + + await activity.logDispatchSummary({ + claimed: units.length, + admitted: admitted.length, + deferred: deferred.length, + started, + alreadyRunning, + failed, + durationMs: Date.now() - startedAt, + }) } From a39d4ec94bf1a5d0d1e0ecf6ce9cf2d71528196e Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Wed, 2 Sep 2026 16:45:48 +0100 Subject: [PATCH 56/69] fix: register logDispatchSummary activity on the connectors worker Signed-off-by: Mouad BANI --- services/apps/connectors_worker/src/activities.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/services/apps/connectors_worker/src/activities.ts b/services/apps/connectors_worker/src/activities.ts index b78114ee2e..9add180943 100644 --- a/services/apps/connectors_worker/src/activities.ts +++ b/services/apps/connectors_worker/src/activities.ts @@ -2,9 +2,18 @@ import { admitByBudget, claimDue, deferUnit, + logDispatchSummary, startRun, touchHeartbeat, } from './activities/dispatcherActivities' import { executeSync } from './activities/syncRunActivities' -export { admitByBudget, claimDue, deferUnit, executeSync, startRun, touchHeartbeat } +export { + admitByBudget, + claimDue, + deferUnit, + executeSync, + logDispatchSummary, + startRun, + touchHeartbeat, +} From d77b008849ac0f845a93019577d19212644cf8e5 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Thu, 3 Sep 2026 15:04:08 +0100 Subject: [PATCH 57/69] feat: make the connector token pool global and installation-backed Signed-off-by: Mouad BANI --- .../src/activities/dispatcherActivities.ts | 2 +- .../src/activities/syncRunActivities.ts | 15 +- .../src/connectors/github/appToken.ts | 51 ++++-- .../src/connectors/github/budget.ts | 2 +- .../connectors/src/connectors/github/index.ts | 5 +- .../libs/connectors/src/pool/tokenPool.ts | 157 ++++++++++++------ services/libs/connectors/src/types.ts | 9 +- 7 files changed, 160 insertions(+), 81 deletions(-) diff --git a/services/apps/connectors_worker/src/activities/dispatcherActivities.ts b/services/apps/connectors_worker/src/activities/dispatcherActivities.ts index 7efa17f270..2a563e2dd0 100644 --- a/services/apps/connectors_worker/src/activities/dispatcherActivities.ts +++ b/services/apps/connectors_worker/src/activities/dispatcherActivities.ts @@ -22,7 +22,7 @@ export async function claimDue(limit: number): Promise { export async function admitByBudget(units: IClaimedUnit[]): Promise { const headrooms = await mapWithConcurrency(units, BUDGET_PROBE_CONCURRENCY, (unit) => { const manifest = findManifest(unit.platform) - const pool = createTokenPool(svc.redis, unit.platform, unit.integrationId, { + const pool = createTokenPool(svc.redis, unit.platform, { probeBudget: manifest?.probeBudget, }) return pool.hasHeadroom(DEFAULT_RUN_ESTIMATE) diff --git a/services/apps/connectors_worker/src/activities/syncRunActivities.ts b/services/apps/connectors_worker/src/activities/syncRunActivities.ts index ab7838731e..72a832090f 100644 --- a/services/apps/connectors_worker/src/activities/syncRunActivities.ts +++ b/services/apps/connectors_worker/src/activities/syncRunActivities.ts @@ -92,15 +92,20 @@ export async function executeSync(unitId: string): Promise { } const manifest = getManifest(unit.platform) - const pool = createTokenPool(svc.redis, unit.platform, unit.integrationId, { + const credential = + manifest.preparePool || manifest.mintToken + ? await getCredential(qx, unit.integrationId) + : null + const pool = createTokenPool(svc.redis, unit.platform, { probeBudget: manifest.probeBudget, + mintToken: credential && manifest.mintToken ? manifest.mintToken(credential) : undefined, }) - if (manifest.seedTokens) { - const credential = await getCredential(qx, unit.integrationId) - await manifest.seedTokens(credential, pool) + let preferredEntryId: string | undefined + if (credential && manifest.preparePool) { + preferredEntryId = (await manifest.preparePool(credential, pool)).preferredEntryId } http = createHttpClient({ - acquireToken: pool.acquire, + acquireToken: () => pool.acquire(preferredEntryId), parkToken: pool.park, quarantineToken: pool.quarantine, interpretResponse: manifest.interpretResponse, diff --git a/services/libs/connectors/src/connectors/github/appToken.ts b/services/libs/connectors/src/connectors/github/appToken.ts index c16c826be5..b25343e58d 100644 --- a/services/libs/connectors/src/connectors/github/appToken.ts +++ b/services/libs/connectors/src/connectors/github/appToken.ts @@ -7,8 +7,8 @@ import { ProviderContractError, ProviderUnavailableError, } from '../../http/errors' -import type { TokenPool } from '../../pool/tokenPool' -import type { Credential } from '../../types' +import type { TokenMinter, TokenPool } from '../../pool/tokenPool' +import type { Credential, PoolPreparation } from '../../types' const GITHUB_API_VERSION = '2022-11-28' @@ -67,14 +67,7 @@ export async function mintInstallationToken( } } -// TODO(CM-1372): POC-only resolution; store the installation id per integration after the POC -export async function resolveInstallationId(credential: Credential): Promise { - const fromEnv = process.env.CROWD_GITHUB_INSTALLATION_ID - if (fromEnv) { - return fromEnv - } - - let installations: { id: number }[] +export async function listInstallationIds(credential: Credential): Promise { try { const response = await axios.get('https://api.github.com/app/installations', { headers: { @@ -82,20 +75,42 @@ export async function resolveInstallationId(credential: Credential): Promise String(installation.id)) } catch (err) { - throw classifyAppApiError(err, 'installation resolution') ?? err + throw classifyAppApiError(err, 'installation listing') ?? err } - if (installations.length === 0) { +} + +// TODO(CM-1372): POC-only resolution; store the installation id per integration after the POC +export async function resolveInstallationId(credential: Credential): Promise { + const fromEnv = process.env.CROWD_GITHUB_INSTALLATION_ID + if (fromEnv) { + return fromEnv + } + + const installationIds = await listInstallationIds(credential) + if (installationIds.length === 0) { + throw new Error('github app has no installations') + } + return installationIds[0] +} + +export async function prepareGithubPool( + credential: Credential, + pool: TokenPool, +): Promise { + const installationIds = await listInstallationIds(credential) + if (installationIds.length === 0) { throw new Error('github app has no installations') } - return String(installations[0].id) + await pool.seed(installationIds) + const preferredEntryId = process.env.CROWD_GITHUB_INSTALLATION_ID ?? installationIds[0] + return { preferredEntryId } } -export async function seedGithubTokens(credential: Credential, pool: TokenPool): Promise { - const installationId = await resolveInstallationId(credential) - const { token } = await mintInstallationToken(credential, installationId) - await pool.seed(`install-${installationId}`, token) +export function createGithubTokenMinter(credential: Credential): TokenMinter { + return (installationId) => mintInstallationToken(credential, installationId) } diff --git a/services/libs/connectors/src/connectors/github/budget.ts b/services/libs/connectors/src/connectors/github/budget.ts index 18b36abff7..3a64e79ba1 100644 --- a/services/libs/connectors/src/connectors/github/budget.ts +++ b/services/libs/connectors/src/connectors/github/budget.ts @@ -10,7 +10,7 @@ interface RateLimitResource { reset: number } -export const probeGithubBudget: BudgetProbe = async (_platform, _connectionId, token) => { +export const probeGithubBudget: BudgetProbe = async (_platform, token) => { try { const response = await axios.get('https://api.github.com/rate_limit', { headers: { diff --git a/services/libs/connectors/src/connectors/github/index.ts b/services/libs/connectors/src/connectors/github/index.ts index 16884e6566..ad84dee579 100644 --- a/services/libs/connectors/src/connectors/github/index.ts +++ b/services/libs/connectors/src/connectors/github/index.ts @@ -1,6 +1,6 @@ import type { Manifest } from '../../types' -import { seedGithubTokens } from './appToken' +import { createGithubTokenMinter, prepareGithubPool } from './appToken' import { probeGithubBudget } from './budget' import { discoverRepos } from './discover' import { interpretGithubResponse } from './interpret' @@ -28,7 +28,8 @@ export const githubConnector: Manifest = { starsSync, ], discover: discoverRepos, - seedTokens: seedGithubTokens, + preparePool: prepareGithubPool, + mintToken: createGithubTokenMinter, probeBudget: probeGithubBudget, interpretResponse: interpretGithubResponse, } diff --git a/services/libs/connectors/src/pool/tokenPool.ts b/services/libs/connectors/src/pool/tokenPool.ts index b120a148e6..8f0e4fce96 100644 --- a/services/libs/connectors/src/pool/tokenPool.ts +++ b/services/libs/connectors/src/pool/tokenPool.ts @@ -1,9 +1,10 @@ import type { RedisClient } from '@crowd/redis' import type { IPooledToken } from '../http/client' -import { ProviderAuthError, RateLimitError } from '../http/errors' +import { ConnectorError, ProviderAuthError, RateLimitError } from '../http/errors' const PROBE_STALENESS_MS = 90_000 +const TOKEN_EXPIRY_MARGIN_MS = 120_000 export interface BudgetSnapshot { limit: number @@ -11,29 +12,29 @@ export interface BudgetSnapshot { resetAt: Date } -export type BudgetProbe = ( - platform: string, - connectionId: string, - token: IPooledToken, -) => Promise +export type BudgetProbe = (platform: string, token: IPooledToken) => Promise + +export type TokenMinter = (entryId: string) => Promise<{ token: string; expiresAt: string }> // POC only: the probe is the single source of truth for budgets (github /rate_limit is free and -// limits are per installation token); budgets for other platforms are a later decision. +// limits are per installation, surviving token re-mints); other platforms are a later decision. export interface TokenPoolOptions { probeBudget?: BudgetProbe + mintToken?: TokenMinter } export interface TokenPool { - acquire(): Promise + acquire(preferredEntryId?: string): Promise hasHeadroom(estimate: number): Promise - park(tokenId: string, resumeAt: Date): Promise - quarantine(tokenId: string): Promise - seed(tokenId: string, value: string): Promise + park(entryId: string, resumeAt: Date): Promise + quarantine(entryId: string): Promise + seed(entryIds: string[]): Promise earliestResumeAt(): Promise } -interface ITokenState { - value: string +interface IEntryState { + token?: string + tokenExpiresAtMs?: number parkedUntil?: string quarantined?: boolean } @@ -48,24 +49,22 @@ interface IBucket { export function createTokenPool( redis: RedisClient, platform: string, - connectionId: string, options?: TokenPoolOptions, ): TokenPool { - const tokensKey = `connectors:pool:${platform}:${connectionId}:tokens` - const lruKey = `connectors:pool:${platform}:${connectionId}:lru` - const bucketKey = (tokenId: string) => - `connectors:pool:${platform}:${connectionId}:budget:${tokenId}` - - async function readStates(): Promise> { - const raw = await redis.hGetAll(tokensKey) - const states = new Map() + const entriesKey = `connectors:pool:${platform}:entries` + const lruKey = `connectors:pool:${platform}:lru` + const bucketKey = (entryId: string) => `connectors:pool:${platform}:budget:${entryId}` + + async function readStates(): Promise> { + const raw = await redis.hGetAll(entriesKey) + const states = new Map() for (const [id, json] of Object.entries(raw)) { - states.set(id, JSON.parse(json) as ITokenState) + states.set(id, JSON.parse(json) as IEntryState) } return states } - function isHealthy(state: ITokenState, nowMs: number): boolean { + function isHealthy(state: IEntryState, nowMs: number): boolean { if (state.quarantined) { return false } @@ -75,7 +74,17 @@ export function createTokenPool( return true } - function earliestParkedUntil(states: Map, nowMs: number): Date | null { + function usableToken(state: IEntryState, nowMs: number): string | null { + if (!state.token || !state.tokenExpiresAtMs) { + return null + } + if (state.tokenExpiresAtMs - TOKEN_EXPIRY_MARGIN_MS <= nowMs) { + return null + } + return state.token + } + + function earliestParkedUntil(states: Map, nowMs: number): Date | null { let earliest: Date | null = null for (const state of states.values()) { if (state.quarantined || !state.parkedUntil) { @@ -92,19 +101,47 @@ export function createTokenPool( return earliest } - // POC only: read-modify-write can lose a concurrent park/quarantine on the same token + // POC only: read-modify-write can lose a concurrent park/quarantine on the same entry // within a ~ms window; accepted tradeoff — fix with atomic writes when productizing. - async function updateState(tokenId: string, update: Partial): Promise { - const json = await redis.hGet(tokensKey, tokenId) + async function updateState(entryId: string, update: Partial): Promise { + const json = await redis.hGet(entriesKey, entryId) if (!json) { return } - const state = JSON.parse(json) as ITokenState - await redis.hSet(tokensKey, tokenId, JSON.stringify({ ...state, ...update })) + const state = JSON.parse(json) as IEntryState + await redis.hSet(entriesKey, entryId, JSON.stringify({ ...state, ...update })) + } + + async function ensureUsableToken( + entryId: string, + state: IEntryState, + nowMs: number, + ): Promise { + const cached = usableToken(state, nowMs) + if (cached) { + return cached + } + const mint = options?.mintToken + if (!mint) { + return null + } + try { + const { token, expiresAt } = await mint(entryId) + await updateState(entryId, { token, tokenExpiresAtMs: new Date(expiresAt).getTime() }) + return token + } catch (err) { + if (err instanceof ConnectorError) { + if (err.errorClass === 'provider.auth' || err.errorClass === 'provider.contract') { + await updateState(entryId, { quarantined: true }) + } + return null + } + throw err + } } - async function readBucket(tokenId: string): Promise { - const raw = await redis.hGetAll(bucketKey(tokenId)) + async function readBucket(entryId: string): Promise { + const raw = await redis.hGetAll(bucketKey(entryId)) if (!raw.probedAt) { return null } @@ -122,14 +159,18 @@ export function createTokenPool( async function loadBucket( probe: BudgetProbe, - token: IPooledToken, + entryId: string, + token: string | null, nowMs: number, ): Promise { - const bucket = await readBucket(token.id) + const bucket = await readBucket(entryId) if (!needsProbe(bucket, nowMs)) { return bucket } - const snapshot = await probe(platform, connectionId, token) + if (!token) { + return null + } + const snapshot = await probe(platform, { id: entryId, value: token }) if (!snapshot) { return null } @@ -139,7 +180,7 @@ export function createTokenPool( resetAtMs: snapshot.resetAt.getTime(), probedAtMs: nowMs, } - await redis.hSet(bucketKey(token.id), { + await redis.hSet(bucketKey(entryId), { limit: String(probed.limit), remaining: String(probed.remaining), resetAt: String(probed.resetAtMs), @@ -149,10 +190,14 @@ export function createTokenPool( } return { - async acquire(): Promise { + async acquire(preferredEntryId?: string): Promise { const nowMs = Date.now() const states = await readStates() - const ordered = await redis.zRange(lruKey, 0, -1) + const lruOrder = await redis.zRange(lruKey, 0, -1) + const ordered = + preferredEntryId && states.has(preferredEntryId) + ? [preferredEntryId, ...lruOrder.filter((id) => id !== preferredEntryId)] + : lruOrder const probe = options?.probeBudget let earliestBudgetResetAt: Date | null = null for (const id of ordered) { @@ -160,8 +205,12 @@ export function createTokenPool( if (!state || !isHealthy(state, nowMs)) { continue } + const token = await ensureUsableToken(id, state, nowMs) + if (!token) { + continue + } if (probe) { - const bucket = await loadBucket(probe, { id, value: state.value }, nowMs) + const bucket = await loadBucket(probe, id, token, nowMs) if (bucket && bucket.remaining <= 0) { const resetAt = new Date(bucket.resetAtMs) if (!earliestBudgetResetAt || resetAt < earliestBudgetResetAt) { @@ -174,7 +223,7 @@ export function createTokenPool( } } await redis.zAdd(lruKey, { score: nowMs, value: id }) - return { id, value: state.value } + return { id, value: token } } const parkedResumeAt = earliestParkedUntil(states, nowMs) const resumeAt = @@ -200,7 +249,7 @@ export function createTokenPool( } let pooledRemaining = 0 for (const [id, state] of healthy) { - const bucket = await loadBucket(probe, { id, value: state.value }, nowMs) + const bucket = await loadBucket(probe, id, usableToken(state, nowMs), nowMs) if (!bucket) { return true } @@ -212,21 +261,25 @@ export function createTokenPool( return false }, - async park(tokenId: string, resumeAt: Date): Promise { - await updateState(tokenId, { parkedUntil: resumeAt.toISOString() }) + async park(entryId: string, resumeAt: Date): Promise { + await updateState(entryId, { parkedUntil: resumeAt.toISOString() }) }, - // POC only: quarantined tokens are kept for inspection and never revived automatically - async quarantine(tokenId: string): Promise { - await updateState(tokenId, { quarantined: true }) + // POC only: quarantined entries are kept for inspection and never revived automatically + async quarantine(entryId: string): Promise { + await updateState(entryId, { quarantined: true }) }, - async seed(tokenId: string, value: string): Promise { - const json = await redis.hGet(tokensKey, tokenId) - const state = json ? (JSON.parse(json) as ITokenState) : null - const next = state ? { ...state, value } : { value } - await redis.hSet(tokensKey, tokenId, JSON.stringify(next)) - await redis.zAdd(lruKey, { score: 0, value: tokenId }, { NX: true }) + async seed(entryIds: string[]): Promise { + if (entryIds.length === 0) { + return + } + const multi = redis.multi() + for (const id of entryIds) { + multi.hSetNX(entriesKey, id, '{}') + multi.zAdd(lruKey, { score: 0, value: id }, { NX: true }) + } + await multi.exec() }, async earliestResumeAt(): Promise { diff --git a/services/libs/connectors/src/types.ts b/services/libs/connectors/src/types.ts index 0354b786aa..d5cd5f8c0a 100644 --- a/services/libs/connectors/src/types.ts +++ b/services/libs/connectors/src/types.ts @@ -3,7 +3,7 @@ import type { ZodType } from 'zod' import type { Logger } from '@crowd/logging' import type { ConnectorHttp, ResponseInterpreter } from './http/client' -import type { BudgetProbe, TokenPool } from './pool/tokenPool' +import type { BudgetProbe, TokenMinter, TokenPool } from './pool/tokenPool' export interface Channel { channelId: string @@ -44,11 +44,16 @@ export interface SyncDefinition { run: (ctx: SyncContext) => Promise } +export interface PoolPreparation { + preferredEntryId?: string +} + export interface Manifest { platform: string syncs: SyncDefinition[] discover: (credential: Credential) => Promise - seedTokens?: (credential: Credential, pool: TokenPool) => Promise + preparePool?: (credential: Credential, pool: TokenPool) => Promise + mintToken?: (credential: Credential) => TokenMinter probeBudget?: BudgetProbe interpretResponse?: ResponseInterpreter } From 37e7ea4a1563d98bbd98d31839391ca4a827658a Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Thu, 3 Sep 2026 15:07:53 +0100 Subject: [PATCH 58/69] feat: replace request-path quarantine with token invalidation Signed-off-by: Mouad BANI --- .../connectors_worker/src/activities/syncRunActivities.ts | 2 +- services/libs/connectors/src/http/client.ts | 6 +++--- services/libs/connectors/src/pool/tokenPool.ts | 5 +++++ 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/services/apps/connectors_worker/src/activities/syncRunActivities.ts b/services/apps/connectors_worker/src/activities/syncRunActivities.ts index 72a832090f..369bbdea16 100644 --- a/services/apps/connectors_worker/src/activities/syncRunActivities.ts +++ b/services/apps/connectors_worker/src/activities/syncRunActivities.ts @@ -107,7 +107,7 @@ export async function executeSync(unitId: string): Promise { http = createHttpClient({ acquireToken: () => pool.acquire(preferredEntryId), parkToken: pool.park, - quarantineToken: pool.quarantine, + invalidateToken: pool.invalidate, interpretResponse: manifest.interpretResponse, log, }) diff --git a/services/libs/connectors/src/http/client.ts b/services/libs/connectors/src/http/client.ts index b31fba7fbc..0520e1a280 100644 --- a/services/libs/connectors/src/http/client.ts +++ b/services/libs/connectors/src/http/client.ts @@ -28,7 +28,7 @@ export type ResponseInterpreter = (response: HttpResponse) => ConnectorError | n export interface HttpClientDeps { acquireToken: () => Promise parkToken: (tokenId: string, resumeAt: Date) => Promise - quarantineToken: (tokenId: string) => Promise + invalidateToken: (tokenId: string) => Promise log: Logger applyToken?: TokenApplier interpretResponse?: ResponseInterpreter @@ -111,10 +111,10 @@ async function attemptRequest( } if (error.errorClass === 'provider.auth') { - await deps.quarantineToken(token.id) + await deps.invalidateToken(token.id) deps.log.warn( { tokenId: token.id, status: response.status }, - 'token quarantined on auth failure', + 'token invalidated on auth failure', ) if (allowTokenRotation) { return attemptRequest(deps, config, false) diff --git a/services/libs/connectors/src/pool/tokenPool.ts b/services/libs/connectors/src/pool/tokenPool.ts index 8f0e4fce96..af3b396dfb 100644 --- a/services/libs/connectors/src/pool/tokenPool.ts +++ b/services/libs/connectors/src/pool/tokenPool.ts @@ -27,6 +27,7 @@ export interface TokenPool { acquire(preferredEntryId?: string): Promise hasHeadroom(estimate: number): Promise park(entryId: string, resumeAt: Date): Promise + invalidate(entryId: string): Promise quarantine(entryId: string): Promise seed(entryIds: string[]): Promise earliestResumeAt(): Promise @@ -265,6 +266,10 @@ export function createTokenPool( await updateState(entryId, { parkedUntil: resumeAt.toISOString() }) }, + async invalidate(entryId: string): Promise { + await updateState(entryId, { token: undefined, tokenExpiresAtMs: undefined }) + }, + // POC only: quarantined entries are kept for inspection and never revived automatically async quarantine(entryId: string): Promise { await updateState(entryId, { quarantined: true }) From 7951a1feef6b1b3fd68303dc882336cfef597b8d Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Thu, 3 Sep 2026 15:16:15 +0100 Subject: [PATCH 59/69] fix: make token pool state updates atomic with a lua script Signed-off-by: Mouad BANI --- .../libs/connectors/src/pool/tokenPool.ts | 37 ++++++++++++++----- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/services/libs/connectors/src/pool/tokenPool.ts b/services/libs/connectors/src/pool/tokenPool.ts index af3b396dfb..18aa1e9e91 100644 --- a/services/libs/connectors/src/pool/tokenPool.ts +++ b/services/libs/connectors/src/pool/tokenPool.ts @@ -40,6 +40,27 @@ interface IEntryState { quarantined?: boolean } +type EntryStateUpdate = { [K in keyof IEntryState]?: IEntryState[K] | null } + +// atomic get-merge-set per entry; JSON null in the update deletes the field +const MERGE_ENTRY_STATE_SCRIPT = ` +local json = redis.call('HGET', KEYS[1], ARGV[1]) +if not json then + return 0 +end +local state = cjson.decode(json) +local update = cjson.decode(ARGV[2]) +for field, value in pairs(update) do + if value == cjson.null then + state[field] = nil + else + state[field] = value + end +end +redis.call('HSET', KEYS[1], ARGV[1], cjson.encode(state)) +return 1 +` + interface IBucket { limit: number remaining: number @@ -102,15 +123,11 @@ export function createTokenPool( return earliest } - // POC only: read-modify-write can lose a concurrent park/quarantine on the same entry - // within a ~ms window; accepted tradeoff — fix with atomic writes when productizing. - async function updateState(entryId: string, update: Partial): Promise { - const json = await redis.hGet(entriesKey, entryId) - if (!json) { - return - } - const state = JSON.parse(json) as IEntryState - await redis.hSet(entriesKey, entryId, JSON.stringify({ ...state, ...update })) + async function updateState(entryId: string, update: EntryStateUpdate): Promise { + await redis.eval(MERGE_ENTRY_STATE_SCRIPT, { + keys: [entriesKey], + arguments: [entryId, JSON.stringify(update)], + }) } async function ensureUsableToken( @@ -267,7 +284,7 @@ export function createTokenPool( }, async invalidate(entryId: string): Promise { - await updateState(entryId, { token: undefined, tokenExpiresAtMs: undefined }) + await updateState(entryId, { token: null, tokenExpiresAtMs: null }) }, // POC only: quarantined entries are kept for inspection and never revived automatically From 41378f694d4d3f77405d11a8fda802bf122027fa Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Thu, 3 Sep 2026 15:20:57 +0100 Subject: [PATCH 60/69] perf: skip github pool re-seeding while the seed marker is fresh Signed-off-by: Mouad BANI --- .../libs/connectors/src/connectors/github/appToken.ts | 9 +++++++-- services/libs/connectors/src/pool/tokenPool.ts | 8 ++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/services/libs/connectors/src/connectors/github/appToken.ts b/services/libs/connectors/src/connectors/github/appToken.ts index b25343e58d..9984f2a96a 100644 --- a/services/libs/connectors/src/connectors/github/appToken.ts +++ b/services/libs/connectors/src/connectors/github/appToken.ts @@ -98,17 +98,22 @@ export async function resolveInstallationId(credential: Credential): Promise { + const preferredEntryId = process.env.CROWD_GITHUB_INSTALLATION_ID + if (!(await pool.needsSeed(POOL_RESEED_INTERVAL_MS))) { + return { preferredEntryId } + } const installationIds = await listInstallationIds(credential) if (installationIds.length === 0) { throw new Error('github app has no installations') } await pool.seed(installationIds) - const preferredEntryId = process.env.CROWD_GITHUB_INSTALLATION_ID ?? installationIds[0] - return { preferredEntryId } + return { preferredEntryId: preferredEntryId ?? installationIds[0] } } export function createGithubTokenMinter(credential: Credential): TokenMinter { diff --git a/services/libs/connectors/src/pool/tokenPool.ts b/services/libs/connectors/src/pool/tokenPool.ts index 18aa1e9e91..b03b50dc5d 100644 --- a/services/libs/connectors/src/pool/tokenPool.ts +++ b/services/libs/connectors/src/pool/tokenPool.ts @@ -30,6 +30,7 @@ export interface TokenPool { invalidate(entryId: string): Promise quarantine(entryId: string): Promise seed(entryIds: string[]): Promise + needsSeed(maxAgeMs: number): Promise earliestResumeAt(): Promise } @@ -75,6 +76,7 @@ export function createTokenPool( ): TokenPool { const entriesKey = `connectors:pool:${platform}:entries` const lruKey = `connectors:pool:${platform}:lru` + const seededAtKey = `connectors:pool:${platform}:seededAt` const bucketKey = (entryId: string) => `connectors:pool:${platform}:budget:${entryId}` async function readStates(): Promise> { @@ -301,9 +303,15 @@ export function createTokenPool( multi.hSetNX(entriesKey, id, '{}') multi.zAdd(lruKey, { score: 0, value: id }, { NX: true }) } + multi.set(seededAtKey, String(Date.now())) await multi.exec() }, + async needsSeed(maxAgeMs: number): Promise { + const seededAt = await redis.get(seededAtKey) + return !seededAt || Date.now() - Number(seededAt) > maxAgeMs + }, + async earliestResumeAt(): Promise { const states = await readStates() return earliestParkedUntil(states, Date.now()) From 62eb1aa5d0351c21eff4a407c3add8f46fe342c6 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Thu, 3 Sep 2026 15:29:07 +0100 Subject: [PATCH 61/69] feat: log token pool rotation events from sync runs Signed-off-by: Mouad BANI --- .../src/activities/syncRunActivities.ts | 1 + .../libs/connectors/src/pool/tokenPool.ts | 58 ++++++++++++++++++- 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/services/apps/connectors_worker/src/activities/syncRunActivities.ts b/services/apps/connectors_worker/src/activities/syncRunActivities.ts index 369bbdea16..ac5a1c9b1a 100644 --- a/services/apps/connectors_worker/src/activities/syncRunActivities.ts +++ b/services/apps/connectors_worker/src/activities/syncRunActivities.ts @@ -99,6 +99,7 @@ export async function executeSync(unitId: string): Promise { const pool = createTokenPool(svc.redis, unit.platform, { probeBudget: manifest.probeBudget, mintToken: credential && manifest.mintToken ? manifest.mintToken(credential) : undefined, + log, }) let preferredEntryId: string | undefined if (credential && manifest.preparePool) { diff --git a/services/libs/connectors/src/pool/tokenPool.ts b/services/libs/connectors/src/pool/tokenPool.ts index b03b50dc5d..53fe397eda 100644 --- a/services/libs/connectors/src/pool/tokenPool.ts +++ b/services/libs/connectors/src/pool/tokenPool.ts @@ -1,3 +1,4 @@ +import type { Logger } from '@crowd/logging' import type { RedisClient } from '@crowd/redis' import type { IPooledToken } from '../http/client' @@ -21,6 +22,7 @@ export type TokenMinter = (entryId: string) => Promise<{ token: string; expiresA export interface TokenPoolOptions { probeBudget?: BudgetProbe mintToken?: TokenMinter + log?: Logger } export interface TokenPool { @@ -77,6 +79,35 @@ export function createTokenPool( const entriesKey = `connectors:pool:${platform}:entries` const lruKey = `connectors:pool:${platform}:lru` const seededAtKey = `connectors:pool:${platform}:seededAt` + const log = options?.log + + const loggedSkips = new Set() + let lastAcquiredEntryId: string | null = null + + function logSkip(entryId: string, reason: string, fields: Record = {}): void { + const key = `${entryId}:${reason}` + if (!log || loggedSkips.has(key)) { + return + } + loggedSkips.add(key) + log.info({ event: 'pool_entry_skipped', entryId, reason, ...fields }, 'pool entry skipped') + } + + function logAcquired(entryId: string, preferredEntryId?: string): void { + if (!log || lastAcquiredEntryId === entryId) { + return + } + lastAcquiredEntryId = entryId + log.info( + { + event: 'pool_token_acquired', + entryId, + preferredEntryId, + borrowed: preferredEntryId ? entryId !== preferredEntryId : false, + }, + 'pool token acquired', + ) + } const bucketKey = (entryId: string) => `connectors:pool:${platform}:budget:${entryId}` async function readStates(): Promise> { @@ -148,12 +179,27 @@ export function createTokenPool( try { const { token, expiresAt } = await mint(entryId) await updateState(entryId, { token, tokenExpiresAtMs: new Date(expiresAt).getTime() }) + log?.info({ event: 'pool_token_minted', entryId, expiresAt }, 'pool token minted') return token } catch (err) { if (err instanceof ConnectorError) { if (err.errorClass === 'provider.auth' || err.errorClass === 'provider.contract') { await updateState(entryId, { quarantined: true }) + log?.warn( + { + event: 'pool_entry_quarantined', + entryId, + errorClass: err.errorClass, + errMsg: err.message, + }, + 'pool entry quarantined on mint failure', + ) + return null } + log?.warn( + { event: 'pool_mint_failed', entryId, errorClass: err.errorClass, errMsg: err.message }, + 'token mint failed, skipping entry', + ) return null } throw err @@ -222,7 +268,15 @@ export function createTokenPool( let earliestBudgetResetAt: Date | null = null for (const id of ordered) { const state = states.get(id) - if (!state || !isHealthy(state, nowMs)) { + if (!state) { + continue + } + if (state.quarantined) { + logSkip(id, 'quarantined') + continue + } + if (state.parkedUntil && new Date(state.parkedUntil).getTime() > nowMs) { + logSkip(id, 'parked', { parkedUntil: state.parkedUntil }) continue } const token = await ensureUsableToken(id, state, nowMs) @@ -236,6 +290,7 @@ export function createTokenPool( if (!earliestBudgetResetAt || resetAt < earliestBudgetResetAt) { earliestBudgetResetAt = resetAt } + logSkip(id, 'budget_exhausted', { resetAt }) continue } if (bucket) { @@ -243,6 +298,7 @@ export function createTokenPool( } } await redis.zAdd(lruKey, { score: nowMs, value: id }) + logAcquired(id, preferredEntryId) return { id, value: token } } const parkedResumeAt = earliestParkedUntil(states, nowMs) From f688c281d37ff2de7c18e95362617af42a647132 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Thu, 3 Sep 2026 17:12:08 +0100 Subject: [PATCH 62/69] feat: allow restricting the github pool to specific installation ids Signed-off-by: Mouad BANI --- .../src/connectors/github/appToken.ts | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/services/libs/connectors/src/connectors/github/appToken.ts b/services/libs/connectors/src/connectors/github/appToken.ts index 9984f2a96a..9df8c02ec1 100644 --- a/services/libs/connectors/src/connectors/github/appToken.ts +++ b/services/libs/connectors/src/connectors/github/appToken.ts @@ -100,6 +100,18 @@ export async function resolveInstallationId(credential: Credential): Promise | null { + const raw = process.env.CROWD_GITHUB_POOL_INSTALLATION_IDS + if (!raw) { + return null + } + const ids = raw + .split(',') + .map((id) => id.trim()) + .filter(Boolean) + return ids.length > 0 ? new Set(ids) : null +} + export async function prepareGithubPool( credential: Credential, pool: TokenPool, @@ -108,9 +120,16 @@ export async function prepareGithubPool( if (!(await pool.needsSeed(POOL_RESEED_INTERVAL_MS))) { return { preferredEntryId } } - const installationIds = await listInstallationIds(credential) + const allowlist = poolInstallationAllowlist() + const installationIds = (await listInstallationIds(credential)).filter( + (id) => !allowlist || allowlist.has(id), + ) if (installationIds.length === 0) { - throw new Error('github app has no installations') + throw new Error( + allowlist + ? 'none of CROWD_GITHUB_POOL_INSTALLATION_IDS match an installation of this github app' + : 'github app has no installations', + ) } await pool.seed(installationIds) return { preferredEntryId: preferredEntryId ?? installationIds[0] } From 9c4022fb10b08cadef2bb433685da829b5c97c98 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Thu, 3 Sep 2026 17:25:13 +0100 Subject: [PATCH 63/69] chore: disable the github stars sync Signed-off-by: Mouad BANI --- services/libs/connectors/src/connectors/github/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/libs/connectors/src/connectors/github/index.ts b/services/libs/connectors/src/connectors/github/index.ts index ad84dee579..062d853678 100644 --- a/services/libs/connectors/src/connectors/github/index.ts +++ b/services/libs/connectors/src/connectors/github/index.ts @@ -12,7 +12,7 @@ import { pullRequestCommentsSync } from './syncs/pullRequestComments' import { pullRequestCommitsSync } from './syncs/pullRequestCommits' import { pullRequestReviewCommentsSync } from './syncs/pullRequestReviewComments' import { pullRequestsSync } from './syncs/pullRequests' -import { starsSync } from './syncs/stars' +// import { starsSync } from './syncs/stars' export const githubConnector: Manifest = { platform: 'github', @@ -25,7 +25,7 @@ export const githubConnector: Manifest = { pullRequestCommentsSync, pullRequestReviewCommentsSync, pullRequestCommitsSync, - starsSync, + // starsSync, ], discover: discoverRepos, preparePool: prepareGithubPool, From 18c19668f5be4603655005a00209e2ef80911c5f Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Thu, 3 Sep 2026 17:52:10 +0100 Subject: [PATCH 64/69] fix: paginate github app installation listing Signed-off-by: Mouad BANI --- .../src/connectors/github/appToken.ts | 35 +++++++++++++------ 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/services/libs/connectors/src/connectors/github/appToken.ts b/services/libs/connectors/src/connectors/github/appToken.ts index 9df8c02ec1..9f8b8eeb3b 100644 --- a/services/libs/connectors/src/connectors/github/appToken.ts +++ b/services/libs/connectors/src/connectors/github/appToken.ts @@ -67,21 +67,36 @@ export async function mintInstallationToken( } } +const INSTALLATIONS_PER_PAGE = 100 +const MAX_INSTALLATION_PAGES = 100 + export async function listInstallationIds(credential: Credential): Promise { + const appJwt = mintAppJwt(credential) + const installationIds: string[] = [] try { - const response = await axios.get('https://api.github.com/app/installations', { - headers: { - Authorization: `Bearer ${mintAppJwt(credential)}`, - Accept: 'application/vnd.github+json', - 'X-GitHub-Api-Version': GITHUB_API_VERSION, - }, - params: { per_page: 100 }, - timeout: GITHUB_REQUEST_TIMEOUT_MS, - }) - return (response.data as { id: number }[]).map((installation) => String(installation.id)) + for (let page = 1; page <= MAX_INSTALLATION_PAGES; page++) { + const response = await axios.get('https://api.github.com/app/installations', { + headers: { + Authorization: `Bearer ${appJwt}`, + Accept: 'application/vnd.github+json', + 'X-GitHub-Api-Version': GITHUB_API_VERSION, + }, + params: { per_page: INSTALLATIONS_PER_PAGE, page }, + timeout: GITHUB_REQUEST_TIMEOUT_MS, + }) + const installations = response.data as { id: number }[] + installationIds.push(...installations.map((installation) => String(installation.id))) + if (installations.length < INSTALLATIONS_PER_PAGE) { + return installationIds + } + } } catch (err) { throw classifyAppApiError(err, 'installation listing') ?? err } + // truncating would silently shrink the pool, so refuse instead + throw new Error( + `github app has more than ${MAX_INSTALLATION_PAGES * INSTALLATIONS_PER_PAGE} installations`, + ) } // TODO(CM-1372): POC-only resolution; store the installation id per integration after the POC From 60884ade5ba062a140f0a3e640f9f04e5a6dd6ca Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 4 Sep 2026 09:29:14 +0100 Subject: [PATCH 65/69] fix: stop discarding usable github responses and over-parking tokens Signed-off-by: Mouad BANI --- .../connectors/src/connectors/github/gql.ts | 20 ++++++++++++++++--- services/libs/connectors/src/http/client.ts | 11 +++++++--- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/services/libs/connectors/src/connectors/github/gql.ts b/services/libs/connectors/src/connectors/github/gql.ts index baa4d70856..aefb880353 100644 --- a/services/libs/connectors/src/connectors/github/gql.ts +++ b/services/libs/connectors/src/connectors/github/gql.ts @@ -1,9 +1,22 @@ import type { ConnectorHttp } from '../../http/client' import { ProviderContractError } from '../../http/errors' +interface GraphqlError { + type?: string + message?: string + path?: (string | number)[] +} + interface GraphqlEnvelope { data?: T - errors?: { type?: string; message?: string }[] + errors?: GraphqlError[] +} + +// GitHub resolves what it can and reports unreachable nodes (actors in orgs with +// IP allow lists) as errors with a deep path, alongside usable data. Only errors +// at or above the root field mean the whole response is unusable. +function isFatal(error: GraphqlError): boolean { + return (error.path?.length ?? 0) <= 1 } export async function githubGraphql( @@ -16,8 +29,9 @@ export async function githubGraphql( url: 'https://api.github.com/graphql', data: { query, variables }, }) - if (body.errors?.length) { - const details = body.errors.map((e) => `${e.type ?? 'ERROR'}: ${e.message ?? ''}`).join('; ') + const fatal = body.errors?.filter(isFatal) ?? [] + if (fatal.length > 0) { + const details = fatal.map((e) => `${e.type ?? 'ERROR'}: ${e.message ?? ''}`).join('; ') throw new ProviderContractError(`github graphql errors: ${details}`) } if (!body.data) { diff --git a/services/libs/connectors/src/http/client.ts b/services/libs/connectors/src/http/client.ts index 0520e1a280..4f4843d180 100644 --- a/services/libs/connectors/src/http/client.ts +++ b/services/libs/connectors/src/http/client.ts @@ -191,9 +191,14 @@ function computeResumeAt(headers: Record): Date { if (Number.isFinite(retryAfterDateMs) && retryAfterDateMs > Date.now()) { return new Date(retryAfterDateMs) } - const resetEpochSeconds = Number(headers['x-ratelimit-reset']) - if (Number.isFinite(resetEpochSeconds) && resetEpochSeconds * 1000 > Date.now()) { - return new Date(resetEpochSeconds * 1000) + // x-ratelimit-reset describes the primary bucket only. Honouring it on a + // secondary/abuse block parks the token for the rest of the hour instead of + // the minute or so the block actually lasts. + if (headers['x-ratelimit-remaining'] === '0') { + const resetEpochSeconds = Number(headers['x-ratelimit-reset']) + if (Number.isFinite(resetEpochSeconds) && resetEpochSeconds * 1000 > Date.now()) { + return new Date(resetEpochSeconds * 1000) + } } return new Date(Date.now() + RATE_LIMIT_FALLBACK_MS) } From 83b4b009cdc8d1c11319102833fb6a52fcfebf4d Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 4 Sep 2026 09:42:01 +0100 Subject: [PATCH 66/69] fix: rotate across the whole token pool before parking a run Signed-off-by: Mouad BANI --- services/libs/connectors/src/http/client.ts | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/services/libs/connectors/src/http/client.ts b/services/libs/connectors/src/http/client.ts index 4f4843d180..fedf21c349 100644 --- a/services/libs/connectors/src/http/client.ts +++ b/services/libs/connectors/src/http/client.ts @@ -67,7 +67,7 @@ async function requestWithRetry( let lastError: ConnectorError = new ProviderUnavailableError() for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { try { - return await attemptRequest(deps, config, true) + return await attemptRequest(deps, config) } catch (err) { if (!(err instanceof ConnectorError) || err.errorClass !== 'provider.unavailable') { throw err @@ -86,7 +86,7 @@ async function requestWithRetry( async function attemptRequest( deps: CountingHttpClientDeps, config: AxiosRequestConfig, - allowTokenRotation: boolean, + authRetried = false, ): Promise { const token = await deps.acquireToken() const response = await send(deps, config, token) @@ -99,15 +99,11 @@ async function attemptRequest( if (error.errorClass === 'provider.rate_limit') { const resumeAt = error.options?.resumeAt ?? computeResumeAt(headers) + // parking makes acquire() skip this entry, so rotation ends when the pool + // itself reports exhaustion rather than after a fixed number of tries await deps.parkToken(token.id, resumeAt) - if (allowTokenRotation) { - deps.log.info( - { tokenId: token.id, resumeAt }, - 'token rate limited, retrying with fresh token', - ) - return attemptRequest(deps, config, false) - } - throw new RateLimitError(error.message, { ...error.options, resumeAt }) + deps.log.info({ tokenId: token.id, resumeAt }, 'token rate limited, retrying with fresh token') + return attemptRequest(deps, config, authRetried) } if (error.errorClass === 'provider.auth') { @@ -116,8 +112,8 @@ async function attemptRequest( { tokenId: token.id, status: response.status }, 'token invalidated on auth failure', ) - if (allowTokenRotation) { - return attemptRequest(deps, config, false) + if (!authRetried) { + return attemptRequest(deps, config, true) } } From e1a31497cf21a5420a2336732535b19b9dbdadd6 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 4 Sep 2026 10:25:24 +0100 Subject: [PATCH 67/69] chore: log github rate-limit response detail on token park Signed-off-by: Mouad BANI --- services/libs/connectors/src/http/client.ts | 37 ++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/services/libs/connectors/src/http/client.ts b/services/libs/connectors/src/http/client.ts index fedf21c349..d006a9ac0d 100644 --- a/services/libs/connectors/src/http/client.ts +++ b/services/libs/connectors/src/http/client.ts @@ -102,7 +102,25 @@ async function attemptRequest( // parking makes acquire() skip this entry, so rotation ends when the pool // itself reports exhaustion rather than after a fixed number of tries await deps.parkToken(token.id, resumeAt) - deps.log.info({ tokenId: token.id, resumeAt }, 'token rate limited, retrying with fresh token') + // TODO(CM-1372): temporary — drop once the staging secondary-limit parks are explained + deps.log.info( + { + event: 'pool_rate_limit_detail', + tokenId: token.id, + resumeAt, + resumeSource: error.options?.resumeAt ? 'error' : 'headers', + status: response.status, + reason: error.message, + resource: headers['x-ratelimit-resource'], + limit: headers['x-ratelimit-limit'], + remaining: headers['x-ratelimit-remaining'], + used: headers['x-ratelimit-used'], + reset: headers['x-ratelimit-reset'], + retryAfter: headers['retry-after'], + body: rateLimitBodySummary(response.data), + }, + 'token rate limited, retrying with fresh token', + ) return attemptRequest(deps, config, authRetried) } @@ -120,6 +138,23 @@ async function attemptRequest( throw error } +const BODY_SUMMARY_MAX_LENGTH = 200 + +function rateLimitBodySummary(data: unknown): string | undefined { + if (!data || typeof data !== 'object') { + return undefined + } + const body = data as { message?: string; errors?: { type?: string; message?: string }[] } + if (body.message) { + return body.message.slice(0, BODY_SUMMARY_MAX_LENGTH) + } + const first = body.errors?.[0] + if (!first) { + return undefined + } + return `${first.type ?? 'ERROR'}: ${first.message ?? ''}`.slice(0, BODY_SUMMARY_MAX_LENGTH) +} + async function send( deps: CountingHttpClientDeps, config: AxiosRequestConfig, From 3cb979aad5eaefd19a4b56de01afbaff09adf874 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 4 Sep 2026 11:01:42 +0100 Subject: [PATCH 68/69] revert: throw on any github graphql error again Signed-off-by: Mouad BANI --- .../connectors/src/connectors/github/gql.ts | 20 +++---------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/services/libs/connectors/src/connectors/github/gql.ts b/services/libs/connectors/src/connectors/github/gql.ts index aefb880353..baa4d70856 100644 --- a/services/libs/connectors/src/connectors/github/gql.ts +++ b/services/libs/connectors/src/connectors/github/gql.ts @@ -1,22 +1,9 @@ import type { ConnectorHttp } from '../../http/client' import { ProviderContractError } from '../../http/errors' -interface GraphqlError { - type?: string - message?: string - path?: (string | number)[] -} - interface GraphqlEnvelope { data?: T - errors?: GraphqlError[] -} - -// GitHub resolves what it can and reports unreachable nodes (actors in orgs with -// IP allow lists) as errors with a deep path, alongside usable data. Only errors -// at or above the root field mean the whole response is unusable. -function isFatal(error: GraphqlError): boolean { - return (error.path?.length ?? 0) <= 1 + errors?: { type?: string; message?: string }[] } export async function githubGraphql( @@ -29,9 +16,8 @@ export async function githubGraphql( url: 'https://api.github.com/graphql', data: { query, variables }, }) - const fatal = body.errors?.filter(isFatal) ?? [] - if (fatal.length > 0) { - const details = fatal.map((e) => `${e.type ?? 'ERROR'}: ${e.message ?? ''}`).join('; ') + if (body.errors?.length) { + const details = body.errors.map((e) => `${e.type ?? 'ERROR'}: ${e.message ?? ''}`).join('; ') throw new ProviderContractError(`github graphql errors: ${details}`) } if (!body.data) { From 8571bf41ac082e57989d5802b2bcd71b46fb0355 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 4 Sep 2026 11:18:14 +0100 Subject: [PATCH 69/69] feat: add a github app budget audit script Signed-off-by: Mouad BANI --- services/apps/connectors_worker/package.json | 3 +- .../src/bin/audit-github-app-budget.ts | 225 ++++++++++++++++++ .../src/connectors/github/appToken.ts | 40 +++- 3 files changed, 262 insertions(+), 6 deletions(-) create mode 100644 services/apps/connectors_worker/src/bin/audit-github-app-budget.ts diff --git a/services/apps/connectors_worker/package.json b/services/apps/connectors_worker/package.json index 50bb105c79..065fbf84e8 100644 --- a/services/apps/connectors_worker/package.json +++ b/services/apps/connectors_worker/package.json @@ -11,7 +11,8 @@ "format": "npx prettier --write \"src/**/*.ts\"", "format-check": "npx prettier --check .", "tsc-check": "tsc --noEmit", - "script:seed-github-sync-units": "tsx src/bin/seed-github-sync-units.ts" + "script:seed-github-sync-units": "tsx src/bin/seed-github-sync-units.ts", + "script:audit-github-app-budget": "tsx src/bin/audit-github-app-budget.ts" }, "dependencies": { "@crowd/archetype-standard": "workspace:*", diff --git a/services/apps/connectors_worker/src/bin/audit-github-app-budget.ts b/services/apps/connectors_worker/src/bin/audit-github-app-budget.ts new file mode 100644 index 0000000000..ddffdffc3d --- /dev/null +++ b/services/apps/connectors_worker/src/bin/audit-github-app-budget.ts @@ -0,0 +1,225 @@ +import type { Credential } from '@crowd/connectors' +import { getCredential, mapWithConcurrency } from '@crowd/connectors' +import type { InstallationSummary } from '@crowd/connectors/src/connectors/github/appToken' +import { + listInstallations, + mintInstallationToken, +} from '@crowd/connectors/src/connectors/github/appToken' +import { WRITE_DB_CONFIG, getDbConnection } from '@crowd/data-access-layer/src/database' +import { pgpQx } from '@crowd/data-access-layer/src/queryExecutor' +import { getServiceLogger } from '@crowd/logging' + +const log = getServiceLogger() + +const GITHUB_API_VERSION = '2022-11-28' +const REQUEST_TIMEOUT_MS = 30_000 +const DEFAULT_CONCURRENCY = 4 + +interface RateLimitResource { + limit: number + remaining: number + reset: number +} + +interface InstallationBudget { + installation: InstallationSummary + repoCount: number | null + graphql: RateLimitResource | null + core: RateLimitResource | null + error?: string +} + +function usage(): never { + log.error( + 'Usage: audit-github-app-budget --integration-id [--concurrency ] [--json] [--include-suspended]', + ) + process.exit(1) +} + +function takeFlag(argv: string[], flag: string): string | undefined { + const flagIndex = argv.indexOf(flag) + if (flagIndex === -1) { + return undefined + } + const value = argv[flagIndex + 1] + argv.splice(flagIndex, 2) + return value +} + +function takeSwitch(argv: string[], flag: string): boolean { + const flagIndex = argv.indexOf(flag) + if (flagIndex === -1) { + return false + } + argv.splice(flagIndex, 1) + return true +} + +function parseArgs(rawArgv: string[]): { + integrationId: string + concurrency: number + asJson: boolean + includeSuspended: boolean +} { + const argv = rawArgv.filter((arg) => arg !== '--') + const integrationId = takeFlag(argv, '--integration-id') + const concurrency = Number(takeFlag(argv, '--concurrency') ?? DEFAULT_CONCURRENCY) + const asJson = takeSwitch(argv, '--json') + const includeSuspended = takeSwitch(argv, '--include-suspended') + if (!integrationId || !Number.isInteger(concurrency) || concurrency < 1) { + usage() + } + return { integrationId, concurrency, asJson, includeSuspended } +} + +async function githubGet(token: string, url: string): Promise { + const response = await fetch(url, { + headers: { + Authorization: `Bearer ${token}`, + Accept: 'application/vnd.github+json', + 'X-GitHub-Api-Version': GITHUB_API_VERSION, + }, + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }) + const body = await response.json().catch(() => undefined) + if (!response.ok) { + const message = (body as { message?: string } | undefined)?.message ?? response.statusText + throw new Error(`${response.status}: ${message}`) + } + return body +} + +async function readRepoCount(token: string): Promise { + const body = await githubGet(token, 'https://api.github.com/installation/repositories?per_page=1') + return (body as { total_count: number }).total_count +} + +async function auditInstallation( + credential: Credential, + installation: InstallationSummary, +): Promise { + try { + const { token } = await mintInstallationToken(credential, installation.id) + const [rateLimit, repoCount] = await Promise.all([ + githubGet(token, 'https://api.github.com/rate_limit'), + readRepoCount(token), + ]) + const resources = (rateLimit as { resources?: Record }).resources + return { + installation, + repoCount, + graphql: resources?.graphql ?? null, + core: resources?.core ?? null, + } + } catch (err) { + return { + installation, + repoCount: null, + graphql: null, + core: null, + error: err instanceof Error ? err.message : String(err), + } + } +} + +function pad(value: string | number, width: number, alignRight = false): string { + const text = String(value) + return alignRight ? text.padStart(width) : text.padEnd(width) +} + +function renderTable(budgets: InstallationBudget[]): void { + const accountWidth = Math.max( + 7, + ...budgets.map((budget) => (budget.installation.accountLogin ?? 'unknown').length), + ) + const columns: [string | number, number, boolean][][] = budgets.map((budget) => [ + [budget.installation.id, 12, false], + [budget.installation.accountLogin ?? 'unknown', accountWidth, false], + [budget.installation.accountType ?? '-', 12, false], + [budget.installation.repositorySelection ?? '-', 10, false], + [budget.repoCount ?? '-', 7, true], + [budget.graphql?.limit ?? '-', 10, true], + [budget.graphql?.remaining ?? '-', 10, true], + [budget.core?.limit ?? '-', 11, true], + [budget.error ?? (budget.installation.suspendedAt ? 'suspended' : ''), 0, false], + ]) + + const header = [ + pad('installation', 12), + pad('account', accountWidth), + pad('type', 12), + pad('selection', 10), + pad('repos', 7, true), + pad('gql limit', 10, true), + pad('remaining', 10, true), + pad('core limit', 11, true), + 'note', + ].join(' ') + process.stdout.write(`${header}\n${'-'.repeat(header.length)}\n`) + for (const row of columns) { + process.stdout.write(`${row.map(([v, w, r]) => pad(v, w, r)).join(' ')}\n`) + } +} + +function renderTotals(budgets: InstallationBudget[]): void { + const usable = budgets.filter((budget) => budget.graphql && !budget.installation.suspendedAt) + const sum = (pick: (budget: InstallationBudget) => number | null | undefined): number => + usable.reduce((total, budget) => total + (pick(budget) ?? 0), 0) + + const graphqlCeiling = sum((budget) => budget.graphql?.limit) + const repos = sum((budget) => budget.repoCount) + const failed = budgets.filter((budget) => budget.error).length + + process.stdout.write( + `${[ + '', + `installations discovered ${budgets.length}`, + `installations usable ${usable.length}`, + `installations failed ${failed}`, + `repositories reachable ${repos}`, + '', + `GraphQL points / hour ${graphqlCeiling.toLocaleString()}`, + `GraphQL points available ${sum((budget) => budget.graphql?.remaining).toLocaleString()}`, + `REST core points / hour ${sum((budget) => budget.core?.limit).toLocaleString()}`, + `GraphQL points / day ${(graphqlCeiling * 24).toLocaleString()}`, + '', + ].join('\n')}\n`, + ) +} + +setImmediate(async () => { + try { + const { integrationId, concurrency, asJson, includeSuspended } = parseArgs( + process.argv.slice(2), + ) + + const db = await getDbConnection(WRITE_DB_CONFIG()) + const credential = await getCredential(pgpQx(db), integrationId) + + const discovered = await listInstallations(credential) + const installations = includeSuspended + ? discovered + : discovered.filter((installation) => !installation.suspendedAt) + log.info( + { discovered: discovered.length, auditing: installations.length, concurrency }, + 'auditing github app installations', + ) + + const budgets = await mapWithConcurrency(installations, concurrency, (installation) => + auditInstallation(credential, installation), + ) + budgets.sort((a, b) => (b.graphql?.limit ?? -1) - (a.graphql?.limit ?? -1)) + + if (asJson) { + process.stdout.write(`${JSON.stringify(budgets, null, 2)}\n`) + } else { + renderTable(budgets) + } + renderTotals(budgets) + + process.exit(0) + } catch (err) { + log.error(err, 'github app budget audit failed') + process.exit(1) + } +}) diff --git a/services/libs/connectors/src/connectors/github/appToken.ts b/services/libs/connectors/src/connectors/github/appToken.ts index 9f8b8eeb3b..170deea13e 100644 --- a/services/libs/connectors/src/connectors/github/appToken.ts +++ b/services/libs/connectors/src/connectors/github/appToken.ts @@ -70,9 +70,34 @@ export async function mintInstallationToken( const INSTALLATIONS_PER_PAGE = 100 const MAX_INSTALLATION_PAGES = 100 -export async function listInstallationIds(credential: Credential): Promise { +export interface InstallationSummary { + id: string + accountLogin: string | null + accountType: string | null + repositorySelection: string | null + suspendedAt: string | null +} + +interface InstallationPayload { + id: number + account: { login?: string; type?: string } | null + repository_selection?: string + suspended_at?: string | null +} + +function toInstallationSummary(installation: InstallationPayload): InstallationSummary { + return { + id: String(installation.id), + accountLogin: installation.account?.login ?? null, + accountType: installation.account?.type ?? null, + repositorySelection: installation.repository_selection ?? null, + suspendedAt: installation.suspended_at ?? null, + } +} + +export async function listInstallations(credential: Credential): Promise { const appJwt = mintAppJwt(credential) - const installationIds: string[] = [] + const summaries: InstallationSummary[] = [] try { for (let page = 1; page <= MAX_INSTALLATION_PAGES; page++) { const response = await axios.get('https://api.github.com/app/installations', { @@ -84,10 +109,10 @@ export async function listInstallationIds(credential: Credential): Promise String(installation.id))) + const installations = response.data as InstallationPayload[] + summaries.push(...installations.map(toInstallationSummary)) if (installations.length < INSTALLATIONS_PER_PAGE) { - return installationIds + return summaries } } } catch (err) { @@ -99,6 +124,11 @@ export async function listInstallationIds(credential: Credential): Promise { + const installations = await listInstallations(credential) + return installations.map((installation) => installation.id) +} + // TODO(CM-1372): POC-only resolution; store the installation id per integration after the POC export async function resolveInstallationId(credential: Credential): Promise { const fromEnv = process.env.CROWD_GITHUB_INSTALLATION_ID