diff --git a/backend/src/database/migrations/V1787829533__createSyncUnitsTable.sql b/backend/src/database/migrations/V1787829533__createSyncUnitsTable.sql new file mode 100644 index 0000000000..893657e826 --- /dev/null +++ b/backend/src/database/migrations/V1787829533__createSyncUnitsTable.sql @@ -0,0 +1,25 @@ +CREATE TABLE IF NOT EXISTS integration.sync_units ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + "integrationId" UUID NOT NULL REFERENCES public.integrations(id), + platform TEXT NOT NULL, + "channelId" TEXT NOT NULL, + "channelName" TEXT NOT NULL, + "syncName" TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'active' + CHECK (status IN ('active','paused','dead_letter','decommissioned')), + "nextRunAt" TIMESTAMPTZ NOT NULL DEFAULT now(), + "lockedAt" TIMESTAMPTZ, + "lastRunAt" TIMESTAMPTZ, + "lastSuccessAt" TIMESTAMPTZ, + "consecutiveFailures" INT NOT NULL DEFAULT 0, + "lastErrorClass" TEXT, + watermark JSONB, + "emittedCount" INT, + "createdAt" TIMESTAMPTZ NOT NULL DEFAULT now(), + "updatedAt" TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE ("integrationId", "channelId", "syncName") +); + +CREATE INDEX IF NOT EXISTS "ix_sync_units_due" + ON integration.sync_units ("nextRunAt") + WHERE status = 'active'; diff --git a/backend/src/database/migrations/V1788362681__addSyncUnitObservabilityColumns.sql b/backend/src/database/migrations/V1788362681__addSyncUnitObservabilityColumns.sql new file mode 100644 index 0000000000..81466cd2db --- /dev/null +++ b/backend/src/database/migrations/V1788362681__addSyncUnitObservabilityColumns.sql @@ -0,0 +1,3 @@ +ALTER TABLE integration.sync_units + ADD COLUMN "lastErrorMessage" TEXT, + ADD COLUMN "lastRunComplete" BOOLEAN; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 74fb6469c2..199d6c5d85 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -602,6 +602,58 @@ importers: specifier: ^3.0.1 version: 3.1.0 + services/apps/connectors_worker: + dependencies: + '@crowd/archetype-standard': + specifier: workspace:* + version: link:../../archetypes/standard + '@crowd/archetype-worker': + specifier: workspace:* + version: link:../../archetypes/worker + '@crowd/common': + specifier: workspace:* + version: link:../../libs/common + '@crowd/connectors': + specifier: workspace:* + version: link:../../libs/connectors + '@crowd/data-access-layer': + specifier: workspace:* + version: link:../../libs/data-access-layer + '@crowd/logging': + specifier: workspace:* + version: link:../../libs/logging + '@crowd/redis': + specifier: workspace:* + version: link:../../libs/redis + '@crowd/temporal': + specifier: workspace:* + version: link:../../libs/temporal + '@crowd/types': + specifier: workspace:* + version: link:../../libs/types + '@temporalio/activity': + specifier: ~1.17.2 + version: 1.17.2 + '@temporalio/client': + specifier: ~1.17.2 + version: 1.17.2 + '@temporalio/workflow': + specifier: ~1.17.2 + version: 1.17.2 + tsx: + specifier: ^4.7.1 + version: 4.23.12 + typescript: + specifier: ^5.6.3 + version: 5.6.3 + devDependencies: + '@types/node': + specifier: ^24.13.3 + version: 24.13.3 + nodemon: + specifier: ^3.0.1 + version: 3.1.0 + services/apps/cron_service: dependencies: '@aws-sdk/client-s3': @@ -2215,6 +2267,49 @@ importers: specifier: ^5.6.3 version: 5.6.3 + services/libs/connectors: + dependencies: + '@crowd/common': + specifier: workspace:* + version: link:../common + '@crowd/common_services': + specifier: workspace:* + version: link:../common_services + '@crowd/data-access-layer': + specifier: workspace:* + version: link:../data-access-layer + '@crowd/integrations': + specifier: workspace:* + version: link:../integrations + '@crowd/logging': + specifier: workspace:* + version: link:../logging + '@crowd/redis': + specifier: workspace:* + version: link:../redis + '@crowd/types': + specifier: workspace:* + version: link:../types + axios: + specifier: ^1.6.8 + version: 1.19.0 + jsonwebtoken: + specifier: ^9.0.0 + version: 9.0.3 + zod: + specifier: ^3.22.0 + version: 3.25.76 + devDependencies: + '@types/jsonwebtoken': + specifier: ^9.0.0 + version: 9.0.6 + '@types/node': + specifier: ^24.13.3 + version: 24.13.3 + typescript: + specifier: ^5.6.3 + version: 5.6.3 + services/libs/data-access-layer: dependencies: '@crowd/common': @@ -3482,8 +3577,8 @@ packages: cpu: [ppc64] os: [aix] - '@esbuild/aix-ppc64@0.28.2': - resolution: {integrity: sha512-XExcO+dvLKvVtNTibSTBej1NCAbaGhWn9Ww1ZPx80qsahhPFe/8jgWP0IchNe0F3HwkU7n8ejhH8bjonqht8mQ==} + '@esbuild/aix-ppc64@0.28.0': + resolution: {integrity: sha512-lhRUCeuOyJQURhTxl4WkpFTjIsbDayJHih5kZC1giwE+MhIzAb7mEsQMqMf18rHLsrb5qI1tafG20mLxEWcWlA==} engines: {node: '>=18'} cpu: [ppc64] os: [aix] @@ -3494,8 +3589,8 @@ packages: cpu: [arm64] os: [android] - '@esbuild/android-arm64@0.28.2': - resolution: {integrity: sha512-5YfKeeI8qWfBZIX+u2xZC3Zlb3Os/gLS2sbEKM+I4ZOcsWmHS2WLysCcQZDAFRslDUU5Oiq44gf6PYN1vGwG5A==} + '@esbuild/android-arm64@0.28.0': + resolution: {integrity: sha512-+WzIXQOSaGs33tLEgYPYe/yQHf0WTU0X42Jca3y8NWMbUVhp7rUnw+vAsRC/QiDrdD31IszMrZy+qwPOPjd+rw==} engines: {node: '>=18'} cpu: [arm64] os: [android] @@ -3506,8 +3601,8 @@ packages: cpu: [arm] os: [android] - '@esbuild/android-arm@0.28.2': - resolution: {integrity: sha512-kXXoiPVVGQcnIYGOeaovwOURpniDBpSq4A03qkQ+BMQqtGG6HYap3xne9C1O1yo4TR3qxlCX5IqqmX6fFo2Lqg==} + '@esbuild/android-arm@0.28.0': + resolution: {integrity: sha512-wqh0ByljabXLKHeWXYLqoJ5jKC4XBaw6Hk08OfMrCRd2nP2ZQ5eleDZC41XHyCNgktBGYMbqnrJKq/K/lzPMSQ==} engines: {node: '>=18'} cpu: [arm] os: [android] @@ -3518,8 +3613,8 @@ packages: cpu: [x64] os: [android] - '@esbuild/android-x64@0.28.2': - resolution: {integrity: sha512-O387ite7SzUyCcy3JQX4P4bLtEA7bLLkx+esve5JHnyYfNTxcVpXZo9jhdB0lTKN44gztELTdU7nS8Nr16Fs1Q==} + '@esbuild/android-x64@0.28.0': + resolution: {integrity: sha512-+VJggoaKhk2VNNqVL7f6S189UzShHC/mR9EE8rDdSkdpN0KflSwWY/gWjDrNxxisg8Fp1ZCD9jLMo4m0OUfeUA==} engines: {node: '>=18'} cpu: [x64] os: [android] @@ -3530,8 +3625,8 @@ packages: cpu: [arm64] os: [darwin] - '@esbuild/darwin-arm64@0.28.2': - resolution: {integrity: sha512-n4KqkOQrraxHJcgjM1RvwbigfQKIKJVpM7xp+KsxiyUSrRdIXnt73VhrPAx0fV44hgfmIVKjxMN9J1t5jySVkw==} + '@esbuild/darwin-arm64@0.28.0': + resolution: {integrity: sha512-0T+A9WZm+bZ84nZBtk1ckYsOvyA3x7e2Acj1KdVfV4/2tdG4fzUp91YHx+GArWLtwqp77pBXVCPn2We7Letr0Q==} engines: {node: '>=18'} cpu: [arm64] os: [darwin] @@ -3542,8 +3637,8 @@ packages: cpu: [x64] os: [darwin] - '@esbuild/darwin-x64@0.28.2': - resolution: {integrity: sha512-uq6suIWYP37qzGddBKPw5QEQPi6HiLGsO7UmkpfyaYNQ3D+rN6w6WfwH+nuqcGXWvawGwxOEroO4YGnFh95azw==} + '@esbuild/darwin-x64@0.28.0': + resolution: {integrity: sha512-fyzLm/DLDl/84OCfp2f/XQ4flmORsjU7VKt8HLjvIXChJoFFOIL6pLJPH4Yhd1n1gGFF9mPwtlN5Wf82DZs+LQ==} engines: {node: '>=18'} cpu: [x64] os: [darwin] @@ -3554,8 +3649,8 @@ packages: cpu: [arm64] os: [freebsd] - '@esbuild/freebsd-arm64@0.28.2': - resolution: {integrity: sha512-n+I0BTSRIoy+d6RPKnEVwql5UwBJolytvY4mAOIEJorKlqgPII8ix6slVVrfZ5Tnj7glIZvloylbB/EJPMWEXw==} + '@esbuild/freebsd-arm64@0.28.0': + resolution: {integrity: sha512-l9GeW5UZBT9k9brBYI+0WDffcRxgHQD8ShN2Ur4xWq/NFzUKm3k5lsH4PdaRgb2w7mI9u61nr2gI2mLI27Nh3Q==} engines: {node: '>=18'} cpu: [arm64] os: [freebsd] @@ -3566,8 +3661,8 @@ packages: cpu: [x64] os: [freebsd] - '@esbuild/freebsd-x64@0.28.2': - resolution: {integrity: sha512-78XJTJkvPs0kz2w61301PJjXl4g7q3JqiYMZ/M/yVI73EHBrCRTgkhu9oqG7vPqq+a/yadEW8aD+agKlk5xrmg==} + '@esbuild/freebsd-x64@0.28.0': + resolution: {integrity: sha512-BXoQai/A0wPO6Es3yFJ7APCiKGc1tdAEOgeTNy3SsB491S3aHn4S4r3e976eUnPdU+NbdtmBuLncYir2tMU9Nw==} engines: {node: '>=18'} cpu: [x64] os: [freebsd] @@ -3578,8 +3673,8 @@ packages: cpu: [arm64] os: [linux] - '@esbuild/linux-arm64@0.28.2': - resolution: {integrity: sha512-pW4AC0P3it8c7do9MVM4p51FzHzdM/TZrerurgRcHJ2WTa1VQ1CIq18xncfpBJw4ojkiZZrKW2yIBWBP92j6Ug==} + '@esbuild/linux-arm64@0.28.0': + resolution: {integrity: sha512-RVyzfb3FWsGA55n6WY0MEIEPURL1FcbhFE6BffZEMEekfCzCIMtB5yyDcFnVbTnwk+CLAgTujmV/Lgvih56W+A==} engines: {node: '>=18'} cpu: [arm64] os: [linux] @@ -3590,8 +3685,8 @@ packages: cpu: [arm] os: [linux] - '@esbuild/linux-arm@0.28.2': - resolution: {integrity: sha512-XlDnu2q5yoqems+xay6wSAcg9DDD7K9RLKZEBOMZm3ckNpJBvOX20tSfby8KfrrhINDyv9V2YVZKY/SpoGJI8w==} + '@esbuild/linux-arm@0.28.0': + resolution: {integrity: sha512-CjaaREJagqJp7iTaNQjjidaNbCKYcd4IDkzbwwxtSvjI7NZm79qiHc8HqciMddQ6CKvJT6aBd8lO9kN/ZudLlw==} engines: {node: '>=18'} cpu: [arm] os: [linux] @@ -3602,8 +3697,8 @@ packages: cpu: [ia32] os: [linux] - '@esbuild/linux-ia32@0.28.2': - resolution: {integrity: sha512-CYbnj78HsIeA+DhgUKgFCfvNsTHFhMMrinUrMZpDXJXKN8T3XViTZ/+wtHeVxEWY8ewSzTFN+nRmSwO2tZaLUQ==} + '@esbuild/linux-ia32@0.28.0': + resolution: {integrity: sha512-KBnSTt1kxl9x70q+ydterVdl+Cn0H18ngRMRCEQfrbqdUuntQQ0LoMZv47uB97NljZFzY6HcfqEZ2SAyIUTQBQ==} engines: {node: '>=18'} cpu: [ia32] os: [linux] @@ -3614,8 +3709,8 @@ packages: cpu: [loong64] os: [linux] - '@esbuild/linux-loong64@0.28.2': - resolution: {integrity: sha512-buwkd8nsph4R+ajRvw0qM5Hja/TXQow3ptzWO2EbG/cqcIkHloRrdlBtQlshyYGTNFvfkfJ5tpPLVkY4DtsPfQ==} + '@esbuild/linux-loong64@0.28.0': + resolution: {integrity: sha512-zpSlUce1mnxzgBADvxKXX5sl8aYQHo2ezvMNI8I0lbblJtp8V4odlm3Yzlj7gPyt3T8ReksE6bK+pT3WD+aJRg==} engines: {node: '>=18'} cpu: [loong64] os: [linux] @@ -3626,8 +3721,8 @@ packages: cpu: [mips64el] os: [linux] - '@esbuild/linux-mips64el@0.28.2': - resolution: {integrity: sha512-ZVykbDyk7519VwiNb9Lcj9m8XM6v5V9uKPvrEMkkEedVewf+0itkhahp4HDpgERXhwLRpWFypsGbG/J8s0QjJA==} + '@esbuild/linux-mips64el@0.28.0': + resolution: {integrity: sha512-2jIfP6mmjkdmeTlsX/9vmdmhBmKADrWqN7zcdtHIeNSCH1SqIoNI63cYsjQR8J+wGa4Y5izRcSHSm8K3QWmk3w==} engines: {node: '>=18'} cpu: [mips64el] os: [linux] @@ -3638,8 +3733,8 @@ packages: cpu: [ppc64] os: [linux] - '@esbuild/linux-ppc64@0.28.2': - resolution: {integrity: sha512-CAXl+Dtd9UUuJd8pKKdwh6MLm3MUMiqMPmhZ3tTSXPqfyQ3vDl6R5hZdZ/kYojK4ofXtdfSv1tFq8XzWx3heNQ==} + '@esbuild/linux-ppc64@0.28.0': + resolution: {integrity: sha512-bc0FE9wWeC0WBm49IQMPSPILRocGTQt3j5KPCA8os6VprfuJ7KD+5PzESSrJ6GmPIPJK965ZJHTUlSA6GNYEhg==} engines: {node: '>=18'} cpu: [ppc64] os: [linux] @@ -3650,8 +3745,8 @@ packages: cpu: [riscv64] os: [linux] - '@esbuild/linux-riscv64@0.28.2': - resolution: {integrity: sha512-GeXCej4IQtU1B+QlDV8W/RRvbzI3O/Stss+/bCXv4lZls5WGRtu2a+3JkA3i4qIUlMXpcHebWpF8AkJhATowuA==} + '@esbuild/linux-riscv64@0.28.0': + resolution: {integrity: sha512-SQPZOwoTTT/HXFXQJG/vBX8sOFagGqvZyXcgLA3NhIqcBv1BJU1d46c0rGcrij2B56Z2rNiSLaZOYW5cUk7yLQ==} engines: {node: '>=18'} cpu: [riscv64] os: [linux] @@ -3662,8 +3757,8 @@ packages: cpu: [s390x] os: [linux] - '@esbuild/linux-s390x@0.28.2': - resolution: {integrity: sha512-3H1weTYZPxt/WOhByszQZybS9w5lKzUn1FDMsgEChbHWQwHYQQRfBxgCcZvPhjHfKyJjIievvMmEUawJrdY9Dg==} + '@esbuild/linux-s390x@0.28.0': + resolution: {integrity: sha512-SCfR0HN8CEEjnYnySJTd2cw0k9OHB/YFzt5zgJEwa+wL/T/raGWYMBqwDNAC6dqFKmJYZoQBRfHjgwLHGSrn3Q==} engines: {node: '>=18'} cpu: [s390x] os: [linux] @@ -3674,8 +3769,8 @@ packages: cpu: [x64] os: [linux] - '@esbuild/linux-x64@0.28.2': - resolution: {integrity: sha512-4xTZr1FUmSoQW4XIWmit3tzQrUTZM+N3P0XV8xROKYF50XfI7xeO90+1bZvNwxIufQ9hDQVRJH5YhgPVF8A/HQ==} + '@esbuild/linux-x64@0.28.0': + resolution: {integrity: sha512-us0dSb9iFxIi8srnpl931Nvs65it/Jd2a2K3qs7fz2WfGPHqzfzZTfec7oxZJRNPXPnNYZtanmRc4AL/JwVzHQ==} engines: {node: '>=18'} cpu: [x64] os: [linux] @@ -3686,8 +3781,8 @@ packages: cpu: [arm64] os: [netbsd] - '@esbuild/netbsd-arm64@0.28.2': - resolution: {integrity: sha512-sSATRjPeDBg3pdgHoQfoYBob11Kk1FGa9lui5RIHZCoCkJa9QKlvl3/vKz2usCmYYjs7ymJR/2Nnsqe+Hjt5nw==} + '@esbuild/netbsd-arm64@0.28.0': + resolution: {integrity: sha512-CR/RYotgtCKwtftMwJlUU7xCVNg3lMYZ0RzTmAHSfLCXw3NtZtNpswLEj/Kkf6kEL3Gw+BpOekRX0BYCtklhUw==} engines: {node: '>=18'} cpu: [arm64] os: [netbsd] @@ -3698,8 +3793,8 @@ packages: cpu: [x64] os: [netbsd] - '@esbuild/netbsd-x64@0.28.2': - resolution: {integrity: sha512-lqnzCV+mM0gIADaKihiCg6ifgfU2L3h5E33rNQBN1Y4MaVGnzryzmvvf7UHxprpQdE8hpqLolJ9Rl+SkIRDpyw==} + '@esbuild/netbsd-x64@0.28.0': + resolution: {integrity: sha512-nU1yhmYutL+fQ71Kxnhg8uEOdC0pwEW9entHykTgEbna2pw2dkbFSMeqjjyHZoCmt8SBkOSvV+yNmm94aUrrqw==} engines: {node: '>=18'} cpu: [x64] os: [netbsd] @@ -3710,8 +3805,8 @@ packages: cpu: [arm64] os: [openbsd] - '@esbuild/openbsd-arm64@0.28.2': - resolution: {integrity: sha512-AL2qJILH7lNjrDmCQDvdxMfAUIv8KMNZOvrwAQ8i8//ntL9FflhOyMJ8OZSMBb8/AWXe3/5v5S20y3zCoZWKoQ==} + '@esbuild/openbsd-arm64@0.28.0': + resolution: {integrity: sha512-cXb5vApOsRsxsEl4mcZ1XY3D4DzcoMxR/nnc4IyqYs0rTI8ZKmW6kyyg+11Z8yvgMfAEldKzP7AdP64HnSC/6g==} engines: {node: '>=18'} cpu: [arm64] os: [openbsd] @@ -3722,8 +3817,8 @@ packages: cpu: [x64] os: [openbsd] - '@esbuild/openbsd-x64@0.28.2': - resolution: {integrity: sha512-QtiuPytchRyC4rwUKhexJdQKvDuZ6hWloi3igqPQNUJCS1/v9EiO3UTOXR6A3FoMo4fnAKbWJdqaIwhOzh8qEw==} + '@esbuild/openbsd-x64@0.28.0': + resolution: {integrity: sha512-8wZM2qqtv9UP3mzy7HiGYNH/zjTA355mpeuA+859TyR+e+Tc08IHYpLJuMsfpDJwoLo1ikIJI8jC3GFjnRClzA==} engines: {node: '>=18'} cpu: [x64] os: [openbsd] @@ -3734,8 +3829,8 @@ packages: cpu: [arm64] os: [openharmony] - '@esbuild/openharmony-arm64@0.28.2': - resolution: {integrity: sha512-WkhYDmpTjLvGlScA1rwjRUmhl4k8oXR3cIbtqWmELgU/dFeHHlEllxDvdWcNJV9rbzCexB5vz8gtNewWLgCT7Q==} + '@esbuild/openharmony-arm64@0.28.0': + resolution: {integrity: sha512-FLGfyizszcef5C3YtoyQDACyg95+dndv79i2EekILBofh5wpCa1KuBqOWKrEHZg3zrL3t5ouE5jgr94vA+Wb2w==} engines: {node: '>=18'} cpu: [arm64] os: [openharmony] @@ -3746,8 +3841,8 @@ packages: cpu: [x64] os: [sunos] - '@esbuild/sunos-x64@0.28.2': - resolution: {integrity: sha512-GPMSkTOtMnv2U2F8gxe4Io6qmVs+YKyp832Etqqxr0hFngmXQ3rzwytelm3GIn7T4VviRUlf3sOgBOiTdvaf7g==} + '@esbuild/sunos-x64@0.28.0': + resolution: {integrity: sha512-1ZgjUoEdHZZl/YlV76TSCz9Hqj9h9YmMGAgAPYd+q4SicWNX3G5GCyx9uhQWSLcbvPW8Ni7lj4gDa1T40akdlw==} engines: {node: '>=18'} cpu: [x64] os: [sunos] @@ -3758,8 +3853,8 @@ packages: cpu: [arm64] os: [win32] - '@esbuild/win32-arm64@0.28.2': - resolution: {integrity: sha512-PIhhEkE9uPBleRBrQEJpUn7MBnibZzbGzYWPmY3x+YoVg/95zbjB4CxPPOQ8l5tYYM4mMaCthF8/1DIfBQQyWQ==} + '@esbuild/win32-arm64@0.28.0': + resolution: {integrity: sha512-Q9StnDmQ/enxnpxCCLSg0oo4+34B9TdXpuyPeTedN/6+iXBJ4J+zwfQI28u/Jl40nOYAxGoNi7mFP40RUtkmUA==} engines: {node: '>=18'} cpu: [arm64] os: [win32] @@ -3770,8 +3865,8 @@ packages: cpu: [ia32] os: [win32] - '@esbuild/win32-ia32@0.28.2': - resolution: {integrity: sha512-YmJbfTlvU7Sdn9BB+4PRES4oB6pxgS37MAONj+hBr/cpXS1aBPKXxNnDbu+QCWPj0o9dgyxeq79g6c5P8KeuYA==} + '@esbuild/win32-ia32@0.28.0': + resolution: {integrity: sha512-zF3ag/gfiCe6U2iczcRzSYJKH1DCI+ByzSENHlM2FcDbEeo5Zd2C86Aq0tKUYAJJ1obRP84ymxIAksZUcdztHA==} engines: {node: '>=18'} cpu: [ia32] os: [win32] @@ -3782,8 +3877,8 @@ packages: cpu: [x64] os: [win32] - '@esbuild/win32-x64@0.28.2': - resolution: {integrity: sha512-5ebpxr3nWMzrL/rnUI755Jkuee0bHL/Gq0WTF9lvcpv73wAp5eu8MfBUgWK9bhWvZjj7yX8etf/8tI8Ney695g==} + '@esbuild/win32-x64@0.28.0': + resolution: {integrity: sha512-pEl1bO9mfAmIC+tW5btTmrKaujg3zGtUmWNdCw/xs70FBjwAL3o9OEKNHvNmnyylD6ubxUERiEhdsL0xBQ9efw==} engines: {node: '>=18'} cpu: [x64] os: [win32] @@ -6810,8 +6905,8 @@ packages: engines: {node: '>=18'} hasBin: true - esbuild@0.28.2: - resolution: {integrity: sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==} + esbuild@0.28.0: + resolution: {integrity: sha512-sNR9MHpXSUV/XB4zmsFKN+QgVG82Cc7+/aaxJ8Adi8hyOac+EXptIp45QBPaVyX3N70664wRbTcLTOemCAnyqw==} engines: {node: '>=18'} hasBin: true @@ -7202,15 +7297,6 @@ packages: fn.name@1.1.0: resolution: {integrity: sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==} - follow-redirects@1.15.6: - resolution: {integrity: sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==} - engines: {node: '>=4.0'} - peerDependencies: - debug: '*' - peerDependenciesMeta: - debug: - optional: true - follow-redirects@1.16.0: resolution: {integrity: sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==} engines: {node: '>=4.0'} @@ -7396,11 +7482,11 @@ packages: glob@6.0.4: resolution: {integrity: sha512-MKZeRNyYZAVVVG1oZeLaWie1uweH40m9AZwIwxyPbTSX4hHrVYSzLg0Ro5Z5R7XKkIX+Cc6oD1rqeDJnwsB8/A==} - deprecated: Glob versions prior to v9 are no longer supported + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me glob@7.2.3: resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==} - deprecated: Glob versions prior to v9 are no longer supported + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me global-directory@4.0.1: resolution: {integrity: sha512-wHTUcDUoZ1H5/0iVqEudYW4/kAlN5cZ3j/bXn0Dpbizl9iaUVeWSHqiOjsgk6OW2bkLclbBjzewBz6weQ1zA2Q==} @@ -10806,6 +10892,9 @@ packages: peerDependencies: zod: ^3.25.28 || ^4 + zod@3.25.76: + resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} + zod@4.3.6: resolution: {integrity: sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==} @@ -10973,8 +11062,8 @@ snapshots: dependencies: '@aws-crypto/sha256-browser': 3.0.0 '@aws-crypto/sha256-js': 3.0.0 - '@aws-sdk/client-sso-oidc': 3.572.0 - '@aws-sdk/client-sts': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) + '@aws-sdk/client-sso-oidc': 3.572.0(@aws-sdk/client-sts@3.572.0) + '@aws-sdk/client-sts': 3.572.0 '@aws-sdk/core': 3.572.0 '@aws-sdk/credential-provider-node': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0)(@aws-sdk/client-sts@3.572.0) '@aws-sdk/middleware-host-header': 3.567.0 @@ -11182,11 +11271,11 @@ snapshots: transitivePeerDependencies: - aws-crt - '@aws-sdk/client-sso-oidc@3.572.0': + '@aws-sdk/client-sso-oidc@3.572.0(@aws-sdk/client-sts@3.572.0)': dependencies: '@aws-crypto/sha256-browser': 3.0.0 '@aws-crypto/sha256-js': 3.0.0 - '@aws-sdk/client-sts': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) + '@aws-sdk/client-sts': 3.572.0 '@aws-sdk/core': 3.572.0 '@aws-sdk/credential-provider-node': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0)(@aws-sdk/client-sts@3.572.0) '@aws-sdk/middleware-host-header': 3.567.0 @@ -11225,6 +11314,7 @@ snapshots: '@smithy/util-utf8': 2.3.0 tslib: 2.6.2 transitivePeerDependencies: + - '@aws-sdk/client-sts' - aws-crt '@aws-sdk/client-sso@3.556.0': @@ -11412,11 +11502,11 @@ snapshots: transitivePeerDependencies: - aws-crt - '@aws-sdk/client-sts@3.572.0(@aws-sdk/client-sso-oidc@3.572.0)': + '@aws-sdk/client-sts@3.572.0': dependencies: '@aws-crypto/sha256-browser': 3.0.0 '@aws-crypto/sha256-js': 3.0.0 - '@aws-sdk/client-sso-oidc': 3.572.0 + '@aws-sdk/client-sso-oidc': 3.572.0(@aws-sdk/client-sts@3.572.0) '@aws-sdk/core': 3.572.0 '@aws-sdk/credential-provider-node': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0)(@aws-sdk/client-sts@3.572.0) '@aws-sdk/middleware-host-header': 3.567.0 @@ -11455,7 +11545,6 @@ snapshots: '@smithy/util-utf8': 2.3.0 tslib: 2.6.2 transitivePeerDependencies: - - '@aws-sdk/client-sso-oidc' - aws-crt '@aws-sdk/core@3.556.0': @@ -11606,7 +11695,7 @@ snapshots: '@aws-sdk/credential-provider-ini@3.572.0(@aws-sdk/client-sso-oidc@3.572.0)(@aws-sdk/client-sts@3.572.0)': dependencies: - '@aws-sdk/client-sts': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) + '@aws-sdk/client-sts': 3.572.0 '@aws-sdk/credential-provider-env': 3.568.0 '@aws-sdk/credential-provider-process': 3.572.0 '@aws-sdk/credential-provider-sso': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) @@ -11840,7 +11929,7 @@ snapshots: '@aws-sdk/credential-provider-web-identity@3.568.0(@aws-sdk/client-sts@3.572.0)': dependencies: - '@aws-sdk/client-sts': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) + '@aws-sdk/client-sts': 3.572.0 '@aws-sdk/types': 3.567.0 '@smithy/property-provider': 2.2.0 '@smithy/types': 2.12.0 @@ -12195,7 +12284,7 @@ snapshots: '@aws-sdk/token-providers@3.572.0(@aws-sdk/client-sso-oidc@3.572.0)': dependencies: - '@aws-sdk/client-sso-oidc': 3.572.0 + '@aws-sdk/client-sso-oidc': 3.572.0(@aws-sdk/client-sts@3.572.0) '@aws-sdk/types': 3.567.0 '@smithy/property-provider': 2.2.0 '@smithy/shared-ini-file-loader': 2.4.0 @@ -12543,7 +12632,7 @@ snapshots: '@babel/helper-split-export-declaration': 7.22.6 '@babel/parser': 7.24.4 '@babel/types': 7.24.0 - debug: 4.3.7 + debug: 4.4.3(supports-color@5.5.0) globals: 11.12.0 transitivePeerDependencies: - supports-color @@ -12754,157 +12843,157 @@ snapshots: '@esbuild/aix-ppc64@0.25.12': optional: true - '@esbuild/aix-ppc64@0.28.2': + '@esbuild/aix-ppc64@0.28.0': optional: true '@esbuild/android-arm64@0.25.12': optional: true - '@esbuild/android-arm64@0.28.2': + '@esbuild/android-arm64@0.28.0': optional: true '@esbuild/android-arm@0.25.12': optional: true - '@esbuild/android-arm@0.28.2': + '@esbuild/android-arm@0.28.0': optional: true '@esbuild/android-x64@0.25.12': optional: true - '@esbuild/android-x64@0.28.2': + '@esbuild/android-x64@0.28.0': optional: true '@esbuild/darwin-arm64@0.25.12': optional: true - '@esbuild/darwin-arm64@0.28.2': + '@esbuild/darwin-arm64@0.28.0': optional: true '@esbuild/darwin-x64@0.25.12': optional: true - '@esbuild/darwin-x64@0.28.2': + '@esbuild/darwin-x64@0.28.0': optional: true '@esbuild/freebsd-arm64@0.25.12': optional: true - '@esbuild/freebsd-arm64@0.28.2': + '@esbuild/freebsd-arm64@0.28.0': optional: true '@esbuild/freebsd-x64@0.25.12': optional: true - '@esbuild/freebsd-x64@0.28.2': + '@esbuild/freebsd-x64@0.28.0': optional: true '@esbuild/linux-arm64@0.25.12': optional: true - '@esbuild/linux-arm64@0.28.2': + '@esbuild/linux-arm64@0.28.0': optional: true '@esbuild/linux-arm@0.25.12': optional: true - '@esbuild/linux-arm@0.28.2': + '@esbuild/linux-arm@0.28.0': optional: true '@esbuild/linux-ia32@0.25.12': optional: true - '@esbuild/linux-ia32@0.28.2': + '@esbuild/linux-ia32@0.28.0': optional: true '@esbuild/linux-loong64@0.25.12': optional: true - '@esbuild/linux-loong64@0.28.2': + '@esbuild/linux-loong64@0.28.0': optional: true '@esbuild/linux-mips64el@0.25.12': optional: true - '@esbuild/linux-mips64el@0.28.2': + '@esbuild/linux-mips64el@0.28.0': optional: true '@esbuild/linux-ppc64@0.25.12': optional: true - '@esbuild/linux-ppc64@0.28.2': + '@esbuild/linux-ppc64@0.28.0': optional: true '@esbuild/linux-riscv64@0.25.12': optional: true - '@esbuild/linux-riscv64@0.28.2': + '@esbuild/linux-riscv64@0.28.0': optional: true '@esbuild/linux-s390x@0.25.12': optional: true - '@esbuild/linux-s390x@0.28.2': + '@esbuild/linux-s390x@0.28.0': optional: true '@esbuild/linux-x64@0.25.12': optional: true - '@esbuild/linux-x64@0.28.2': + '@esbuild/linux-x64@0.28.0': optional: true '@esbuild/netbsd-arm64@0.25.12': optional: true - '@esbuild/netbsd-arm64@0.28.2': + '@esbuild/netbsd-arm64@0.28.0': optional: true '@esbuild/netbsd-x64@0.25.12': optional: true - '@esbuild/netbsd-x64@0.28.2': + '@esbuild/netbsd-x64@0.28.0': optional: true '@esbuild/openbsd-arm64@0.25.12': optional: true - '@esbuild/openbsd-arm64@0.28.2': + '@esbuild/openbsd-arm64@0.28.0': optional: true '@esbuild/openbsd-x64@0.25.12': optional: true - '@esbuild/openbsd-x64@0.28.2': + '@esbuild/openbsd-x64@0.28.0': optional: true '@esbuild/openharmony-arm64@0.25.12': optional: true - '@esbuild/openharmony-arm64@0.28.2': + '@esbuild/openharmony-arm64@0.28.0': optional: true '@esbuild/sunos-x64@0.25.12': optional: true - '@esbuild/sunos-x64@0.28.2': + '@esbuild/sunos-x64@0.28.0': optional: true '@esbuild/win32-arm64@0.25.12': optional: true - '@esbuild/win32-arm64@0.28.2': + '@esbuild/win32-arm64@0.28.0': optional: true '@esbuild/win32-ia32@0.25.12': optional: true - '@esbuild/win32-ia32@0.28.2': + '@esbuild/win32-ia32@0.28.0': optional: true '@esbuild/win32-x64@0.25.12': optional: true - '@esbuild/win32-x64@0.28.2': + '@esbuild/win32-x64@0.28.0': optional: true '@eslint-community/eslint-utils@4.4.0(eslint@8.57.0)': @@ -12917,7 +13006,7 @@ snapshots: '@eslint/eslintrc@2.1.4': dependencies: ajv: 6.12.6 - debug: 4.3.4 + debug: 4.4.3(supports-color@5.5.0) espree: 9.6.1 globals: 13.24.0 ignore: 5.3.1 @@ -13090,7 +13179,7 @@ snapshots: '@humanwhocodes/config-array@0.11.14': dependencies: '@humanwhocodes/object-schema': 2.0.3 - debug: 4.3.4 + debug: 4.4.3(supports-color@5.5.0) minimatch: 3.1.2 transitivePeerDependencies: - supports-color @@ -15011,7 +15100,7 @@ snapshots: dependencies: '@typescript-eslint/types': 6.21.0 '@typescript-eslint/visitor-keys': 6.21.0 - debug: 4.3.4 + debug: 4.4.3(supports-color@5.5.0) globby: 11.1.0 is-glob: 4.0.3 minimatch: 9.0.3 @@ -15523,7 +15612,7 @@ snapshots: axios@0.27.2: dependencies: - follow-redirects: 1.15.6 + follow-redirects: 1.16.0 form-data: 4.0.6 transitivePeerDependencies: - debug @@ -16637,34 +16726,34 @@ snapshots: '@esbuild/win32-ia32': 0.25.12 '@esbuild/win32-x64': 0.25.12 - esbuild@0.28.2: + esbuild@0.28.0: optionalDependencies: - '@esbuild/aix-ppc64': 0.28.2 - '@esbuild/android-arm': 0.28.2 - '@esbuild/android-arm64': 0.28.2 - '@esbuild/android-x64': 0.28.2 - '@esbuild/darwin-arm64': 0.28.2 - '@esbuild/darwin-x64': 0.28.2 - '@esbuild/freebsd-arm64': 0.28.2 - '@esbuild/freebsd-x64': 0.28.2 - '@esbuild/linux-arm': 0.28.2 - '@esbuild/linux-arm64': 0.28.2 - '@esbuild/linux-ia32': 0.28.2 - '@esbuild/linux-loong64': 0.28.2 - '@esbuild/linux-mips64el': 0.28.2 - '@esbuild/linux-ppc64': 0.28.2 - '@esbuild/linux-riscv64': 0.28.2 - '@esbuild/linux-s390x': 0.28.2 - '@esbuild/linux-x64': 0.28.2 - '@esbuild/netbsd-arm64': 0.28.2 - '@esbuild/netbsd-x64': 0.28.2 - '@esbuild/openbsd-arm64': 0.28.2 - '@esbuild/openbsd-x64': 0.28.2 - '@esbuild/openharmony-arm64': 0.28.2 - '@esbuild/sunos-x64': 0.28.2 - '@esbuild/win32-arm64': 0.28.2 - '@esbuild/win32-ia32': 0.28.2 - '@esbuild/win32-x64': 0.28.2 + '@esbuild/aix-ppc64': 0.28.0 + '@esbuild/android-arm': 0.28.0 + '@esbuild/android-arm64': 0.28.0 + '@esbuild/android-x64': 0.28.0 + '@esbuild/darwin-arm64': 0.28.0 + '@esbuild/darwin-x64': 0.28.0 + '@esbuild/freebsd-arm64': 0.28.0 + '@esbuild/freebsd-x64': 0.28.0 + '@esbuild/linux-arm': 0.28.0 + '@esbuild/linux-arm64': 0.28.0 + '@esbuild/linux-ia32': 0.28.0 + '@esbuild/linux-loong64': 0.28.0 + '@esbuild/linux-mips64el': 0.28.0 + '@esbuild/linux-ppc64': 0.28.0 + '@esbuild/linux-riscv64': 0.28.0 + '@esbuild/linux-s390x': 0.28.0 + '@esbuild/linux-x64': 0.28.0 + '@esbuild/netbsd-arm64': 0.28.0 + '@esbuild/netbsd-x64': 0.28.0 + '@esbuild/openbsd-arm64': 0.28.0 + '@esbuild/openbsd-x64': 0.28.0 + '@esbuild/openharmony-arm64': 0.28.0 + '@esbuild/sunos-x64': 0.28.0 + '@esbuild/win32-arm64': 0.28.0 + '@esbuild/win32-ia32': 0.28.0 + '@esbuild/win32-x64': 0.28.0 escalade@3.1.2: {} @@ -17234,8 +17323,6 @@ snapshots: fn.name@1.1.0: {} - follow-redirects@1.15.6: {} - follow-redirects@1.16.0: {} for-each@0.3.3: @@ -20488,7 +20575,7 @@ snapshots: tsx@4.23.12: dependencies: - esbuild: 0.28.2 + esbuild: 0.28.0 optionalDependencies: fsevents: 2.3.3 @@ -21105,4 +21192,6 @@ snapshots: dependencies: zod: 4.3.6 + zod@3.25.76: {} + zod@4.3.6: {} diff --git a/scripts/builders/connectors-worker.env b/scripts/builders/connectors-worker.env new file mode 100644 index 0000000000..e2858df2a7 --- /dev/null +++ b/scripts/builders/connectors-worker.env @@ -0,0 +1,4 @@ +DOCKERFILE="./services/docker/Dockerfile.connectors_worker" +CONTEXT="../" +REPO="sjc.ocir.io/axbydjxa5zuh/connectors-worker" +SERVICES="connectors-worker" diff --git a/scripts/cli b/scripts/cli index 15614dc4e0..16fbb5f811 100755 --- a/scripts/cli +++ b/scripts/cli @@ -1190,7 +1190,7 @@ while test $# -gt 0; do exit ;; clean-start-fe-dev) - IGNORED_SERVICES=("frontend" "python-worker" "job-generator" "webhook-api" "profiles-worker" "organizations-enrichment-worker" "merge-suggestions-worker" "members-enrichment-worker" "exports-worker" "entity-merging-worker" "cache-worker" "categorization-worker" "cron-service" "data-sink-worker" "git-integration" "mailing-list-integration" "integration-run-worker" "integration-stream-worker" "nango-webhook-api" "nango-worker" "script-executor-worker" "search-sync-api" "search-sync-worker" "security-best-practices-worker" "snowflake-connectors-worker" "automatic-projects-discovery-worker" "pcc-sync-worker" "projects-evaluation-worker" "bq-dataset-ingest" "cargo-worker" "dockerhub-sync" "github-repos-enricher" "go-worker" "maven-worker" "npm-worker" "nuget-worker" "osv-worker" "packagist-worker" "pypi-worker" "rubygems-worker" "security-contacts-worker") + IGNORED_SERVICES=("frontend" "python-worker" "job-generator" "webhook-api" "profiles-worker" "organizations-enrichment-worker" "merge-suggestions-worker" "members-enrichment-worker" "exports-worker" "entity-merging-worker" "cache-worker" "categorization-worker" "cron-service" "data-sink-worker" "git-integration" "mailing-list-integration" "integration-run-worker" "integration-stream-worker" "nango-webhook-api" "nango-worker" "connectors-worker" "script-executor-worker" "search-sync-api" "search-sync-worker" "security-best-practices-worker" "snowflake-connectors-worker" "automatic-projects-discovery-worker" "pcc-sync-worker" "projects-evaluation-worker" "bq-dataset-ingest" "cargo-worker" "dockerhub-sync" "github-repos-enricher" "go-worker" "maven-worker" "npm-worker" "nuget-worker" "osv-worker" "packagist-worker" "pypi-worker" "rubygems-worker" "security-contacts-worker") CLEAN_START=1 DEV=1 start diff --git a/scripts/services/connectors-worker.yaml b/scripts/services/connectors-worker.yaml new file mode 100644 index 0000000000..140b27639e --- /dev/null +++ b/scripts/services/connectors-worker.yaml @@ -0,0 +1,57 @@ +version: '3.1' + +x-env-args: &env-args + DOCKER_BUILDKIT: 1 + NODE_ENV: docker + SERVICE: connectors-worker + CROWD_TEMPORAL_TASKQUEUE: connectors + SHELL: /bin/sh + +services: + connectors-worker: + build: + context: ../../ + dockerfile: ./scripts/services/docker/Dockerfile.connectors_worker + command: 'pnpm run start' + working_dir: /usr/crowd/app/services/apps/connectors_worker + env_file: + - ../../backend/.env.dist.local + - ../../backend/.env.dist.composed + - ../../backend/.env.override.local + - ../../backend/.env.override.composed + environment: + <<: *env-args + restart: always + networks: + - crowd-bridge + + connectors-worker-dev: + build: + context: ../../ + dockerfile: ./scripts/services/docker/Dockerfile.connectors_worker + command: 'pnpm run dev' + working_dir: /usr/crowd/app/services/apps/connectors_worker + env_file: + - ../../backend/.env.dist.local + - ../../backend/.env.dist.composed + - ../../backend/.env.override.local + - ../../backend/.env.override.composed + environment: + <<: *env-args + hostname: connectors-worker + networks: + - crowd-bridge + volumes: + - ../../services/libs/common/src:/usr/crowd/app/services/libs/common/src + - ../../services/libs/connectors/src:/usr/crowd/app/services/libs/connectors/src + - ../../services/libs/data-access-layer/src:/usr/crowd/app/services/libs/data-access-layer/src + - ../../services/libs/database/src:/usr/crowd/app/services/libs/database/src + - ../../services/libs/logging/src:/usr/crowd/app/services/libs/logging/src + - ../../services/libs/redis/src:/usr/crowd/app/services/libs/redis/src + - ../../services/libs/temporal/src:/usr/crowd/app/services/libs/temporal/src + - ../../services/libs/types/src:/usr/crowd/app/services/libs/types/src + - ../../services/apps/connectors_worker/src:/usr/crowd/app/services/apps/connectors_worker/src + +networks: + crowd-bridge: + external: true diff --git a/scripts/services/docker/Dockerfile.connectors_worker b/scripts/services/docker/Dockerfile.connectors_worker new file mode 100644 index 0000000000..e17f7bcb4c --- /dev/null +++ b/scripts/services/docker/Dockerfile.connectors_worker @@ -0,0 +1,23 @@ +FROM node:24-alpine as builder + +RUN apk add --no-cache python3 make g++ + +WORKDIR /usr/crowd/app +RUN npm install -g corepack@latest && corepack enable pnpm && corepack prepare pnpm@9.15.0 --activate + +COPY ./pnpm-workspace.yaml ./pnpm-lock.yaml ./ +RUN pnpm fetch + +COPY ./services ./services +RUN pnpm i --frozen-lockfile + +FROM node:24-bookworm-slim as runner + +WORKDIR /usr/crowd/app +RUN npm install -g corepack@latest && corepack enable pnpm && corepack prepare pnpm@9.15.0 --activate && apt update && apt install -y ca-certificates --no-install-recommends && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /usr/crowd/app/node_modules ./node_modules +COPY --from=builder /usr/crowd/app/services/base.tsconfig.json ./services/base.tsconfig.json +COPY --from=builder /usr/crowd/app/services/libs ./services/libs +COPY --from=builder /usr/crowd/app/services/archetypes/ ./services/archetypes +COPY --from=builder /usr/crowd/app/services/apps/connectors_worker/ ./services/apps/connectors_worker diff --git a/scripts/services/docker/Dockerfile.connectors_worker.dockerignore b/scripts/services/docker/Dockerfile.connectors_worker.dockerignore new file mode 100644 index 0000000000..4b74fc87af --- /dev/null +++ b/scripts/services/docker/Dockerfile.connectors_worker.dockerignore @@ -0,0 +1,18 @@ +**/.git +**/node_modules +**/venv* +**/.webpack +**/.serverless +**/.env +**/.env.* +**/.idea +**/.vscode +**/dist +.vscode/ +.github/ +frontend/ +scripts/ +.flake8 +*.md +Makefile +backend/ diff --git a/services/apps/connectors_worker/package.json b/services/apps/connectors_worker/package.json new file mode 100644 index 0000000000..065fbf84e8 --- /dev/null +++ b/services/apps/connectors_worker/package.json @@ -0,0 +1,37 @@ +{ + "name": "@crowd/connectors-worker", + "private": true, + "scripts": { + "start": "CROWD_TEMPORAL_TASKQUEUE=connectors SERVICE=connectors-worker tsx src/main.ts", + "start:debug:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=connectors SERVICE=connectors-worker LOG_LEVEL=trace tsx --inspect=0.0.0.0:9243 src/main.ts", + "start:debug": "CROWD_TEMPORAL_TASKQUEUE=connectors SERVICE=connectors-worker LOG_LEVEL=trace tsx --inspect=0.0.0.0:9243 src/main.ts", + "dev:local": "nodemon --watch src --watch ../../libs --ext ts --exec pnpm run start:debug:local", + "dev": "nodemon --watch src --watch ../../libs --ext ts --exec pnpm run start:debug", + "lint": "npx eslint --ext .ts src --max-warnings=0", + "format": "npx prettier --write \"src/**/*.ts\"", + "format-check": "npx prettier --check .", + "tsc-check": "tsc --noEmit", + "script:seed-github-sync-units": "tsx src/bin/seed-github-sync-units.ts", + "script:audit-github-app-budget": "tsx src/bin/audit-github-app-budget.ts" + }, + "dependencies": { + "@crowd/archetype-standard": "workspace:*", + "@crowd/archetype-worker": "workspace:*", + "@crowd/common": "workspace:*", + "@crowd/data-access-layer": "workspace:*", + "@crowd/connectors": "workspace:*", + "@crowd/logging": "workspace:*", + "@crowd/redis": "workspace:*", + "@crowd/temporal": "workspace:*", + "@crowd/types": "workspace:*", + "@temporalio/activity": "~1.17.2", + "@temporalio/client": "~1.17.2", + "@temporalio/workflow": "~1.17.2", + "tsx": "^4.7.1", + "typescript": "^5.6.3" + }, + "devDependencies": { + "@types/node": "^24.13.3", + "nodemon": "^3.0.1" + } +} diff --git a/services/apps/connectors_worker/src/activities.ts b/services/apps/connectors_worker/src/activities.ts new file mode 100644 index 0000000000..9add180943 --- /dev/null +++ b/services/apps/connectors_worker/src/activities.ts @@ -0,0 +1,19 @@ +import { + admitByBudget, + claimDue, + deferUnit, + logDispatchSummary, + startRun, + touchHeartbeat, +} from './activities/dispatcherActivities' +import { executeSync } from './activities/syncRunActivities' + +export { + admitByBudget, + claimDue, + deferUnit, + executeSync, + logDispatchSummary, + startRun, + touchHeartbeat, +} diff --git a/services/apps/connectors_worker/src/activities/dispatcherActivities.ts b/services/apps/connectors_worker/src/activities/dispatcherActivities.ts new file mode 100644 index 0000000000..2a563e2dd0 --- /dev/null +++ b/services/apps/connectors_worker/src/activities/dispatcherActivities.ts @@ -0,0 +1,110 @@ +import { createTokenPool, findManifest, mapWithConcurrency } from '@crowd/connectors' +import { claimDueUnits, rescheduleUnit } from '@crowd/data-access-layer/src/connectors' +import type { IClaimedUnit } from '@crowd/data-access-layer/src/connectors' +import { dbStoreQx } from '@crowd/data-access-layer/src/queryExecutor' +import { RedisCache } from '@crowd/redis' +import { WorkflowIdConflictPolicy, WorkflowIdReusePolicy } from '@crowd/temporal' + +import { svc } from '../main' +import { runningProbeRunAt, shortDeferRunAt } from '../scheduling' +import type { IAdmissionResult, IDispatchCounts, StartRunResult } from '../types' + +const TASK_QUEUE = 'connectors' +const HEARTBEAT_TTL_SECONDS = 300 +const DEFAULT_RUN_ESTIMATE = 50 +const BUDGET_PROBE_CONCURRENCY = 10 +const CHANNEL_LABEL_MAX_LENGTH = 80 + +export async function claimDue(limit: number): Promise { + return claimDueUnits(dbStoreQx(svc.postgres.writer), limit) +} + +export async function admitByBudget(units: IClaimedUnit[]): Promise { + const headrooms = await mapWithConcurrency(units, BUDGET_PROBE_CONCURRENCY, (unit) => { + const manifest = findManifest(unit.platform) + const pool = createTokenPool(svc.redis, unit.platform, { + probeBudget: manifest?.probeBudget, + }) + return pool.hasHeadroom(DEFAULT_RUN_ESTIMATE) + }) + const result = { + admitted: units.filter((_, index) => headrooms[index]), + deferred: units.filter((_, index) => !headrooms[index]), + } + if (result.deferred.length > 0) { + svc.log.info( + { unitIds: result.deferred.map((unit) => unit.id) }, + 'units deferred: no token headroom', + ) + } + return result +} + +export async function deferUnit(unitId: string): Promise { + await rescheduleUnit(dbStoreQx(svc.postgres.writer), unitId, shortDeferRunAt()) +} + +function channelLabel(channelName: string): string { + const label = channelName + .replace(/^[a-z][a-z0-9+.-]*:\/\/[^/]+\//i, '') + .replace(/^\/+|\/+$/g, '') + .replace(/\s+/g, '-') + .slice(0, CHANNEL_LABEL_MAX_LENGTH) + + return label || 'unknown-channel' +} + +export function syncRunWorkflowId(unit: IClaimedUnit): string { + return `sync-run/${unit.platform}/${unit.syncName}/${channelLabel(unit.channelName)}/${unit.id}` +} + +export async function startRun(unit: IClaimedUnit): Promise { + try { + await svc.temporal.workflow.start('syncRun', { + taskQueue: TASK_QUEUE, + workflowId: syncRunWorkflowId(unit), + workflowIdReusePolicy: WorkflowIdReusePolicy.ALLOW_DUPLICATE, + workflowIdConflictPolicy: WorkflowIdConflictPolicy.FAIL, + args: [unit.id], + memo: { + unitId: unit.id, + integrationId: unit.integrationId, + platform: unit.platform, + syncName: unit.syncName, + channelId: unit.channelId, + channelName: unit.channelName, + }, + }) + return 'started' + } catch (err) { + if (err instanceof Error && err.name === 'WorkflowExecutionAlreadyStartedError') { + await backOffRunningUnit(unit) + return 'alreadyRunning' + } + throw err + } +} + +async function backOffRunningUnit(unit: IClaimedUnit): Promise { + const nextRunAt = await runningSince(unit) + .then(runningProbeRunAt) + .catch(() => shortDeferRunAt()) + + await rescheduleUnit(dbStoreQx(svc.postgres.writer), unit.id, nextRunAt) + svc.log.info({ unitId: unit.id, nextRunAt }, 'unit already running, rescheduled probe') +} + +async function runningSince(unit: IClaimedUnit): Promise { + const description = await svc.temporal.workflow.getHandle(syncRunWorkflowId(unit)).describe() + + return description.startTime +} + +export async function logDispatchSummary(counts: IDispatchCounts): Promise { + svc.log.info({ event: 'dispatcher_tick', ...counts }, 'dispatch summary') +} + +export async function touchHeartbeat(): Promise { + const cache = new RedisCache('connectors', svc.redis, svc.log) + await cache.set('dispatcherHeartbeat', new Date().toISOString(), HEARTBEAT_TTL_SECONDS) +} diff --git a/services/apps/connectors_worker/src/activities/syncRunActivities.ts b/services/apps/connectors_worker/src/activities/syncRunActivities.ts new file mode 100644 index 0000000000..ac5a1c9b1a --- /dev/null +++ b/services/apps/connectors_worker/src/activities/syncRunActivities.ts @@ -0,0 +1,214 @@ +import { Context } from '@temporalio/activity' + +import { + ConnectorError, + createEmit, + createHttpClient, + createTokenPool, + findSync, + getCredential, + getManifest, + getSync, +} from '@crowd/connectors' +import type { ConnectorHttp, Emitter, SyncContext } from '@crowd/connectors' +import { + getUnitById, + parkUnit, + recordRunFailure, + recordRunPartial, + recordRunSuccess, +} from '@crowd/data-access-layer/src/connectors' +import { fetchIntegrationById } from '@crowd/data-access-layer/src/integrations' +import IntegrationStreamRepository from '@crowd/data-access-layer/src/old/apps/integration_stream_worker/integrationStream.repo' +import { dbStoreQx } from '@crowd/data-access-layer/src/queryExecutor' +import { getChildLogger } from '@crowd/logging' + +import { svc } from '../main' +import { RUN_BUDGET_MS } from '../runLimits' +import { cadenceRunAt, failureRunAt, shortDeferRunAt } from '../scheduling' + +const DEAD_LETTER_AFTER = 5 +const HEARTBEAT_INTERVAL_MS = 10_000 +const RATE_LIMIT_FALLBACK_MS = 60_000 +const UNAVAILABLE_PARK_MS = 300_000 + +const PARKED_ERROR_CLASSES = ['provider.rate_limit', 'provider.unavailable'] as const + +export async function executeSync(unitId: string): Promise { + const qx = dbStoreQx(svc.postgres.writer) + + const unit = await getUnitById(qx, unitId) + if (!unit) { + throw new Error(`sync unit ${unitId} not found`) + } + + const activityContext = Context.current() + const log = getChildLogger('syncRun', svc.log, { + workflowId: activityContext.info.workflowExecution.workflowId, + runId: activityContext.info.workflowExecution.runId, + unitId: unit.id, + integrationId: unit.integrationId, + platform: unit.platform, + syncName: unit.syncName, + channelId: unit.channelId, + channelName: unit.channelName, + }) + + const heartbeat = setInterval(() => { + try { + activityContext.heartbeat() + } catch (err) { + log.warn({ errMsg: (err as Error).message }, 'heartbeat failed') + } + }, HEARTBEAT_INTERVAL_MS) + + let emitter: Emitter | null = null + let http: ConnectorHttp | null = null + let committedWatermark = unit.watermark + const startedAt = Date.now() + const runDeadline = startedAt + RUN_BUDGET_MS + + const runSummary = (fields: Record) => ({ + event: 'sync_run_summary', + durationMs: Date.now() - startedAt, + emittedCount: emitter?.emittedCount() ?? 0, + requestCount: http?.requestCount() ?? 0, + complete: null as boolean | null, + ...fields, + }) + + log.info( + { event: 'sync_run_started', consecutiveFailures: unit.consecutiveFailures }, + 'sync run started', + ) + + try { + const integration = await fetchIntegrationById(qx, unit.integrationId) + if (!integration?.segmentId) { + throw new Error(`integration ${unit.integrationId} not found or has no segmentId`) + } + if (!svc.dataSinkWorkerEmitter) { + throw new Error('data sink worker emitter not initialized') + } + + const manifest = getManifest(unit.platform) + const credential = + manifest.preparePool || manifest.mintToken + ? await getCredential(qx, unit.integrationId) + : null + const pool = createTokenPool(svc.redis, unit.platform, { + probeBudget: manifest.probeBudget, + mintToken: credential && manifest.mintToken ? manifest.mintToken(credential) : undefined, + log, + }) + let preferredEntryId: string | undefined + if (credential && manifest.preparePool) { + preferredEntryId = (await manifest.preparePool(credential, pool)).preferredEntryId + } + http = createHttpClient({ + acquireToken: () => pool.acquire(preferredEntryId), + parkToken: pool.park, + invalidateToken: pool.invalidate, + interpretResponse: manifest.interpretResponse, + log, + }) + + const sync = getSync(unit.platform, unit.syncName) + + const streamRepo = new IntegrationStreamRepository(svc.postgres.writer, log) + emitter = createEmit({ + publishResult: streamRepo.publishExternalResult.bind(streamRepo), + sinkEmitter: svc.dataSinkWorkerEmitter, + unit, + segmentId: integration.segmentId, + schema: sync.schema, + log, + }) + + const ctx: SyncContext = { + channel: { channelId: unit.channelId, channelName: unit.channelName }, + watermark: unit.watermark, + emit: emitter.emit, + commitWatermark: async (watermark) => { + committedWatermark = watermark + }, + hasRunBudget: () => Date.now() < runDeadline, + http, + log, + } + + const outcome = await sync.run(ctx) + const nextRunAt = outcome.complete ? cadenceRunAt(sync.cadenceMinutes) : shortDeferRunAt() + + await recordRunSuccess( + qx, + unitId, + { + watermark: committedWatermark ?? {}, + emittedCount: emitter.emittedCount(), + complete: outcome.complete, + }, + nextRunAt, + ) + log.info( + runSummary({ outcome: 'success', complete: outcome.complete, nextRunAt }), + 'sync run summary', + ) + } catch (err) { + if ( + err instanceof ConnectorError && + (PARKED_ERROR_CLASSES as readonly string[]).includes(err.errorClass) + ) { + const fallbackMs = + err.errorClass === 'provider.rate_limit' ? RATE_LIMIT_FALLBACK_MS : UNAVAILABLE_PARK_MS + const resumeAt = err.options?.resumeAt ?? new Date(Date.now() + fallbackMs) + const progressCommitted = Boolean(emitter && committedWatermark) + if (emitter && committedWatermark) { + await recordRunPartial( + qx, + unitId, + { watermark: committedWatermark, emittedCount: emitter.emittedCount() }, + resumeAt, + err.errorClass, + err.message, + ) + } else { + await parkUnit(qx, unitId, resumeAt, err.errorClass, err.message) + } + log.info( + runSummary({ + outcome: 'parked', + errorClass: err.errorClass, + errorMessage: err.message, + nextRunAt: resumeAt, + progressCommitted, + }), + 'sync run summary', + ) + return + } + const errorClass = err instanceof ConnectorError ? err.errorClass : 'unknown' + const deadLetterAfter = errorClass === 'provider.auth' ? DEAD_LETTER_AFTER : null + const consecutiveFailures = unit.consecutiveFailures + 1 + const nextRunAt = failureRunAt( + consecutiveFailures, + findSync(unit.platform, unit.syncName)?.cadenceMinutes ?? null, + ) + const errorMessage = err instanceof Error ? err.message : String(err) + log.error( + runSummary({ + outcome: 'failed', + errorClass, + errorMessage, + consecutiveFailures, + nextRunAt, + err, + }), + 'sync run summary', + ) + await recordRunFailure(qx, unitId, errorClass, errorMessage, deadLetterAfter, nextRunAt) + throw err + } finally { + clearInterval(heartbeat) + } +} diff --git a/services/apps/connectors_worker/src/bin/audit-github-app-budget.ts b/services/apps/connectors_worker/src/bin/audit-github-app-budget.ts new file mode 100644 index 0000000000..ddffdffc3d --- /dev/null +++ b/services/apps/connectors_worker/src/bin/audit-github-app-budget.ts @@ -0,0 +1,225 @@ +import type { Credential } from '@crowd/connectors' +import { getCredential, mapWithConcurrency } from '@crowd/connectors' +import type { InstallationSummary } from '@crowd/connectors/src/connectors/github/appToken' +import { + listInstallations, + mintInstallationToken, +} from '@crowd/connectors/src/connectors/github/appToken' +import { WRITE_DB_CONFIG, getDbConnection } from '@crowd/data-access-layer/src/database' +import { pgpQx } from '@crowd/data-access-layer/src/queryExecutor' +import { getServiceLogger } from '@crowd/logging' + +const log = getServiceLogger() + +const GITHUB_API_VERSION = '2022-11-28' +const REQUEST_TIMEOUT_MS = 30_000 +const DEFAULT_CONCURRENCY = 4 + +interface RateLimitResource { + limit: number + remaining: number + reset: number +} + +interface InstallationBudget { + installation: InstallationSummary + repoCount: number | null + graphql: RateLimitResource | null + core: RateLimitResource | null + error?: string +} + +function usage(): never { + log.error( + 'Usage: audit-github-app-budget --integration-id [--concurrency ] [--json] [--include-suspended]', + ) + process.exit(1) +} + +function takeFlag(argv: string[], flag: string): string | undefined { + const flagIndex = argv.indexOf(flag) + if (flagIndex === -1) { + return undefined + } + const value = argv[flagIndex + 1] + argv.splice(flagIndex, 2) + return value +} + +function takeSwitch(argv: string[], flag: string): boolean { + const flagIndex = argv.indexOf(flag) + if (flagIndex === -1) { + return false + } + argv.splice(flagIndex, 1) + return true +} + +function parseArgs(rawArgv: string[]): { + integrationId: string + concurrency: number + asJson: boolean + includeSuspended: boolean +} { + const argv = rawArgv.filter((arg) => arg !== '--') + const integrationId = takeFlag(argv, '--integration-id') + const concurrency = Number(takeFlag(argv, '--concurrency') ?? DEFAULT_CONCURRENCY) + const asJson = takeSwitch(argv, '--json') + const includeSuspended = takeSwitch(argv, '--include-suspended') + if (!integrationId || !Number.isInteger(concurrency) || concurrency < 1) { + usage() + } + return { integrationId, concurrency, asJson, includeSuspended } +} + +async function githubGet(token: string, url: string): Promise { + const response = await fetch(url, { + headers: { + Authorization: `Bearer ${token}`, + Accept: 'application/vnd.github+json', + 'X-GitHub-Api-Version': GITHUB_API_VERSION, + }, + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }) + const body = await response.json().catch(() => undefined) + if (!response.ok) { + const message = (body as { message?: string } | undefined)?.message ?? response.statusText + throw new Error(`${response.status}: ${message}`) + } + return body +} + +async function readRepoCount(token: string): Promise { + const body = await githubGet(token, 'https://api.github.com/installation/repositories?per_page=1') + return (body as { total_count: number }).total_count +} + +async function auditInstallation( + credential: Credential, + installation: InstallationSummary, +): Promise { + try { + const { token } = await mintInstallationToken(credential, installation.id) + const [rateLimit, repoCount] = await Promise.all([ + githubGet(token, 'https://api.github.com/rate_limit'), + readRepoCount(token), + ]) + const resources = (rateLimit as { resources?: Record }).resources + return { + installation, + repoCount, + graphql: resources?.graphql ?? null, + core: resources?.core ?? null, + } + } catch (err) { + return { + installation, + repoCount: null, + graphql: null, + core: null, + error: err instanceof Error ? err.message : String(err), + } + } +} + +function pad(value: string | number, width: number, alignRight = false): string { + const text = String(value) + return alignRight ? text.padStart(width) : text.padEnd(width) +} + +function renderTable(budgets: InstallationBudget[]): void { + const accountWidth = Math.max( + 7, + ...budgets.map((budget) => (budget.installation.accountLogin ?? 'unknown').length), + ) + const columns: [string | number, number, boolean][][] = budgets.map((budget) => [ + [budget.installation.id, 12, false], + [budget.installation.accountLogin ?? 'unknown', accountWidth, false], + [budget.installation.accountType ?? '-', 12, false], + [budget.installation.repositorySelection ?? '-', 10, false], + [budget.repoCount ?? '-', 7, true], + [budget.graphql?.limit ?? '-', 10, true], + [budget.graphql?.remaining ?? '-', 10, true], + [budget.core?.limit ?? '-', 11, true], + [budget.error ?? (budget.installation.suspendedAt ? 'suspended' : ''), 0, false], + ]) + + const header = [ + pad('installation', 12), + pad('account', accountWidth), + pad('type', 12), + pad('selection', 10), + pad('repos', 7, true), + pad('gql limit', 10, true), + pad('remaining', 10, true), + pad('core limit', 11, true), + 'note', + ].join(' ') + process.stdout.write(`${header}\n${'-'.repeat(header.length)}\n`) + for (const row of columns) { + process.stdout.write(`${row.map(([v, w, r]) => pad(v, w, r)).join(' ')}\n`) + } +} + +function renderTotals(budgets: InstallationBudget[]): void { + const usable = budgets.filter((budget) => budget.graphql && !budget.installation.suspendedAt) + const sum = (pick: (budget: InstallationBudget) => number | null | undefined): number => + usable.reduce((total, budget) => total + (pick(budget) ?? 0), 0) + + const graphqlCeiling = sum((budget) => budget.graphql?.limit) + const repos = sum((budget) => budget.repoCount) + const failed = budgets.filter((budget) => budget.error).length + + process.stdout.write( + `${[ + '', + `installations discovered ${budgets.length}`, + `installations usable ${usable.length}`, + `installations failed ${failed}`, + `repositories reachable ${repos}`, + '', + `GraphQL points / hour ${graphqlCeiling.toLocaleString()}`, + `GraphQL points available ${sum((budget) => budget.graphql?.remaining).toLocaleString()}`, + `REST core points / hour ${sum((budget) => budget.core?.limit).toLocaleString()}`, + `GraphQL points / day ${(graphqlCeiling * 24).toLocaleString()}`, + '', + ].join('\n')}\n`, + ) +} + +setImmediate(async () => { + try { + const { integrationId, concurrency, asJson, includeSuspended } = parseArgs( + process.argv.slice(2), + ) + + const db = await getDbConnection(WRITE_DB_CONFIG()) + const credential = await getCredential(pgpQx(db), integrationId) + + const discovered = await listInstallations(credential) + const installations = includeSuspended + ? discovered + : discovered.filter((installation) => !installation.suspendedAt) + log.info( + { discovered: discovered.length, auditing: installations.length, concurrency }, + 'auditing github app installations', + ) + + const budgets = await mapWithConcurrency(installations, concurrency, (installation) => + auditInstallation(credential, installation), + ) + budgets.sort((a, b) => (b.graphql?.limit ?? -1) - (a.graphql?.limit ?? -1)) + + if (asJson) { + process.stdout.write(`${JSON.stringify(budgets, null, 2)}\n`) + } else { + renderTable(budgets) + } + renderTotals(budgets) + + process.exit(0) + } catch (err) { + log.error(err, 'github app budget audit failed') + process.exit(1) + } +}) diff --git a/services/apps/connectors_worker/src/bin/seed-github-sync-units.ts b/services/apps/connectors_worker/src/bin/seed-github-sync-units.ts new file mode 100644 index 0000000000..04ff732d24 --- /dev/null +++ b/services/apps/connectors_worker/src/bin/seed-github-sync-units.ts @@ -0,0 +1,154 @@ +import { generateUUIDv4 } from '@crowd/common' +import type { Channel } from '@crowd/connectors' +import { getCredential } from '@crowd/connectors' +import { githubConnector } from '@crowd/connectors/src/connectors/github' +import { + mintInstallationToken, + resolveInstallationId, +} from '@crowd/connectors/src/connectors/github/appToken' +import { resolveRepoChannel } from '@crowd/connectors/src/connectors/github/discover' +import type { SyncUnitUpsert } from '@crowd/data-access-layer/src/connectors' +import { upsertSyncUnits } from '@crowd/data-access-layer/src/connectors' +import { WRITE_DB_CONFIG, getDbConnection } from '@crowd/data-access-layer/src/database' +import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' +import { pgpQx } from '@crowd/data-access-layer/src/queryExecutor' +import { upsertRepository } from '@crowd/data-access-layer/src/repositories' +import { getServiceLogger } from '@crowd/logging' + +const log = getServiceLogger() + +interface RepoRef { + owner: string + name: string +} + +function usage(): never { + log.error( + 'Usage: seed-github-sync-units --integration-id [--installation-id ] [more repos ...]', + ) + process.exit(1) +} + +function takeFlag(argv: string[], flag: string): string | undefined { + const flagIndex = argv.indexOf(flag) + if (flagIndex === -1) { + return undefined + } + const value = argv[flagIndex + 1] + argv.splice(flagIndex, 2) + return value +} + +function parseRepoArg(arg: string): RepoRef { + const path = arg.replace(/^https:\/\/github\.com\//, '').replace(/\/+$/, '') + const parts = path.split('/') + if (parts.length !== 2 || !parts[0] || !parts[1]) { + throw new Error(`invalid repo argument "${arg}" - expected owner/repo or a github repo URL`) + } + return { owner: parts[0], name: parts[1] } +} + +function parseArgs(rawArgv: string[]): { + integrationId: string + installationId?: string + repos: RepoRef[] +} { + const argv = rawArgv.filter((arg) => arg !== '--') + const integrationId = takeFlag(argv, '--integration-id') + const installationId = takeFlag(argv, '--installation-id') + if (!integrationId || argv.length === 0) { + usage() + } + return { integrationId, installationId, repos: argv.map(parseRepoArg) } +} + +async function getSegmentContext( + qx: QueryExecutor, + integrationId: string, +): Promise<{ segmentId: string; gitIntegrationId: string; insightsProjectId: string }> { + const integration: { segmentId: string | null } | null = await qx.selectOneOrNone( + `SELECT "segmentId" FROM integrations WHERE id = $(integrationId) AND "deletedAt" IS NULL`, + { integrationId }, + ) + if (!integration?.segmentId) { + throw new Error(`integration ${integrationId} not found or has no segmentId`) + } + + const gitIntegration: { id: string } | null = await qx.selectOneOrNone( + `SELECT id FROM integrations + WHERE "segmentId" = $(segmentId) AND platform = 'git' AND "deletedAt" IS NULL`, + { segmentId: integration.segmentId }, + ) + if (!gitIntegration) { + throw new Error(`git integration not found for segment ${integration.segmentId}`) + } + + const insightsProject: { id: string } | null = await qx.selectOneOrNone( + `SELECT id FROM "insightsProjects" + WHERE "segmentId" = $(segmentId) AND "deletedAt" IS NULL`, + { segmentId: integration.segmentId }, + ) + if (!insightsProject) { + throw new Error(`insights project not found for segment ${integration.segmentId}`) + } + + return { + segmentId: integration.segmentId, + gitIntegrationId: gitIntegration.id, + insightsProjectId: insightsProject.id, + } +} + +setImmediate(async () => { + try { + const args = parseArgs(process.argv.slice(2)) + const { integrationId, repos } = args + + const db = await getDbConnection(WRITE_DB_CONFIG()) + const qx = pgpQx(db) + + const credential = await getCredential(qx, integrationId) + const installationId = args.installationId ?? (await resolveInstallationId(credential)) + const { token, expiresAt } = await mintInstallationToken(credential, installationId) + log.info({ installationId, expiresAt }, 'github app auth verified, installation token minted') + + const segmentContext = await getSegmentContext(qx, integrationId) + + const channels: Channel[] = [] + for (const repo of repos) { + const channel = await resolveRepoChannel(token, repo.owner, repo.name) + const repositoryUpsert = await upsertRepository(qx, { + id: generateUUIDv4(), + url: channel.channelName, + segmentId: segmentContext.segmentId, + gitIntegrationId: segmentContext.gitIntegrationId, + sourceIntegrationId: integrationId, + insightsProjectId: segmentContext.insightsProjectId, + }) + log.info({ ...channel, repositoryUpsert }, 'repo resolved') + channels.push(channel) + } + + const syncNames = githubConnector.syncs.map((s) => s.name) + if (syncNames.length === 0) { + log.warn('github manifest has no syncs registered yet - seeded repositories only') + } + + const units: SyncUnitUpsert[] = channels.flatMap((channel) => + syncNames.map((syncName) => ({ + integrationId, + platform: githubConnector.platform, + channelId: channel.channelId, + channelName: channel.channelName, + syncName, + })), + ) + const unitsUpserted = await upsertSyncUnits(qx, units) + + log.info({ repos: channels.length, syncNames, unitsUpserted }, 'seeding complete') + process.exit(0) + } catch (err) { + log.error(err, 'seeding failed') + process.exit(1) + } +}) diff --git a/services/apps/connectors_worker/src/main.ts b/services/apps/connectors_worker/src/main.ts new file mode 100644 index 0000000000..4e0810eb7f --- /dev/null +++ b/services/apps/connectors_worker/src/main.ts @@ -0,0 +1,47 @@ +import { Config } from '@crowd/archetype-standard' +import { Options, ServiceWorker } from '@crowd/archetype-worker' +import { IS_DEV_ENV } from '@crowd/common' +import { registerConnector } from '@crowd/connectors' +import { githubConnector } from '@crowd/connectors/src/connectors/github' +import { dummyConnector } from '@crowd/connectors/src/testing/dummyConnector' + +import { scheduleDispatcher } from './schedules/dispatcher' + +const config: Config = { + envvars: [], + producer: { + enabled: true, + }, + temporal: { + enabled: true, + }, + redis: { + enabled: true, + }, +} + +const options: Options = { + postgres: { + enabled: true, + }, + opensearch: { + enabled: false, + }, +} + +export const svc = new ServiceWorker(config, options) + +registerConnector(githubConnector) + +// POC only: dummy connector drives the control-plane end-to-end in dev +if (IS_DEV_ENV) { + registerConnector(dummyConnector) +} + +setImmediate(async () => { + await svc.init() + + await scheduleDispatcher() + + await svc.start() +}) diff --git a/services/apps/connectors_worker/src/runLimits.ts b/services/apps/connectors_worker/src/runLimits.ts new file mode 100644 index 0000000000..65db1da714 --- /dev/null +++ b/services/apps/connectors_worker/src/runLimits.ts @@ -0,0 +1,5 @@ +const MINUTE_MS = 60_000 + +export const RUN_START_TO_CLOSE_TIMEOUT_MS = 60 * MINUTE_MS + +export const RUN_BUDGET_MS = RUN_START_TO_CLOSE_TIMEOUT_MS - 10 * MINUTE_MS diff --git a/services/apps/connectors_worker/src/schedules/dispatcher.ts b/services/apps/connectors_worker/src/schedules/dispatcher.ts new file mode 100644 index 0000000000..63ed04d218 --- /dev/null +++ b/services/apps/connectors_worker/src/schedules/dispatcher.ts @@ -0,0 +1,32 @@ +import { ScheduleAlreadyRunning, ScheduleOverlapPolicy } from '@temporalio/client' + +import { svc } from '../main' +import { dispatcher } from '../workflows/dispatcher' + +export async function scheduleDispatcher(): Promise { + try { + await svc.temporal.schedule.create({ + scheduleId: 'connectors-dispatcher', + spec: { + intervals: [{ every: '30s' }], + }, + policies: { + overlap: ScheduleOverlapPolicy.SKIP, + catchupWindow: '1 minute', + }, + action: { + type: 'startWorkflow', + workflowType: dispatcher, + taskQueue: 'connectors', + args: [], + workflowExecutionTimeout: '5 minutes', + }, + }) + } catch (err) { + if (err instanceof ScheduleAlreadyRunning) { + svc.log.info('Dispatcher schedule already registered in Temporal.') + } else { + throw new Error(err) + } + } +} diff --git a/services/apps/connectors_worker/src/scheduling.ts b/services/apps/connectors_worker/src/scheduling.ts new file mode 100644 index 0000000000..ee5c886e07 --- /dev/null +++ b/services/apps/connectors_worker/src/scheduling.ts @@ -0,0 +1,31 @@ +const JITTER_RATIO = 0.1 +const SHORT_DEFER_MIN_MS = 30_000 +const SHORT_DEFER_JITTER_MS = 60_000 +const FAILURE_BACKOFF_BASE_MS = 60_000 +const RUNNING_PROBE_CAP_MS = 5 * 60_000 +const FALLBACK_FAILURE_CAP_MS = 60 * 60_000 + +function withJitter(delayMs: number): number { + return delayMs + delayMs * JITTER_RATIO * (Math.random() * 2 - 1) +} + +export function cadenceRunAt(cadenceMinutes: number): Date { + return new Date(Date.now() + withJitter(cadenceMinutes * 60_000)) +} + +export function shortDeferRunAt(): Date { + return new Date(Date.now() + SHORT_DEFER_MIN_MS + Math.random() * SHORT_DEFER_JITTER_MS) +} + +export function runningProbeRunAt(startedAt: Date): Date { + const elapsedMs = Math.max(SHORT_DEFER_MIN_MS, Date.now() - startedAt.getTime()) + + return new Date(Date.now() + withJitter(Math.min(elapsedMs, RUNNING_PROBE_CAP_MS))) +} + +export function failureRunAt(consecutiveFailures: number, cadenceMinutes: number | null): Date { + const capMs = cadenceMinutes === null ? FALLBACK_FAILURE_CAP_MS : cadenceMinutes * 60_000 + const backoffMs = FAILURE_BACKOFF_BASE_MS * 2 ** (Math.max(1, consecutiveFailures) - 1) + + return new Date(Date.now() + withJitter(Math.min(backoffMs, capMs))) +} diff --git a/services/apps/connectors_worker/src/types.ts b/services/apps/connectors_worker/src/types.ts new file mode 100644 index 0000000000..f16800666c --- /dev/null +++ b/services/apps/connectors_worker/src/types.ts @@ -0,0 +1,18 @@ +import type { IClaimedUnit } from '@crowd/data-access-layer/src/connectors' + +export type StartRunResult = 'started' | 'alreadyRunning' + +export interface IAdmissionResult { + admitted: IClaimedUnit[] + deferred: IClaimedUnit[] +} + +export interface IDispatchCounts { + claimed: number + admitted: number + deferred: number + started: number + alreadyRunning: number + failed: number + durationMs: number +} diff --git a/services/apps/connectors_worker/src/workflows.ts b/services/apps/connectors_worker/src/workflows.ts new file mode 100644 index 0000000000..67cb17400b --- /dev/null +++ b/services/apps/connectors_worker/src/workflows.ts @@ -0,0 +1,4 @@ +import { dispatcher } from './workflows/dispatcher' +import { syncRun } from './workflows/syncRun' + +export { dispatcher, syncRun } diff --git a/services/apps/connectors_worker/src/workflows/dispatcher.ts b/services/apps/connectors_worker/src/workflows/dispatcher.ts new file mode 100644 index 0000000000..7cd25d05f2 --- /dev/null +++ b/services/apps/connectors_worker/src/workflows/dispatcher.ts @@ -0,0 +1,51 @@ +import { log, proxyActivities } from '@temporalio/workflow' + +import type * as activities from '../activities/dispatcherActivities' + +const activity = proxyActivities({ + startToCloseTimeout: '1 minute', + retry: { maximumAttempts: 3, backoffCoefficient: 2 }, +}) + +const CLAIM_LIMIT = 100 + +export async function dispatcher(): Promise { + const startedAt = Date.now() + + await activity.touchHeartbeat() + + const units = await activity.claimDue(CLAIM_LIMIT) + + const { admitted, deferred } = await activity.admitByBudget(units) + + let started = 0 + let alreadyRunning = 0 + let failed = 0 + for (const unit of admitted) { + try { + const result = await activity.startRun(unit) + if (result === 'started') { + started += 1 + } else { + alreadyRunning += 1 + } + } catch (err) { + failed += 1 + log.error('failed to dispatch sync unit', { unitId: unit.id, err }) + } + } + + for (const unit of deferred) { + await activity.deferUnit(unit.id) + } + + await activity.logDispatchSummary({ + claimed: units.length, + admitted: admitted.length, + deferred: deferred.length, + started, + alreadyRunning, + failed, + durationMs: Date.now() - startedAt, + }) +} diff --git a/services/apps/connectors_worker/src/workflows/syncRun.ts b/services/apps/connectors_worker/src/workflows/syncRun.ts new file mode 100644 index 0000000000..4132c68566 --- /dev/null +++ b/services/apps/connectors_worker/src/workflows/syncRun.ts @@ -0,0 +1,14 @@ +import { proxyActivities } from '@temporalio/workflow' + +import type * as activities from '../activities/syncRunActivities' +import { RUN_START_TO_CLOSE_TIMEOUT_MS } from '../runLimits' + +const activity = proxyActivities({ + startToCloseTimeout: RUN_START_TO_CLOSE_TIMEOUT_MS, + heartbeatTimeout: '1 minute', + retry: { maximumAttempts: 1 }, +}) + +export async function syncRun(unitId: string): Promise { + await activity.executeSync(unitId) +} diff --git a/services/apps/connectors_worker/tsconfig.json b/services/apps/connectors_worker/tsconfig.json new file mode 100644 index 0000000000..bf7f183850 --- /dev/null +++ b/services/apps/connectors_worker/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../base.tsconfig.json", + "include": ["src/**/*"] +} diff --git a/services/libs/connectors/package.json b/services/libs/connectors/package.json new file mode 100644 index 0000000000..55a9d42958 --- /dev/null +++ b/services/libs/connectors/package.json @@ -0,0 +1,28 @@ +{ + "name": "@crowd/connectors", + "private": true, + "main": "src/index.ts", + "scripts": { + "lint": "npx eslint --ext .ts src --max-warnings=0", + "format": "npx prettier --write \"src/**/*.ts\"", + "format-check": "npx prettier --check .", + "tsc-check": "tsc --noEmit" + }, + "devDependencies": { + "@types/jsonwebtoken": "^9.0.0", + "@types/node": "^24.13.3", + "typescript": "^5.6.3" + }, + "dependencies": { + "@crowd/common": "workspace:*", + "@crowd/common_services": "workspace:*", + "@crowd/data-access-layer": "workspace:*", + "@crowd/integrations": "workspace:*", + "@crowd/logging": "workspace:*", + "@crowd/redis": "workspace:*", + "@crowd/types": "workspace:*", + "axios": "^1.6.8", + "jsonwebtoken": "^9.0.0", + "zod": "^3.22.0" + } +} diff --git a/services/libs/connectors/src/concurrency.ts b/services/libs/connectors/src/concurrency.ts new file mode 100644 index 0000000000..dcafecaefb --- /dev/null +++ b/services/libs/connectors/src/concurrency.ts @@ -0,0 +1,16 @@ +export async function mapWithConcurrency( + items: T[], + limit: number, + fn: (item: T) => Promise, +): Promise { + const results = new Array(items.length) + let nextIndex = 0 + const workers = Array.from({ length: Math.min(limit, items.length) }, async () => { + while (nextIndex < items.length) { + const index = nextIndex++ + results[index] = await fn(items[index]) + } + }) + await Promise.all(workers) + return results +} diff --git a/services/libs/connectors/src/connectors/github/appToken.ts b/services/libs/connectors/src/connectors/github/appToken.ts new file mode 100644 index 0000000000..170deea13e --- /dev/null +++ b/services/libs/connectors/src/connectors/github/appToken.ts @@ -0,0 +1,185 @@ +import axios from 'axios' +import * as jwt from 'jsonwebtoken' + +import { + ConnectorError, + ProviderAuthError, + ProviderContractError, + ProviderUnavailableError, +} from '../../http/errors' +import type { TokenMinter, TokenPool } from '../../pool/tokenPool' +import type { Credential, PoolPreparation } from '../../types' + +const GITHUB_API_VERSION = '2022-11-28' + +export const GITHUB_REQUEST_TIMEOUT_MS = 30_000 + +function mintAppJwt(credential: Credential): string { + const now = Math.floor(Date.now() / 1000) + return jwt.sign( + { iat: now - 60, exp: now + 600, iss: credential.data.appId }, + credential.data.privateKey, + { algorithm: 'RS256' }, + ) +} + +function classifyAppApiError(err: unknown, operation: string): ConnectorError | null { + if (!axios.isAxiosError(err)) { + return null + } + const status = err.response?.status + const body = err.response?.data as { message?: string } | undefined + const detail = body?.message ?? err.message + const options = { status, cause: err } + if (status === 401) { + return new ProviderAuthError(`github app ${operation} unauthorized: ${detail}`, options) + } + if (status === undefined || status >= 500) { + return new ProviderUnavailableError(`github app ${operation} failed: ${detail}`, options) + } + return new ProviderContractError( + `github app ${operation} returned status ${status}: ${detail}`, + options, + ) +} + +export async function mintInstallationToken( + credential: Credential, + installationId: string, +): Promise<{ token: string; expiresAt: string }> { + try { + // POC only: minting cannot go through ConnectorHttp — it feeds the pool the client draws from + const response = await axios.post( + `https://api.github.com/app/installations/${installationId}/access_tokens`, + {}, + { + headers: { + Authorization: `Bearer ${mintAppJwt(credential)}`, + Accept: 'application/vnd.github+json', + 'X-GitHub-Api-Version': GITHUB_API_VERSION, + }, + timeout: GITHUB_REQUEST_TIMEOUT_MS, + }, + ) + return { token: response.data.token, expiresAt: response.data.expires_at } + } catch (err) { + throw classifyAppApiError(err, 'installation token minting') ?? err + } +} + +const INSTALLATIONS_PER_PAGE = 100 +const MAX_INSTALLATION_PAGES = 100 + +export interface InstallationSummary { + id: string + accountLogin: string | null + accountType: string | null + repositorySelection: string | null + suspendedAt: string | null +} + +interface InstallationPayload { + id: number + account: { login?: string; type?: string } | null + repository_selection?: string + suspended_at?: string | null +} + +function toInstallationSummary(installation: InstallationPayload): InstallationSummary { + return { + id: String(installation.id), + accountLogin: installation.account?.login ?? null, + accountType: installation.account?.type ?? null, + repositorySelection: installation.repository_selection ?? null, + suspendedAt: installation.suspended_at ?? null, + } +} + +export async function listInstallations(credential: Credential): Promise { + const appJwt = mintAppJwt(credential) + const summaries: InstallationSummary[] = [] + try { + for (let page = 1; page <= MAX_INSTALLATION_PAGES; page++) { + const response = await axios.get('https://api.github.com/app/installations', { + headers: { + Authorization: `Bearer ${appJwt}`, + Accept: 'application/vnd.github+json', + 'X-GitHub-Api-Version': GITHUB_API_VERSION, + }, + params: { per_page: INSTALLATIONS_PER_PAGE, page }, + timeout: GITHUB_REQUEST_TIMEOUT_MS, + }) + const installations = response.data as InstallationPayload[] + summaries.push(...installations.map(toInstallationSummary)) + if (installations.length < INSTALLATIONS_PER_PAGE) { + return summaries + } + } + } catch (err) { + throw classifyAppApiError(err, 'installation listing') ?? err + } + // truncating would silently shrink the pool, so refuse instead + throw new Error( + `github app has more than ${MAX_INSTALLATION_PAGES * INSTALLATIONS_PER_PAGE} installations`, + ) +} + +export async function listInstallationIds(credential: Credential): Promise { + const installations = await listInstallations(credential) + return installations.map((installation) => installation.id) +} + +// TODO(CM-1372): POC-only resolution; store the installation id per integration after the POC +export async function resolveInstallationId(credential: Credential): Promise { + const fromEnv = process.env.CROWD_GITHUB_INSTALLATION_ID + if (fromEnv) { + return fromEnv + } + + const installationIds = await listInstallationIds(credential) + if (installationIds.length === 0) { + throw new Error('github app has no installations') + } + return installationIds[0] +} + +const POOL_RESEED_INTERVAL_MS = 3_600_000 + +function poolInstallationAllowlist(): Set | null { + const raw = process.env.CROWD_GITHUB_POOL_INSTALLATION_IDS + if (!raw) { + return null + } + const ids = raw + .split(',') + .map((id) => id.trim()) + .filter(Boolean) + return ids.length > 0 ? new Set(ids) : null +} + +export async function prepareGithubPool( + credential: Credential, + pool: TokenPool, +): Promise { + const preferredEntryId = process.env.CROWD_GITHUB_INSTALLATION_ID + if (!(await pool.needsSeed(POOL_RESEED_INTERVAL_MS))) { + return { preferredEntryId } + } + const allowlist = poolInstallationAllowlist() + const installationIds = (await listInstallationIds(credential)).filter( + (id) => !allowlist || allowlist.has(id), + ) + if (installationIds.length === 0) { + throw new Error( + allowlist + ? 'none of CROWD_GITHUB_POOL_INSTALLATION_IDS match an installation of this github app' + : 'github app has no installations', + ) + } + await pool.seed(installationIds) + return { preferredEntryId: preferredEntryId ?? installationIds[0] } +} + +export function createGithubTokenMinter(credential: Credential): TokenMinter { + return (installationId) => mintInstallationToken(credential, installationId) +} diff --git a/services/libs/connectors/src/connectors/github/budget.ts b/services/libs/connectors/src/connectors/github/budget.ts new file mode 100644 index 0000000000..3a64e79ba1 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/budget.ts @@ -0,0 +1,34 @@ +import axios from 'axios' + +import type { BudgetProbe } from '../../pool/tokenPool' + +import { GITHUB_REQUEST_TIMEOUT_MS } from './appToken' + +interface RateLimitResource { + limit: number + remaining: number + reset: number +} + +export const probeGithubBudget: BudgetProbe = async (_platform, token) => { + try { + const response = await axios.get('https://api.github.com/rate_limit', { + headers: { + Authorization: `Bearer ${token.value}`, + Accept: 'application/vnd.github+json', + }, + timeout: GITHUB_REQUEST_TIMEOUT_MS, + }) + const graphql = response.data?.resources?.graphql as RateLimitResource | undefined + if (!graphql) { + return null + } + return { + limit: graphql.limit, + remaining: graphql.remaining, + resetAt: new Date(graphql.reset * 1000), + } + } catch { + return null + } +} diff --git a/services/libs/connectors/src/connectors/github/discover.ts b/services/libs/connectors/src/connectors/github/discover.ts new file mode 100644 index 0000000000..8b20b1e2f9 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/discover.ts @@ -0,0 +1,51 @@ +import axios from 'axios' + +import type { Channel, Credential } from '../../types' + +import { GITHUB_REQUEST_TIMEOUT_MS, mintInstallationToken, resolveInstallationId } from './appToken' + +const PER_PAGE = 100 + +interface InstallationRepo { + node_id: string + html_url: string +} + +export async function resolveRepoChannel( + token: string, + owner: string, + name: string, +): Promise { + const response = await axios.get(`https://api.github.com/repos/${owner}/${name}`, { + headers: { + Authorization: `Bearer ${token}`, + Accept: 'application/vnd.github+json', + }, + timeout: GITHUB_REQUEST_TIMEOUT_MS, + }) + return { channelId: response.data.node_id, channelName: response.data.html_url } +} + +export async function discoverRepos(credential: Credential): Promise { + const installationId = await resolveInstallationId(credential) + const { token } = await mintInstallationToken(credential, installationId) + + const channels: Channel[] = [] + let page = 1 + for (;;) { + const response = await axios.get('https://api.github.com/installation/repositories', { + params: { per_page: PER_PAGE, page }, + headers: { + Authorization: `Bearer ${token}`, + Accept: 'application/vnd.github+json', + }, + timeout: GITHUB_REQUEST_TIMEOUT_MS, + }) + const repos = (response.data?.repositories ?? []) as InstallationRepo[] + channels.push(...repos.map((r) => ({ channelId: r.node_id, channelName: r.html_url }))) + if (repos.length < PER_PAGE) { + return channels + } + page += 1 + } +} diff --git a/services/libs/connectors/src/connectors/github/gql.ts b/services/libs/connectors/src/connectors/github/gql.ts new file mode 100644 index 0000000000..baa4d70856 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/gql.ts @@ -0,0 +1,27 @@ +import type { ConnectorHttp } from '../../http/client' +import { ProviderContractError } from '../../http/errors' + +interface GraphqlEnvelope { + data?: T + errors?: { type?: string; message?: string }[] +} + +export async function githubGraphql( + http: ConnectorHttp, + query: string, + variables: Record, +): Promise { + const body = await http.request>({ + method: 'post', + url: 'https://api.github.com/graphql', + data: { query, variables }, + }) + if (body.errors?.length) { + const details = body.errors.map((e) => `${e.type ?? 'ERROR'}: ${e.message ?? ''}`).join('; ') + throw new ProviderContractError(`github graphql errors: ${details}`) + } + if (!body.data) { + throw new ProviderContractError('github graphql response has no data') + } + return body.data +} diff --git a/services/libs/connectors/src/connectors/github/graphql/discussions.ts b/services/libs/connectors/src/connectors/github/graphql/discussions.ts new file mode 100644 index 0000000000..2dd6d12a87 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/graphql/discussions.ts @@ -0,0 +1,185 @@ +import type { GithubUserNode } from '../mappers/member' + +import { BOT_FIELDS, ORGANIZATION_FIELDS, USER_FIELDS } from './fields' + +export interface DiscussionCategory { + id: string + isAnswerable: boolean + name: string + slug: string + emoji: string + description: string | null +} + +export interface DiscussionNode { + id: string + number: number + title: string + bodyText: string + url: string + createdAt: string + updatedAt: string + isAnswered: boolean | null + author: GithubUserNode | null + category: DiscussionCategory + comments: { totalCount: number } +} + +export interface DiscussionsPage { + repository: { + discussions: { + pageInfo: { + endCursor: string | null + hasNextPage: boolean + } + nodes: (DiscussionNode | null)[] + } + } +} + +export interface DiscussionReplyNode { + id: string + bodyText: string + url: string + createdAt: string +} + +export interface DiscussionCommentNode extends DiscussionReplyNode { + replyTo: { id: string } | null + replies: RepliesConnection +} + +export interface RepliesConnection { + pageInfo: { + endCursor: string | null + hasNextPage: boolean + } + nodes: (DiscussionReplyNode | null)[] +} + +export interface DiscussionCommentsPage { + node: { + comments: { + pageInfo: { + endCursor: string | null + hasNextPage: boolean + } + nodes: (DiscussionCommentNode | null)[] + } + } | null +} + +export interface CommentRepliesPage { + node: { + replies: RepliesConnection + } | null +} + +const AUTHOR_FIELDS = ` + author { + login + ... on User { + ${USER_FIELDS} + } + ... on Bot { + ${BOT_FIELDS} + } + ... on Organization { + ${ORGANIZATION_FIELDS} + } + } +` + +const REPLY_FIELDS = ` + id + bodyText + url + createdAt +` + +export const DISCUSSIONS_QUERY = ` + query ($owner: String!, $repo: String!, $first: Int!, $cursor: String) { + repository(owner: $owner, name: $repo) { + discussions( + first: $first + after: $cursor + orderBy: {field: UPDATED_AT, direction: DESC} + ) { + pageInfo { + endCursor + hasNextPage + } + nodes { + id + number + title + bodyText + url + createdAt + updatedAt + isAnswered + ${AUTHOR_FIELDS} + category { + id + isAnswerable + name + slug + emoji + description + } + comments { + totalCount + } + } + } + } + } +` + +export const DISCUSSION_COMMENTS_QUERY = ` + query ($id: ID!, $first: Int!, $cursor: String, $repliesFirst: Int!) { + node(id: $id) { + ... on Discussion { + comments(first: $first, after: $cursor) { + pageInfo { + endCursor + hasNextPage + } + nodes { + ${REPLY_FIELDS} + replyTo { + id + } + replies(first: $repliesFirst) { + pageInfo { + endCursor + hasNextPage + } + nodes { + ${REPLY_FIELDS} + } + } + } + } + } + } + } +` + +export const COMMENT_REPLIES_QUERY = ` + query ($id: ID!, $first: Int!, $cursor: String) { + node(id: $id) { + ... on DiscussionComment { + replies(first: $first, after: $cursor) { + pageInfo { + endCursor + hasNextPage + } + nodes { + ${REPLY_FIELDS} + } + } + } + } + } +` diff --git a/services/libs/connectors/src/connectors/github/graphql/fields.ts b/services/libs/connectors/src/connectors/github/graphql/fields.ts new file mode 100644 index 0000000000..6ee626154c --- /dev/null +++ b/services/libs/connectors/src/connectors/github/graphql/fields.ts @@ -0,0 +1,48 @@ +export const USER_FIELDS = ` + __typename + login + name + avatarUrl + isHireable + url + bio + company + location + email + websiteUrl + databaseId + organizations(first: 5) { + nodes { + __typename + databaseId + login + name + url + websiteUrl + description + avatarUrl + twitterUsername + location + } + } +` + +export const BOT_FIELDS = ` + __typename + login + avatarUrl + url + databaseId +` + +export const ORGANIZATION_FIELDS = ` + __typename + avatarUrl + databaseId + email + location + login + name + url + websiteUrl +` diff --git a/services/libs/connectors/src/connectors/github/graphql/forks.ts b/services/libs/connectors/src/connectors/github/graphql/forks.ts new file mode 100644 index 0000000000..5c700a5465 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/graphql/forks.ts @@ -0,0 +1,65 @@ +import type { GithubUserNode } from '../mappers/member' + +import { ORGANIZATION_FIELDS, USER_FIELDS } from './fields' + +export interface ForkNode { + id: string + name: string + createdAt: string + updatedAt: string + url: string + isFork: boolean + isInOrganization: boolean + parent: { + nameWithOwner: string + isFork: boolean + } | null + owner: GithubUserNode +} + +export interface ForksPage { + repository: { + forks: { + pageInfo: { + endCursor: string | null + hasNextPage: boolean + } + nodes: (ForkNode | null)[] + } + } +} + +export const FORKS_QUERY = ` + query ($owner: String!, $repo: String!, $first: Int!, $cursor: String) { + repository(owner: $owner, name: $repo) { + forks(first: $first, after: $cursor, orderBy: {field: CREATED_AT, direction: ASC}) { + pageInfo { + endCursor + hasNextPage + } + nodes { + id + name + createdAt + updatedAt + url + isFork + isInOrganization + parent { + nameWithOwner + isFork + } + owner { + login + ... on User { + ${USER_FIELDS} + } + ... on Organization { + ${ORGANIZATION_FIELDS} + } + } + } + } + } + } +` diff --git a/services/libs/connectors/src/connectors/github/graphql/issues.ts b/services/libs/connectors/src/connectors/github/graphql/issues.ts new file mode 100644 index 0000000000..087516979f --- /dev/null +++ b/services/libs/connectors/src/connectors/github/graphql/issues.ts @@ -0,0 +1,183 @@ +import type { GithubUserNode } from '../mappers/member' + +import { BOT_FIELDS, ORGANIZATION_FIELDS, USER_FIELDS } from './fields' + +export interface IssueTimelineNode { + __typename: string + actor?: GithubUserNode | null + createdAt?: string +} + +export interface IssueNode { + id: string + number: number + title: string + url: string + state: string + createdAt: string + updatedAt: string + bodyText: string + author: GithubUserNode | null + timelineItems: { + nodes: (IssueTimelineNode | null)[] + } +} + +export interface IssuesPage { + repository: { + issues: { + pageInfo: { + endCursor: string | null + hasNextPage: boolean + } + nodes: (IssueNode | null)[] + } + } +} + +export interface IssueCommentNode { + id: string + bodyText: string + url: string + createdAt: string + author: GithubUserNode | null +} + +export interface IssueCommentsConnection { + pageInfo: { + endCursor: string | null + hasNextPage: boolean + } + nodes: (IssueCommentNode | null)[] +} + +export interface IssueCommentsBatchPage { + nodes: ({ + id: string + number: number + comments: IssueCommentsConnection + } | null)[] +} + +export interface IssueCommentsPaginatedPage { + repository: { + issue: { + comments: IssueCommentsConnection + } | null + } +} + +const COMMENT_FIELDS = ` + id + bodyText + url + createdAt + author { + login + ... on User { + ${USER_FIELDS} + } + ... on Bot { + ${BOT_FIELDS} + } + ... on Organization { + ${ORGANIZATION_FIELDS} + } + } +` + +export const ISSUE_COMMENTS_QUERY = ` + query ($ids: [ID!]!, $first: Int!) { + nodes(ids: $ids) { + ... on Issue { + id + number + comments(first: $first) { + pageInfo { + endCursor + hasNextPage + } + nodes { + ${COMMENT_FIELDS} + } + } + } + } + } +` + +export const ISSUE_COMMENTS_PAGINATED_QUERY = ` + query ($owner: String!, $repo: String!, $issueNumber: Int!, $first: Int!, $cursor: String) { + repository(owner: $owner, name: $repo) { + issue(number: $issueNumber) { + comments(first: $first, after: $cursor) { + pageInfo { + endCursor + hasNextPage + } + nodes { + ${COMMENT_FIELDS} + } + } + } + } + } +` + +export const ISSUES_QUERY = ` + query ($owner: String!, $repo: String!, $first: Int!, $cursor: String, $since: DateTime) { + repository(owner: $owner, name: $repo) { + issues( + first: $first + after: $cursor + orderBy: {field: UPDATED_AT, direction: ASC} + states: [OPEN, CLOSED] + filterBy: {since: $since} + ) { + pageInfo { + endCursor + hasNextPage + } + nodes { + id + number + title + url + state + createdAt + updatedAt + bodyText + author { + login + ... on User { + ${USER_FIELDS} + } + ... on Bot { + ${BOT_FIELDS} + } + ... on Organization { + ${ORGANIZATION_FIELDS} + } + } + timelineItems(first: 10, itemTypes: [CLOSED_EVENT]) { + nodes { + __typename + ... on ClosedEvent { + createdAt + actor { + login + ... on User { + ${USER_FIELDS} + } + ... on Bot { + ${BOT_FIELDS} + } + } + } + } + } + } + } + } + } +` diff --git a/services/libs/connectors/src/connectors/github/graphql/pullRequestChildren.ts b/services/libs/connectors/src/connectors/github/graphql/pullRequestChildren.ts new file mode 100644 index 0000000000..8085e3f88a --- /dev/null +++ b/services/libs/connectors/src/connectors/github/graphql/pullRequestChildren.ts @@ -0,0 +1,237 @@ +import type { GithubUserNode } from '../mappers/member' + +import { BOT_FIELDS, ORGANIZATION_FIELDS, USER_FIELDS } from './fields' + +export interface PrCommentNode { + id: string + body: string + createdAt: string + url: string + author: GithubUserNode | null +} + +export interface PrCommentsConnection { + pageInfo: { + endCursor: string | null + hasNextPage: boolean + } + edges: ({ node: PrCommentNode | null } | null)[] +} + +export interface PrCommentsBatchPage { + nodes: ({ + id: string + number: number + comments: PrCommentsConnection + } | null)[] +} + +export interface ReviewThreadNode { + id: string + isResolved: boolean +} + +export interface ReviewThreadsBatchPage { + nodes: ({ + id: string + number: number + reviewThreads: { + pageInfo: { + endCursor: string | null + hasNextPage: boolean + } + edges: ({ node: ReviewThreadNode | null } | null)[] + } + } | null)[] +} + +export interface ThreadCommentNode { + id: string + body: string + createdAt: string + url: string + author: GithubUserNode | null +} + +export interface ThreadCommentsBatchPage { + nodes: ({ + id: string + isResolved: boolean + comments: { + pageInfo: { + endCursor: string | null + hasNextPage: boolean + } + edges: ({ node: ThreadCommentNode | null } | null)[] + } + } | null)[] +} + +export interface PrCommitNode { + commit: { + additions: number + deletions: number + parents: { totalCount: number } + id: string + oid: string + message: string + authoredDate: string + url: string + author: { + user: GithubUserNode | null + email: string | null + name: string | null + } | null + } +} + +export interface PrCommitsPage { + repository: { + pullRequest: { + commits: { + pageInfo: { + endCursor: string | null + hasNextPage: boolean + } + nodes: (PrCommitNode | null)[] + } + } | null + } +} + +export const COMMENTS_FOR_PRS_QUERY = ` + query ($ids: [ID!]!, $first: Int!, $after: String) { + nodes(ids: $ids) { + ... on PullRequest { + id + number + comments(first: $first, after: $after) { + pageInfo { + endCursor + hasNextPage + } + edges { + node { + id + body + createdAt + url + author { + __typename + login + ... on User { + ${USER_FIELDS} + } + ... on Bot { + ${BOT_FIELDS} + } + ... on Organization { + ${ORGANIZATION_FIELDS} + } + } + } + } + } + } + } + } +` + +export const REVIEW_THREADS_FOR_PRS_QUERY = ` + query ($ids: [ID!]!, $first: Int!, $after: String) { + nodes(ids: $ids) { + ... on PullRequest { + id + number + reviewThreads(first: $first, after: $after) { + pageInfo { + endCursor + hasNextPage + } + edges { + node { + id + isResolved + } + } + } + } + } + } +` + +export const COMMENTS_FOR_THREADS_QUERY = ` + query ($ids: [ID!]!, $first: Int!, $after: String) { + nodes(ids: $ids) { + ... on PullRequestReviewThread { + id + isResolved + comments(first: $first, after: $after) { + pageInfo { + endCursor + hasNextPage + } + edges { + node { + id + body + createdAt + url + author { + __typename + login + ... on User { + ${USER_FIELDS} + } + ... on Bot { + ${BOT_FIELDS} + } + ... on Organization { + ${ORGANIZATION_FIELDS} + } + } + } + } + } + } + } + } +` + +export const PR_COMMITS_QUERY = ` + query ($owner: String!, $repo: String!, $prNumber: Int!, $first: Int!, $cursor: String) { + repository(name: $repo, owner: $owner) { + pullRequest(number: $prNumber) { + commits(first: $first, after: $cursor) { + pageInfo { + endCursor + hasNextPage + } + nodes { + commit { + additions + deletions + parents { + totalCount + } + id + oid + message + authoredDate + url + author { + user { + ... on User { + ${USER_FIELDS} + } + } + email + name + } + } + } + } + } + } + } +` diff --git a/services/libs/connectors/src/connectors/github/graphql/pullRequests.ts b/services/libs/connectors/src/connectors/github/graphql/pullRequests.ts new file mode 100644 index 0000000000..5428c89f34 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/graphql/pullRequests.ts @@ -0,0 +1,213 @@ +import type { GithubUserNode } from '../mappers/member' + +import { BOT_FIELDS, ORGANIZATION_FIELDS, USER_FIELDS } from './fields' + +export interface PullRequestNode { + id: string + number: number + createdAt: string + updatedAt: string + url: string + title: string + body: string + state: string + authorAssociation: string + labels: { nodes: { name: string }[] | null } | null + additions: number + deletions: number + changedFiles: number + author: GithubUserNode | null +} + +export interface PullRequestsPage { + repository: { + pullRequests: { + pageInfo: { + endCursor: string | null + hasNextPage: boolean + } + nodes: (PullRequestNode | null)[] + } + } +} + +export interface PrTimelineItem { + __typename: string + id?: string + createdAt?: string + submittedAt?: string + state?: string + body?: string + actor?: GithubUserNode | null + author?: GithubUserNode | null + assignee?: GithubUserNode | null + requestedReviewer?: GithubUserNode | null +} + +export interface PrTimelineBatchPage { + nodes: ({ + id: string + timelineItems: { + nodes: (PrTimelineItem | null)[] + } + } | null)[] +} + +const ACTOR_FIELDS = ` + __typename + login + ... on User { + ${USER_FIELDS} + } + ... on Bot { + ${BOT_FIELDS} + } +` + +export const PULL_REQUESTS_QUERY = ` + query ($owner: String!, $repo: String!, $first: Int!, $cursor: String, $direction: OrderDirection!) { + repository(owner: $owner, name: $repo) { + pullRequests(first: $first, after: $cursor, orderBy: {field: UPDATED_AT, direction: $direction}) { + pageInfo { + endCursor + hasNextPage + } + nodes { + id + number + createdAt + updatedAt + url + title + body + state + authorAssociation + labels(first: $first) { + nodes { + name + } + } + additions + deletions + changedFiles + author { + __typename + login + ... on User { + ${USER_FIELDS} + } + ... on Bot { + ${BOT_FIELDS} + } + ... on Organization { + ${ORGANIZATION_FIELDS} + } + } + } + } + } + } +` + +export interface PrTimelinePage { + nodes: ({ + id: string + timelineItems: { + pageInfo: { + endCursor: string | null + hasNextPage: boolean + } + nodes: (PrTimelineItem | null)[] + } + } | null)[] +} + +export const PR_TIMELINE_QUERY = ` + query ($ids: [ID!]!, $first: Int!, $after: String) { + nodes(ids: $ids) { + ... on PullRequest { + id + timelineItems( + first: $first + after: $after + itemTypes: [PULL_REQUEST_REVIEW, MERGED_EVENT, ASSIGNED_EVENT, REVIEW_REQUESTED_EVENT, CLOSED_EVENT] + ) { + pageInfo { + endCursor + hasNextPage + } + nodes { + __typename + ... on ReviewRequestedEvent { + id + createdAt + actor { + ${ACTOR_FIELDS} + } + requestedReviewer { + __typename + ... on User { + login + ${USER_FIELDS} + } + ... on Bot { + login + ${BOT_FIELDS} + } + ... on Team { + name + id + } + } + } + ... on PullRequestReview { + id + state + submittedAt + body + author { + ${ACTOR_FIELDS} + } + } + ... on AssignedEvent { + id + createdAt + assignee { + __typename + ... on User { + login + ${USER_FIELDS} + } + ... on Bot { + login + ${BOT_FIELDS} + } + ... on Organization { + login + ${ORGANIZATION_FIELDS} + } + } + actor { + ${ACTOR_FIELDS} + } + } + ... on MergedEvent { + id + createdAt + actor { + ${ACTOR_FIELDS} + } + } + ... on ClosedEvent { + id + createdAt + actor { + ${ACTOR_FIELDS} + } + } + } + } + } + } + } +` diff --git a/services/libs/connectors/src/connectors/github/graphql/stars.ts b/services/libs/connectors/src/connectors/github/graphql/stars.ts new file mode 100644 index 0000000000..fd0c2f0e56 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/graphql/stars.ts @@ -0,0 +1,40 @@ +import type { GithubUserNode } from '../mappers/member' + +import { USER_FIELDS } from './fields' + +export interface StargazerEdge { + starredAt: string + node: GithubUserNode | null +} + +export interface StargazersPage { + repository: { + stargazers: { + pageInfo: { + endCursor: string | null + hasNextPage: boolean + } + edges: (StargazerEdge | null)[] + } + } +} + +export const STARGAZERS_QUERY = ` + query ($owner: String!, $repo: String!, $first: Int!, $cursor: String) { + repository(owner: $owner, name: $repo) { + stargazers(first: $first, after: $cursor, orderBy: {field: STARRED_AT, direction: ASC}) { + pageInfo { + endCursor + hasNextPage + } + edges { + starredAt + node { + login + ${USER_FIELDS} + } + } + } + } + } +` diff --git a/services/libs/connectors/src/connectors/github/index.ts b/services/libs/connectors/src/connectors/github/index.ts new file mode 100644 index 0000000000..062d853678 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/index.ts @@ -0,0 +1,35 @@ +import type { Manifest } from '../../types' + +import { createGithubTokenMinter, prepareGithubPool } from './appToken' +import { probeGithubBudget } from './budget' +import { discoverRepos } from './discover' +import { interpretGithubResponse } from './interpret' +import { discussionsSync } from './syncs/discussions' +import { forksSync } from './syncs/forks' +import { issueCommentsSync } from './syncs/issueComments' +import { issuesSync } from './syncs/issues' +import { pullRequestCommentsSync } from './syncs/pullRequestComments' +import { pullRequestCommitsSync } from './syncs/pullRequestCommits' +import { pullRequestReviewCommentsSync } from './syncs/pullRequestReviewComments' +import { pullRequestsSync } from './syncs/pullRequests' +// import { starsSync } from './syncs/stars' + +export const githubConnector: Manifest = { + platform: 'github', + syncs: [ + discussionsSync, + forksSync, + issuesSync, + issueCommentsSync, + pullRequestsSync, + pullRequestCommentsSync, + pullRequestReviewCommentsSync, + pullRequestCommitsSync, + // starsSync, + ], + discover: discoverRepos, + preparePool: prepareGithubPool, + mintToken: createGithubTokenMinter, + probeBudget: probeGithubBudget, + interpretResponse: interpretGithubResponse, +} diff --git a/services/libs/connectors/src/connectors/github/interpret.ts b/services/libs/connectors/src/connectors/github/interpret.ts new file mode 100644 index 0000000000..143ad942d4 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/interpret.ts @@ -0,0 +1,46 @@ +import type { ResponseInterpreter } from '../../http/client' +import { ProviderAuthError, ProviderContractError, RateLimitError } from '../../http/errors' + +interface GithubErrorBody { + errors?: { type?: string }[] + message?: string +} + +// Legacy prod parity (baseQuery.ts): GitHub's "temporarily blocked" 403 is retryable +const RETRYABLE_403_MESSAGES = [ + 'secondary rate limit', + 'although you appear to have the correct authorization credentials', +] + +function hasRateLimitHeaders(headers: Record): boolean { + return headers['retry-after'] !== undefined || headers['x-ratelimit-remaining'] === '0' +} + +export const interpretGithubResponse: ResponseInterpreter = (response) => { + const body = response.data as GithubErrorBody | null + if (response.status === 200) { + // GitHub docs say RATE_LIMITED, but installation tokens return RATE_LIMIT + if (body?.errors?.some((e) => e.type?.includes('RATE_LIMIT'))) { + return new RateLimitError('github graphql rate limited') + } + return null + } + if (response.status === 401) { + return new ProviderAuthError(`github unauthorized: ${body?.message ?? 'no message'}`, { + status: 401, + }) + } + if (response.status === 403) { + const message = body?.message?.toLowerCase() ?? '' + if (RETRYABLE_403_MESSAGES.some((m) => message.includes(m))) { + return new RateLimitError(`github rate limited: ${body?.message}`) + } + if (hasRateLimitHeaders(response.headers)) { + return null + } + return new ProviderContractError(`github forbidden: ${body?.message ?? 'no message'}`, { + status: 403, + }) + } + return null +} diff --git a/services/libs/connectors/src/connectors/github/mappers/commit.ts b/services/libs/connectors/src/connectors/github/mappers/commit.ts new file mode 100644 index 0000000000..188e3316ee --- /dev/null +++ b/services/libs/connectors/src/connectors/github/mappers/commit.ts @@ -0,0 +1,28 @@ +import { GITHUB_GRID, GithubActivityType } from '@crowd/integrations' + +import type { PrCommitNode } from '../graphql/pullRequestChildren' +import type { GithubActivity } from '../schemas' + +import { toMember } from './member' + +const DEFAULT_TIMESTAMP = '1970-01-01T00:00:00Z' + +export function toCommit(commit: PrCommitNode['commit'], prId: string): GithubActivity { + return { + type: GithubActivityType.AUTHORED_COMMIT, + timestamp: commit.authoredDate ?? DEFAULT_TIMESTAMP, + sourceId: commit.oid, + sourceParentId: prId, + score: GITHUB_GRID[GithubActivityType.AUTHORED_COMMIT].score, + body: commit.message, + url: commit.url ?? '', + attributes: { + insertions: commit.additions, + deletions: commit.deletions, + // nango quirk kept for exact-match: lines = additions - deletions, not the sum + lines: commit.additions - commit.deletions, + isMerge: commit.parents.totalCount > 1, + }, + member: toMember(commit.author?.user), + } +} diff --git a/services/libs/connectors/src/connectors/github/mappers/discussion.ts b/services/libs/connectors/src/connectors/github/mappers/discussion.ts new file mode 100644 index 0000000000..f23b6889c4 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/mappers/discussion.ts @@ -0,0 +1,52 @@ +import { GITHUB_GRID, GithubActivityType } from '@crowd/integrations' + +import type { DiscussionNode, DiscussionReplyNode } from '../graphql/discussions' +import type { GithubActivity } from '../schemas' + +import { toMember } from './member' + +const DEFAULT_TIMESTAMP = '1970-01-01T00:00:00Z' + +export function toDiscussionStartedActivity(discussion: DiscussionNode): GithubActivity { + return { + type: GithubActivityType.DISCUSSION_STARTED, + timestamp: discussion.createdAt ?? DEFAULT_TIMESTAMP, + sourceId: discussion.id, + score: GITHUB_GRID[GithubActivityType.DISCUSSION_STARTED].score, + title: discussion.title || '', + body: discussion.bodyText || '', + url: discussion.url, + attributes: { + category: { + id: String(discussion.category.id), + isAnswerable: String(discussion.category.isAnswerable), + name: discussion.category.name, + slug: discussion.category.slug, + emoji: discussion.category.emoji, + description: discussion.category.description, + }, + }, + member: toMember(discussion.author), + } +} + +export function toDiscussionCommentActivity( + discussion: DiscussionNode, + comment: DiscussionReplyNode, + isReply: boolean, +): GithubActivity { + return { + type: GithubActivityType.DISCUSSION_COMMENT, + timestamp: comment.createdAt ?? DEFAULT_TIMESTAMP, + sourceId: comment.id, + sourceParentId: discussion.id, + score: + discussion.isAnswered && !isReply + ? GITHUB_GRID[GithubActivityType.DISCUSSION_COMMENT].score + 2 + : GITHUB_GRID[GithubActivityType.DISCUSSION_COMMENT].score, + body: comment.bodyText || '', + url: comment.url, + attributes: { isAnswer: discussion.isAnswered ?? false }, + member: toMember(discussion.author), + } +} diff --git a/services/libs/connectors/src/connectors/github/mappers/fork.ts b/services/libs/connectors/src/connectors/github/mappers/fork.ts new file mode 100644 index 0000000000..8bd9dd1760 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/mappers/fork.ts @@ -0,0 +1,23 @@ +import { GITHUB_GRID, GithubActivityType } from '@crowd/integrations' + +import type { ForkNode } from '../graphql/forks' +import type { GithubActivity } from '../schemas' + +import { toMember } from './member' + +export function toFork(node: ForkNode): GithubActivity { + return { + type: GithubActivityType.FORK, + timestamp: node.createdAt, + sourceId: node.id, + score: GITHUB_GRID[GithubActivityType.FORK].score, + attributes: node.parent?.isFork + ? { + isForkByOrg: node.isInOrganization, + directParent: node.parent.nameWithOwner, + isIndirectFork: String(node.parent.isFork), + } + : {}, + member: toMember(node.owner), + } +} diff --git a/services/libs/connectors/src/connectors/github/mappers/issue.ts b/services/libs/connectors/src/connectors/github/mappers/issue.ts new file mode 100644 index 0000000000..9fc7182705 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/mappers/issue.ts @@ -0,0 +1,49 @@ +import { GITHUB_GRID, GithubActivityType } from '@crowd/integrations' + +import type { IssueNode } from '../graphql/issues' +import type { GithubActivity } from '../schemas' + +import { toMember } from './member' + +const DEFAULT_TIMESTAMP = '1970-01-01T00:00:00Z' + +export function toIssueActivities(issue: IssueNode): GithubActivity[] { + const activities: GithubActivity[] = [ + { + type: GithubActivityType.ISSUE_OPENED, + timestamp: issue.createdAt ?? DEFAULT_TIMESTAMP, + sourceId: issue.id, + score: GITHUB_GRID[GithubActivityType.ISSUE_OPENED].score, + title: issue.title || '', + body: issue.bodyText || '', + url: issue.url, + attributes: { state: 'open', issueNumber: issue.number }, + member: toMember(issue.author), + }, + ] + + const closedEvent = issue.timelineItems.nodes.find((node) => node?.__typename === 'ClosedEvent') + if (issue.state === 'CLOSED' && closedEvent) { + const closedAt = closedEvent.createdAt ?? issue.updatedAt ?? DEFAULT_TIMESTAMP + activities.push({ + type: GithubActivityType.ISSUE_CLOSED, + timestamp: closedAt, + sourceId: `gen-CE_${issue.id}_${closedEvent.actor?.login ?? issue.author?.login}_${new Date( + closedAt, + ).toISOString()}`, + sourceParentId: issue.id, + score: GITHUB_GRID[GithubActivityType.ISSUE_CLOSED].score, + title: issue.title || '', + body: issue.bodyText || '', + url: issue.url, + attributes: { + state: 'closed', + issueNumber: issue.number, + ...(closedEvent.actor ? { closedBy: closedEvent.actor.login } : {}), + }, + member: toMember(closedEvent.actor ?? issue.author), + }) + } + + return activities +} diff --git a/services/libs/connectors/src/connectors/github/mappers/issueComment.ts b/services/libs/connectors/src/connectors/github/mappers/issueComment.ts new file mode 100644 index 0000000000..5919779a0e --- /dev/null +++ b/services/libs/connectors/src/connectors/github/mappers/issueComment.ts @@ -0,0 +1,21 @@ +import { GITHUB_GRID, GithubActivityType } from '@crowd/integrations' + +import type { IssueCommentNode, IssueNode } from '../graphql/issues' +import type { GithubActivity } from '../schemas' + +import { toMember } from './member' + +export function toIssueComment(comment: IssueCommentNode, issue: IssueNode): GithubActivity { + return { + type: GithubActivityType.ISSUE_COMMENT, + timestamp: comment.createdAt, + sourceId: comment.id, + sourceParentId: issue.id, + score: GITHUB_GRID[GithubActivityType.ISSUE_COMMENT].score, + title: issue.title || '', + body: comment.bodyText || '', + url: comment.url || issue.url, + attributes: { state: issue.state.toLowerCase() }, + member: toMember(comment.author), + } +} diff --git a/services/libs/connectors/src/connectors/github/mappers/member.ts b/services/libs/connectors/src/connectors/github/mappers/member.ts new file mode 100644 index 0000000000..8546ce7299 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/mappers/member.ts @@ -0,0 +1,158 @@ +import type { GithubMember, GithubOrganization } from '../schemas' + +export interface GithubOrgNode { + databaseId?: number | null + login: string + name?: string | null + url?: string | null + websiteUrl?: string | null + description?: string | null + avatarUrl?: string | null + twitterUsername?: string | null + location?: string | null +} + +export interface GithubUserNode { + __typename?: string + login?: string | null + name?: string | null + avatarUrl?: string | null + isHireable?: boolean | null + url?: string | null + bio?: string | null + company?: string | null + location?: string | null + email?: string | null + websiteUrl?: string | null + databaseId?: number | null + id?: string | number | null + organizations?: { nodes?: (GithubOrgNode | null)[] | null } | null +} + +// https://github.com/ghost +const GHOST_MEMBER: GithubMember = { + displayName: 'ghost', + identities: [ + { + platform: 'github', + type: 'username', + verified: true, + value: 'ghost', + sourceId: '10137', + }, + ], + attributes: { + url: { github: 'https://github.com/ghost' }, + avatarUrl: { github: 'https://avatars.githubusercontent.com/u/10137?v=4' }, + bio: { + github: + "Hi, I'm @ghost! I take the place of user accounts that have been deleted.\n:ghost:\n", + }, + }, +} + +function toAttributes(user: GithubUserNode): GithubMember['attributes'] { + if (user.__typename === 'Bot') { + return { + url: { github: user.url ?? '' }, + avatarUrl: { github: user.avatarUrl ?? '' }, + sourceId: { github: user.id?.toString() ?? '' }, + isBot: { github: true }, + } + } + + const attributes: GithubMember['attributes'] = { + isHireable: { github: user.isHireable ?? false }, + url: { github: user.url ?? `https://github.com/${user.login ?? ''}` }, + bio: { github: user.bio ?? '' }, + location: { github: user.location ?? '' }, + avatarUrl: { github: user.avatarUrl ?? '' }, + company: { github: user.company ?? '' }, + } + if (user.websiteUrl) { + attributes.websiteUrl = { github: user.websiteUrl } + } + return attributes +} + +function toOrganizations(user: GithubUserNode): GithubOrganization[] { + const nodes = user.organizations?.nodes?.filter((node) => node !== null) ?? [] + + return nodes.map((org) => { + const organization: GithubOrganization = { + displayName: org.name || org.login, + names: [org.name, org.login].filter((name): name is string => Boolean(name)), + description: org.description ?? null, + location: org.location ?? null, + logo: org.avatarUrl ?? null, + source: 'github', + identities: [ + { + platform: 'github', + type: 'username', + value: org.login, + verified: true, + sourceId: org.databaseId?.toString() ?? '', + }, + ], + } + + if (org.websiteUrl) { + organization.identities.push({ + platform: 'github', + type: 'primary-domain', + value: org.websiteUrl, + verified: false, + }) + } + + if (org.twitterUsername) { + organization.identities.push({ + platform: 'twitter', + type: 'username', + value: org.twitterUsername, + verified: false, + }) + } + + return organization + }) +} + +export function toMember(user: GithubUserNode | null | undefined): GithubMember { + if (!user || !user.login) { + return GHOST_MEMBER + } + + if ((user.__typename !== 'User' || !user.databaseId) && user.__typename !== 'Bot') { + return { + displayName: user.name || user.login, + identities: [ + { + platform: 'github', + type: 'username', + verified: true, + value: user.login, + sourceId: user.id?.toString() ?? '', + }, + ], + attributes: toAttributes(user), + organizations: [], + } + } + + return { + displayName: user.login, + identities: [ + { + platform: 'github', + type: 'username', + verified: true, + value: user.login, + sourceId: user.databaseId?.toString() ?? '', + }, + ], + attributes: toAttributes(user), + organizations: toOrganizations(user), + } +} diff --git a/services/libs/connectors/src/connectors/github/mappers/prComment.ts b/services/libs/connectors/src/connectors/github/mappers/prComment.ts new file mode 100644 index 0000000000..e00d80c721 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/mappers/prComment.ts @@ -0,0 +1,29 @@ +import { GITHUB_GRID, GithubActivityType } from '@crowd/integrations' + +import type { PrCommentNode } from '../graphql/pullRequestChildren' +import type { PullRequestNode } from '../graphql/pullRequests' +import type { GithubActivity } from '../schemas' + +import { toMember } from './member' + +export function toPrComment(comment: PrCommentNode, pullRequest: PullRequestNode): GithubActivity { + return { + type: GithubActivityType.PULL_REQUEST_COMMENT, + timestamp: comment.createdAt, + sourceId: comment.id, + sourceParentId: pullRequest.id, + score: GITHUB_GRID[GithubActivityType.PULL_REQUEST_COMMENT].score, + title: pullRequest.title || '', + body: comment.body || '', + url: pullRequest.url, + attributes: { + state: pullRequest.state, + additions: pullRequest.additions, + deletions: pullRequest.deletions, + changedFiles: pullRequest.changedFiles, + authorAssociation: pullRequest.authorAssociation, + labels: pullRequest.labels?.nodes?.map((label) => label.name) || [], + }, + member: toMember(comment.author), + } +} diff --git a/services/libs/connectors/src/connectors/github/mappers/pullRequest.ts b/services/libs/connectors/src/connectors/github/mappers/pullRequest.ts new file mode 100644 index 0000000000..1c357d0938 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/mappers/pullRequest.ts @@ -0,0 +1,159 @@ +import { GITHUB_GRID, GithubActivityType } from '@crowd/integrations' + +import type { PrTimelineItem, PullRequestNode } from '../graphql/pullRequests' +import type { GithubActivity } from '../schemas' + +import { toMember } from './member' + +const DEFAULT_TIMESTAMP = '1970-01-01T00:00:00Z' + +function prAttributes(pullRequest: PullRequestNode): Record { + return { + additions: pullRequest.additions, + deletions: pullRequest.deletions, + changedFiles: pullRequest.changedFiles, + authorAssociation: pullRequest.authorAssociation, + labels: pullRequest.labels?.nodes?.map((label) => label.name) ?? [], + } +} + +function toTimelineActivity( + item: PrTimelineItem, + pullRequest: PullRequestNode, +): GithubActivity | null { + switch (item.__typename) { + case 'AssignedEvent': { + if (!item.assignee || !item.id) { + return null + } + const timestamp = item.createdAt ?? DEFAULT_TIMESTAMP + return { + type: GithubActivityType.PULL_REQUEST_ASSIGNED, + timestamp, + sourceId: `gen-AE_${pullRequest.id}_${item.actor?.login}_${item.assignee?.login}_${new Date(timestamp).toISOString()}`, + sourceParentId: pullRequest.id, + score: GITHUB_GRID[GithubActivityType.PULL_REQUEST_ASSIGNED].score, + title: pullRequest.title ?? '', + body: pullRequest.body ?? '', + url: pullRequest.url, + attributes: { state: pullRequest.state.toLowerCase(), ...prAttributes(pullRequest) }, + member: toMember(item.actor), + objectMember: toMember(item.assignee), + } + } + case 'ReviewRequestedEvent': { + if (!item.requestedReviewer?.login || !item.id) { + return null + } + const timestamp = item.createdAt ?? DEFAULT_TIMESTAMP + return { + type: GithubActivityType.PULL_REQUEST_REVIEW_REQUESTED, + timestamp, + sourceId: `gen-RRE_${pullRequest.id}_${item.actor?.login}_${item.requestedReviewer?.login}_${new Date(timestamp).toISOString()}`, + sourceParentId: pullRequest.id, + score: GITHUB_GRID[GithubActivityType.PULL_REQUEST_REVIEW_REQUESTED].score, + title: pullRequest.title ?? '', + body: pullRequest.body ?? '', + url: pullRequest.url, + attributes: { state: pullRequest.state.toLowerCase(), ...prAttributes(pullRequest) }, + member: toMember(item.actor), + objectMember: toMember(item.requestedReviewer), + } + } + case 'PullRequestReview': { + if (!item.id) { + return null + } + const timestamp = item.submittedAt ?? DEFAULT_TIMESTAMP + return { + type: GithubActivityType.PULL_REQUEST_REVIEWED, + timestamp, + sourceId: `gen-PRR_${pullRequest.id}_${item.author?.login}_${new Date(timestamp).toISOString()}`, + sourceParentId: pullRequest.id, + score: GITHUB_GRID[GithubActivityType.PULL_REQUEST_REVIEWED].score, + title: pullRequest.title ?? '', + body: item.body ?? pullRequest.body ?? '', + url: pullRequest.url, + attributes: { + reviewState: item.state, + state: pullRequest.state.toLowerCase(), + ...prAttributes(pullRequest), + }, + member: toMember(item.author), + } + } + case 'MergedEvent': { + if (!item.id) { + return null + } + const timestamp = item.createdAt ?? DEFAULT_TIMESTAMP + return { + type: GithubActivityType.PULL_REQUEST_MERGED, + timestamp, + sourceId: `gen-ME_${pullRequest.id}_${item.actor?.login}_${new Date(timestamp).toISOString()}`, + sourceParentId: pullRequest.id, + score: GITHUB_GRID[GithubActivityType.PULL_REQUEST_MERGED].score, + title: pullRequest.title ?? '', + body: pullRequest.body ?? '', + url: pullRequest.url, + attributes: { state: 'merged', ...prAttributes(pullRequest) }, + member: toMember(item.actor), + } + } + case 'ClosedEvent': { + if (!item.id) { + return null + } + const timestamp = item.createdAt ?? DEFAULT_TIMESTAMP + return { + type: GithubActivityType.PULL_REQUEST_CLOSED, + timestamp, + sourceId: `gen-CE_${pullRequest.id}_${item.actor?.login}_${new Date(timestamp).toISOString()}`, + sourceParentId: pullRequest.id, + score: GITHUB_GRID[GithubActivityType.PULL_REQUEST_CLOSED].score, + title: pullRequest.title ?? '', + body: pullRequest.body ?? '', + url: pullRequest.url, + attributes: { state: 'closed', ...prAttributes(pullRequest) }, + member: toMember(item.actor), + } + } + default: + return null + } +} + +export function toPullRequestActivities( + pullRequest: PullRequestNode, + timelineItems: (PrTimelineItem | null)[], +): GithubActivity[] { + if (!pullRequest || !pullRequest.id) { + return [] + } + + const activities: GithubActivity[] = [ + { + type: GithubActivityType.PULL_REQUEST_OPENED, + timestamp: pullRequest.createdAt ?? DEFAULT_TIMESTAMP, + sourceId: pullRequest.id, + score: GITHUB_GRID[GithubActivityType.PULL_REQUEST_OPENED].score, + title: pullRequest.title ?? '', + body: pullRequest.body ?? '', + url: pullRequest.url, + attributes: { state: 'open', ...prAttributes(pullRequest) }, + member: toMember(pullRequest.author), + }, + ] + + for (const item of timelineItems) { + if (!item) { + continue + } + const activity = toTimelineActivity(item, pullRequest) + if (activity) { + activities.push(activity) + } + } + + return activities +} diff --git a/services/libs/connectors/src/connectors/github/mappers/reviewThreadComment.ts b/services/libs/connectors/src/connectors/github/mappers/reviewThreadComment.ts new file mode 100644 index 0000000000..0df20bf395 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/mappers/reviewThreadComment.ts @@ -0,0 +1,33 @@ +import { GITHUB_GRID, GithubActivityType } from '@crowd/integrations' + +import type { ReviewThreadNode, ThreadCommentNode } from '../graphql/pullRequestChildren' +import type { PullRequestNode } from '../graphql/pullRequests' +import type { GithubActivity } from '../schemas' + +import { toMember } from './member' + +export function toReviewThreadComment( + comment: ThreadCommentNode, + thread: ReviewThreadNode, + pullRequest: PullRequestNode, +): GithubActivity { + return { + type: GithubActivityType.PULL_REQUEST_REVIEW_THREAD_COMMENT, + timestamp: comment.createdAt, + sourceId: comment.id, + sourceParentId: pullRequest.id, + score: GITHUB_GRID[GithubActivityType.PULL_REQUEST_REVIEW_THREAD_COMMENT].score, + title: pullRequest.title || '', + body: `[Thread ${thread.isResolved ? 'RESOLVED' : 'OPEN'}] ${comment.body || ''}`, + url: comment.url || pullRequest.url, + attributes: { + state: pullRequest.state, + additions: pullRequest.additions, + deletions: pullRequest.deletions, + changedFiles: pullRequest.changedFiles, + authorAssociation: pullRequest.authorAssociation, + labels: pullRequest.labels?.nodes?.map((label) => label.name) || [], + }, + member: toMember(comment.author), + } +} diff --git a/services/libs/connectors/src/connectors/github/mappers/star.ts b/services/libs/connectors/src/connectors/github/mappers/star.ts new file mode 100644 index 0000000000..a44bcff682 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/mappers/star.ts @@ -0,0 +1,26 @@ +import { createHash } from 'crypto' + +import { GITHUB_GRID, GithubActivityType } from '@crowd/integrations' + +import type { StargazerEdge } from '../graphql/stars' +import type { GithubActivity } from '../schemas' + +import { toMember } from './member' + +const DEFAULT_TIMESTAMP = '1970-01-01T00:00:00Z' + +export function toStar(edge: StargazerEdge): GithubActivity { + const timestamp = edge.starredAt ?? DEFAULT_TIMESTAMP + const sourceIdData = `${edge.node?.login}-star-${Math.floor( + new Date(timestamp).getTime() / 1000, + )}-github` + + return { + type: GithubActivityType.STAR, + timestamp, + sourceId: `gen-${createHash('md5').update(sourceIdData).digest('hex')}`, + score: GITHUB_GRID[GithubActivityType.STAR].score, + attributes: {}, + member: toMember(edge.node), + } +} diff --git a/services/libs/connectors/src/connectors/github/paging.ts b/services/libs/connectors/src/connectors/github/paging.ts new file mode 100644 index 0000000000..2f479d418d --- /dev/null +++ b/services/libs/connectors/src/connectors/github/paging.ts @@ -0,0 +1,30 @@ +export interface GithubWatermark { + phase: 'backfill' | 'incremental' + since: string | null + cursor: string | null +} + +export const PAGE_SIZE = 100 +// GitHub GraphQL silently omits timeline/connection items from heavy nodes(ids:) batches +// (no error, pageInfo claims completeness) — fetch per item, bounded by this concurrency. +export const ITEM_FETCH_CONCURRENCY = 5 + +export function readWatermark(raw: Record | null): GithubWatermark { + if (raw && (raw.phase === 'backfill' || raw.phase === 'incremental')) { + return { + phase: raw.phase, + since: typeof raw.since === 'string' ? raw.since : null, + cursor: typeof raw.cursor === 'string' ? raw.cursor : null, + } + } + return { phase: 'backfill', since: null, cursor: null } +} + +export function parseRepoChannel(channelName: string): { owner: string; repo: string } { + const path = channelName.replace(/^https:\/\/github\.com\//, '').replace(/\/+$/, '') + const parts = path.split('/') + if (parts.length !== 2 || !parts[0] || !parts[1]) { + throw new Error(`invalid github repo channel name: ${channelName}`) + } + return { owner: parts[0], repo: parts[1] } +} diff --git a/services/libs/connectors/src/connectors/github/prWalk.ts b/services/libs/connectors/src/connectors/github/prWalk.ts new file mode 100644 index 0000000000..81568177bb --- /dev/null +++ b/services/libs/connectors/src/connectors/github/prWalk.ts @@ -0,0 +1,106 @@ +import type { SyncContext, SyncOutcome } from '../../types' + +import { githubGraphql } from './gql' +import type { PullRequestNode, PullRequestsPage } from './graphql/pullRequests' +import { PULL_REQUESTS_QUERY } from './graphql/pullRequests' +import { parseRepoChannel, readWatermark } from './paging' + +export const PR_PAGE_SIZE = 50 + +export type PrPageHandler = (prs: PullRequestNode[], sinceDate: Date | null) => Promise + +async function runBackfill( + ctx: SyncContext, + owner: string, + repo: string, + processPrs: PrPageHandler, +): Promise { + const watermark = readWatermark(ctx.watermark) + let cursor = watermark.cursor + let since = watermark.since + + while (ctx.hasRunBudget()) { + const data = await githubGraphql(ctx.http, PULL_REQUESTS_QUERY, { + owner, + repo, + first: PR_PAGE_SIZE, + cursor, + direction: 'ASC', + }) + + const { pageInfo, nodes } = data.repository.pullRequests + const pullRequests = nodes.filter((node): node is PullRequestNode => node !== null) + + if (pullRequests.length > 0) { + await processPrs(pullRequests, null) + since = pullRequests[pullRequests.length - 1].updatedAt + } + + if (!pageInfo.hasNextPage) { + await ctx.commitWatermark({ phase: 'incremental', since, cursor: null }) + return { complete: true } + } + + cursor = pageInfo.endCursor + await ctx.commitWatermark({ phase: 'backfill', since, cursor }) + } + + return { complete: false } +} + +async function runIncremental( + ctx: SyncContext, + owner: string, + repo: string, + since: string, + processPrs: PrPageHandler, +): Promise { + const sinceDate = new Date(since) + let cursor: string | null = null + let newSince: string | null = null + + while (ctx.hasRunBudget()) { + const data = await githubGraphql(ctx.http, PULL_REQUESTS_QUERY, { + owner, + repo, + first: PR_PAGE_SIZE, + cursor, + direction: 'DESC', + }) + + const { pageInfo, nodes } = data.repository.pullRequests + const pullRequests = nodes.filter((node): node is PullRequestNode => node !== null) + + if (newSince === null && pullRequests.length > 0) { + newSince = pullRequests[0].updatedAt + } + + const fresh = pullRequests.filter((pr) => new Date(pr.updatedAt) > sinceDate) + if (fresh.length > 0) { + await processPrs(fresh, sinceDate) + } + + const reachedSince = fresh.length < pullRequests.length + if (reachedSince || !pageInfo.hasNextPage) { + await ctx.commitWatermark({ phase: 'incremental', since: newSince ?? since, cursor: null }) + return { complete: true } + } + + cursor = pageInfo.endCursor + } + + return { complete: false } +} + +export async function runDualPhasePrSync( + ctx: SyncContext, + processPrs: PrPageHandler, +): Promise { + const { owner, repo } = parseRepoChannel(ctx.channel.channelName) + const watermark = readWatermark(ctx.watermark) + + if (watermark.phase === 'incremental' && watermark.since) { + return runIncremental(ctx, owner, repo, watermark.since, processPrs) + } + return runBackfill(ctx, owner, repo, processPrs) +} diff --git a/services/libs/connectors/src/connectors/github/schemas.ts b/services/libs/connectors/src/connectors/github/schemas.ts new file mode 100644 index 0000000000..48429149c6 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/schemas.ts @@ -0,0 +1,68 @@ +import { z } from 'zod' + +import { GithubActivityType } from '@crowd/integrations' + +const identitySchema = z.object({ + platform: z.string(), + value: z.string(), + type: z.string(), + verified: z.boolean(), + sourceId: z.string(), +}) + +const githubScoped = (value: T) => z.object({ github: value }) + +const memberAttributesSchema = z.object({ + isHireable: githubScoped(z.boolean()).optional(), + url: githubScoped(z.string()).optional(), + bio: githubScoped(z.string()).optional(), + location: githubScoped(z.string()).optional(), + avatarUrl: githubScoped(z.string()).optional(), + company: githubScoped(z.string()).optional(), + websiteUrl: githubScoped(z.string()).optional(), + sourceId: githubScoped(z.string()).optional(), + isBot: githubScoped(z.boolean()).optional(), +}) + +const organizationIdentitySchema = z.object({ + platform: z.string(), + type: z.string(), + value: z.string(), + verified: z.boolean(), + sourceId: z.string().optional(), +}) + +const organizationSchema = z.object({ + displayName: z.string(), + names: z.array(z.string()), + description: z.string().nullable(), + location: z.string().nullable(), + logo: z.string().nullable(), + source: z.string(), + identities: z.array(organizationIdentitySchema), +}) + +const memberSchema = z.object({ + displayName: z.string(), + identities: z.array(identitySchema).min(1), + attributes: memberAttributesSchema, + organizations: z.array(organizationSchema).optional(), +}) + +export const githubActivitySchema = z.object({ + type: z.nativeEnum(GithubActivityType), + timestamp: z.string(), + sourceId: z.string(), + sourceParentId: z.string().optional(), + score: z.number(), + title: z.string().optional(), + body: z.string().optional(), + url: z.string().optional(), + attributes: z.record(z.unknown()).optional(), + member: memberSchema, + objectMember: memberSchema.optional(), +}) + +export type GithubActivity = z.infer +export type GithubMember = z.infer +export type GithubOrganization = z.infer diff --git a/services/libs/connectors/src/connectors/github/syncs/discussions.ts b/services/libs/connectors/src/connectors/github/syncs/discussions.ts new file mode 100644 index 0000000000..8ea545b843 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/syncs/discussions.ts @@ -0,0 +1,154 @@ +import { mapWithConcurrency } from '../../../concurrency' +import type { SyncContext, SyncDefinition, SyncOutcome } from '../../../types' +import { githubGraphql } from '../gql' +import type { + CommentRepliesPage, + DiscussionCommentNode, + DiscussionCommentsPage, + DiscussionNode, + DiscussionReplyNode, + DiscussionsPage, +} from '../graphql/discussions' +import { + COMMENT_REPLIES_QUERY, + DISCUSSIONS_QUERY, + DISCUSSION_COMMENTS_QUERY, +} from '../graphql/discussions' +import { toDiscussionCommentActivity, toDiscussionStartedActivity } from '../mappers/discussion' +import { ITEM_FETCH_CONCURRENCY, PAGE_SIZE, parseRepoChannel, readWatermark } from '../paging' +import type { GithubActivity } from '../schemas' +import { githubActivitySchema } from '../schemas' + +const COMMENTS_PAGE_SIZE = 50 +const REPLIES_PAGE_SIZE = 100 + +async function runDiscussionsSync(ctx: SyncContext): Promise { + const { owner, repo } = parseRepoChannel(ctx.channel.channelName) + const watermark = readWatermark(ctx.watermark) + + const sinceDate = watermark.since ? new Date(watermark.since) : null + let newestUpdatedAt = watermark.since + let cursor: string | null = null + + const collectReplies = async ( + discussion: DiscussionNode, + comment: DiscussionCommentNode, + ): Promise => { + const activities = comment.replies.nodes + .filter((node): node is DiscussionReplyNode => node !== null) + .map((reply) => toDiscussionCommentActivity(discussion, reply, true)) + + let repliesCursor = comment.replies.pageInfo.hasNextPage + ? comment.replies.pageInfo.endCursor + : null + while (repliesCursor) { + const data = await githubGraphql(ctx.http, COMMENT_REPLIES_QUERY, { + id: comment.id, + first: REPLIES_PAGE_SIZE, + cursor: repliesCursor, + }) + const replies = data.node?.replies + if (!replies) { + break + } + activities.push( + ...replies.nodes + .filter((node): node is DiscussionReplyNode => node !== null) + .map((reply) => toDiscussionCommentActivity(discussion, reply, true)), + ) + repliesCursor = replies.pageInfo.hasNextPage ? replies.pageInfo.endCursor : null + } + + return activities + } + + const collectDiscussionActivities = async ( + discussion: DiscussionNode, + ): Promise => { + const activities: GithubActivity[] = [toDiscussionStartedActivity(discussion)] + + let commentsCursor: string | null = null + let hasMore = discussion.comments.totalCount > 0 + while (hasMore) { + const data = await githubGraphql( + ctx.http, + DISCUSSION_COMMENTS_QUERY, + { + id: discussion.id, + first: COMMENTS_PAGE_SIZE, + cursor: commentsCursor, + repliesFirst: REPLIES_PAGE_SIZE, + }, + ) + const comments = data.node?.comments + if (!comments) { + break + } + for (const comment of comments.nodes.filter( + (node): node is DiscussionCommentNode => node !== null, + )) { + activities.push(toDiscussionCommentActivity(discussion, comment, Boolean(comment.replyTo))) + activities.push(...(await collectReplies(discussion, comment))) + } + hasMore = comments.pageInfo.hasNextPage + commentsCursor = comments.pageInfo.endCursor + } + + return activities + } + + // discussions has no filterBy.since; walk UPDATED_AT DESC and stop at the watermark, + // committing only after the walk so a partial run cannot skip older updates. + let firstPage = true + let walked = false + + while (ctx.hasRunBudget()) { + const data = await githubGraphql(ctx.http, DISCUSSIONS_QUERY, { + owner, + repo, + first: PAGE_SIZE, + cursor, + }) + + const { pageInfo, nodes } = data.repository.discussions + const discussions = nodes.filter((node): node is DiscussionNode => node !== null) + const fresh = sinceDate + ? discussions.filter((discussion) => new Date(discussion.updatedAt) >= sinceDate) + : discussions + + if (firstPage && discussions.length > 0) { + newestUpdatedAt = discussions[0].updatedAt + } + firstPage = false + + if (fresh.length > 0) { + const batches = await mapWithConcurrency( + fresh, + ITEM_FETCH_CONCURRENCY, + collectDiscussionActivities, + ) + await ctx.emit(batches.flat()) + } + + const reachedWatermark = fresh.length < discussions.length + cursor = pageInfo.hasNextPage ? pageInfo.endCursor : null + if (reachedWatermark || !pageInfo.hasNextPage) { + walked = true + break + } + } + + if (!walked) { + return { complete: false } + } + + await ctx.commitWatermark({ phase: 'incremental', since: newestUpdatedAt, cursor: null }) + return { complete: true } +} + +export const discussionsSync: SyncDefinition = { + name: 'discussions', + cadenceMinutes: 60, + schema: githubActivitySchema, + run: runDiscussionsSync, +} diff --git a/services/libs/connectors/src/connectors/github/syncs/forks.ts b/services/libs/connectors/src/connectors/github/syncs/forks.ts new file mode 100644 index 0000000000..a213e08f7d --- /dev/null +++ b/services/libs/connectors/src/connectors/github/syncs/forks.ts @@ -0,0 +1,50 @@ +import type { SyncContext, SyncDefinition, SyncOutcome } from '../../../types' +import { githubGraphql } from '../gql' +import type { ForkNode, ForksPage } from '../graphql/forks' +import { FORKS_QUERY } from '../graphql/forks' +import { toFork } from '../mappers/fork' +import { PAGE_SIZE, parseRepoChannel, readWatermark } from '../paging' +import { githubActivitySchema } from '../schemas' + +async function runForksSync(ctx: SyncContext): Promise { + const { owner, repo } = parseRepoChannel(ctx.channel.channelName) + const watermark = readWatermark(ctx.watermark) + + let since = watermark.since + let cursor = watermark.cursor + + while (ctx.hasRunBudget()) { + const data = await githubGraphql(ctx.http, FORKS_QUERY, { + owner, + repo, + first: PAGE_SIZE, + cursor, + }) + + const { pageInfo, nodes } = data.repository.forks + const forks = nodes + .filter((node): node is ForkNode => node !== null) + .filter((node) => !since || node.createdAt > since) + + if (forks.length > 0) { + await ctx.emit(forks.map(toFork)) + since = forks[forks.length - 1].createdAt + } + + cursor = pageInfo.endCursor ?? cursor + await ctx.commitWatermark({ phase: 'incremental', since, cursor }) + + if (!pageInfo.hasNextPage) { + return { complete: true } + } + } + + return { complete: false } +} + +export const forksSync: SyncDefinition = { + name: 'forks', + cadenceMinutes: 360, + schema: githubActivitySchema, + run: runForksSync, +} diff --git a/services/libs/connectors/src/connectors/github/syncs/issueComments.ts b/services/libs/connectors/src/connectors/github/syncs/issueComments.ts new file mode 100644 index 0000000000..9eb62ab049 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/syncs/issueComments.ts @@ -0,0 +1,133 @@ +import type { SyncContext, SyncDefinition, SyncOutcome } from '../../../types' +import { githubGraphql } from '../gql' +import type { + IssueCommentNode, + IssueCommentsBatchPage, + IssueCommentsPaginatedPage, + IssueNode, + IssuesPage, +} from '../graphql/issues' +import { + ISSUES_QUERY, + ISSUE_COMMENTS_PAGINATED_QUERY, + ISSUE_COMMENTS_QUERY, +} from '../graphql/issues' +import { toIssueComment } from '../mappers/issueComment' +import { PAGE_SIZE, parseRepoChannel, readWatermark } from '../paging' +import { githubActivitySchema } from '../schemas' + +const ISSUE_BATCH_SIZE = 15 +const COMMENTS_BATCH_PAGE_SIZE = 25 +// api is highly unreliable when paginating comments per issue, so keep the page small +// (nango workaround: nango-integrations/github/syncs/issue-comments.ts) +const COMMENTS_PAGINATED_PAGE_SIZE = 5 + +async function runIssueCommentsSync(ctx: SyncContext): Promise { + const { owner, repo } = parseRepoChannel(ctx.channel.channelName) + const watermark = readWatermark(ctx.watermark) + + const querySince = watermark.since + const sinceDate = querySince ? new Date(querySince) : null + let since = watermark.since + let cursor: string | null = null + + const emitComments = async ( + nodes: (IssueCommentNode | null)[], + issue: IssueNode, + ): Promise => { + const comments = nodes + .filter((node): node is IssueCommentNode => node !== null && Boolean(node.id)) + .filter((node) => !sinceDate || new Date(node.createdAt) >= sinceDate) + if (comments.length > 0) { + await ctx.emit(comments.map((comment) => toIssueComment(comment, issue))) + } + } + + const drainRemainingComments = async (issue: IssueNode, startCursor: string | null) => { + let commentsCursor = startCursor + let hasMore = true + while (hasMore) { + const data = await githubGraphql( + ctx.http, + ISSUE_COMMENTS_PAGINATED_QUERY, + { + owner, + repo, + issueNumber: issue.number, + first: COMMENTS_PAGINATED_PAGE_SIZE, + cursor: commentsCursor, + }, + ) + const comments = data.repository.issue?.comments + if (!comments?.nodes) { + return + } + await emitComments(comments.nodes, issue) + hasMore = comments.pageInfo.hasNextPage + commentsCursor = comments.pageInfo.endCursor + } + } + + while (ctx.hasRunBudget()) { + const data = await githubGraphql(ctx.http, ISSUES_QUERY, { + owner, + repo, + first: PAGE_SIZE, + cursor, + since: querySince, + }) + + const { pageInfo, nodes } = data.repository.issues + const issues = nodes.filter((node): node is IssueNode => node !== null) + + for (let i = 0; i < issues.length; i += ISSUE_BATCH_SIZE) { + const batch = issues.slice(i, i + ISSUE_BATCH_SIZE) + const batchData = await githubGraphql( + ctx.http, + ISSUE_COMMENTS_QUERY, + { + ids: batch.map((issue) => issue.id), + first: COMMENTS_BATCH_PAGE_SIZE, + }, + ) + + const toPaginate: { issue: IssueNode; commentsCursor: string | null }[] = [] + for (const node of batchData.nodes) { + if (!node?.comments?.nodes) { + continue + } + const issue = batch.find((candidate) => candidate.id === node.id) + if (!issue) { + continue + } + await emitComments(node.comments.nodes, issue) + if (node.comments.pageInfo.hasNextPage) { + toPaginate.push({ issue, commentsCursor: node.comments.pageInfo.endCursor }) + } + } + + for (const { issue, commentsCursor } of toPaginate) { + await drainRemainingComments(issue, commentsCursor) + } + } + + if (issues.length > 0) { + since = issues[issues.length - 1].updatedAt + } + cursor = pageInfo.hasNextPage ? pageInfo.endCursor : null + await ctx.commitWatermark({ phase: 'incremental', since, cursor: null }) + + if (!pageInfo.hasNextPage) { + return { complete: true } + } + } + + return { complete: false } +} + +export const issueCommentsSync: SyncDefinition = { + name: 'issue-comments', + cadenceMinutes: 60, + schema: githubActivitySchema, + run: runIssueCommentsSync, +} diff --git a/services/libs/connectors/src/connectors/github/syncs/issues.ts b/services/libs/connectors/src/connectors/github/syncs/issues.ts new file mode 100644 index 0000000000..d53c0d50dc --- /dev/null +++ b/services/libs/connectors/src/connectors/github/syncs/issues.ts @@ -0,0 +1,52 @@ +import type { SyncContext, SyncDefinition, SyncOutcome } from '../../../types' +import { githubGraphql } from '../gql' +import type { IssueNode, IssuesPage } from '../graphql/issues' +import { ISSUES_QUERY } from '../graphql/issues' +import { toIssueActivities } from '../mappers/issue' +import { PAGE_SIZE, parseRepoChannel, readWatermark } from '../paging' +import { githubActivitySchema } from '../schemas' + +async function runIssuesSync(ctx: SyncContext): Promise { + const { owner, repo } = parseRepoChannel(ctx.channel.channelName) + const watermark = readWatermark(ctx.watermark) + + // filterBy.since is coupled to the cursor's result set: advancing it between + // pages would invalidate cursors, so the query keeps the run-start since. + const querySince = watermark.since + let since = watermark.since + let cursor: string | null = null + + while (ctx.hasRunBudget()) { + const data = await githubGraphql(ctx.http, ISSUES_QUERY, { + owner, + repo, + first: PAGE_SIZE, + cursor, + since: querySince, + }) + + const { pageInfo, nodes } = data.repository.issues + const issues = nodes.filter((node): node is IssueNode => node !== null) + + if (issues.length > 0) { + await ctx.emit(issues.flatMap(toIssueActivities)) + since = issues[issues.length - 1].updatedAt + } + + cursor = pageInfo.hasNextPage ? pageInfo.endCursor : null + await ctx.commitWatermark({ phase: 'incremental', since, cursor: null }) + + if (!pageInfo.hasNextPage) { + return { complete: true } + } + } + + return { complete: false } +} + +export const issuesSync: SyncDefinition = { + name: 'issues', + cadenceMinutes: 60, + schema: githubActivitySchema, + run: runIssuesSync, +} diff --git a/services/libs/connectors/src/connectors/github/syncs/pullRequestComments.ts b/services/libs/connectors/src/connectors/github/syncs/pullRequestComments.ts new file mode 100644 index 0000000000..a888979b35 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/syncs/pullRequestComments.ts @@ -0,0 +1,57 @@ +import { mapWithConcurrency } from '../../../concurrency' +import type { SyncContext, SyncDefinition, SyncOutcome } from '../../../types' +import { githubGraphql } from '../gql' +import type { PrCommentNode, PrCommentsBatchPage } from '../graphql/pullRequestChildren' +import { COMMENTS_FOR_PRS_QUERY } from '../graphql/pullRequestChildren' +import { toPrComment } from '../mappers/prComment' +import { ITEM_FETCH_CONCURRENCY } from '../paging' +import { runDualPhasePrSync } from '../prWalk' +import { githubActivitySchema } from '../schemas' + +const COMMENTS_PAGE_SIZE = 50 + +async function fetchComments(ctx: SyncContext, prId: string): Promise { + const comments: PrCommentNode[] = [] + let cursor: string | null = null + do { + const data = await githubGraphql(ctx.http, COMMENTS_FOR_PRS_QUERY, { + ids: [prId], + first: COMMENTS_PAGE_SIZE, + after: cursor, + }) + const connection = data.nodes[0]?.comments + if (!connection?.edges) { + break + } + comments.push( + ...connection.edges + .map((edge) => edge?.node) + .filter((node): node is PrCommentNode => Boolean(node?.id)), + ) + cursor = connection.pageInfo.hasNextPage ? connection.pageInfo.endCursor : null + } while (cursor) + return comments +} + +async function runPullRequestCommentsSync(ctx: SyncContext): Promise { + return runDualPhasePrSync(ctx, async (prs, sinceDate) => { + const commentsPerPr = await mapWithConcurrency(prs, ITEM_FETCH_CONCURRENCY, (pr) => + fetchComments(ctx, pr.id), + ) + const activities = prs.flatMap((pullRequest, index) => + commentsPerPr[index] + .filter((comment) => !sinceDate || new Date(comment.createdAt) >= sinceDate) + .map((comment) => toPrComment(comment, pullRequest)), + ) + if (activities.length > 0) { + await ctx.emit(activities) + } + }) +} + +export const pullRequestCommentsSync: SyncDefinition = { + name: 'pull-request-comments', + cadenceMinutes: 60, + schema: githubActivitySchema, + run: runPullRequestCommentsSync, +} diff --git a/services/libs/connectors/src/connectors/github/syncs/pullRequestCommits.ts b/services/libs/connectors/src/connectors/github/syncs/pullRequestCommits.ts new file mode 100644 index 0000000000..7fc0b1417b --- /dev/null +++ b/services/libs/connectors/src/connectors/github/syncs/pullRequestCommits.ts @@ -0,0 +1,56 @@ +import type { SyncContext, SyncDefinition, SyncOutcome } from '../../../types' +import { githubGraphql } from '../gql' +import type { PrCommitNode, PrCommitsPage } from '../graphql/pullRequestChildren' +import { PR_COMMITS_QUERY } from '../graphql/pullRequestChildren' +import { toCommit } from '../mappers/commit' +import { parseRepoChannel } from '../paging' +import { runDualPhasePrSync } from '../prWalk' +import { githubActivitySchema } from '../schemas' + +const COMMITS_PAGE_SIZE = 50 + +async function runPullRequestCommitsSync(ctx: SyncContext): Promise { + const { owner, repo } = parseRepoChannel(ctx.channel.channelName) + + return runDualPhasePrSync(ctx, async (prs) => { + for (const pullRequest of prs) { + let cursor: string | null = null + let hasMore = true + + while (hasMore) { + const data = await githubGraphql(ctx.http, PR_COMMITS_QUERY, { + owner, + repo, + prNumber: pullRequest.number, + first: COMMITS_PAGE_SIZE, + cursor, + }) + + const commits = data.repository.pullRequest?.commits + if (!commits) { + break + } + + const fresh = commits.nodes + .map((node) => node?.commit) + .filter((commit): commit is PrCommitNode['commit'] => + Boolean(commit?.author?.user?.login), + ) + + if (fresh.length > 0) { + await ctx.emit(fresh.map((commit) => toCommit(commit, pullRequest.id))) + } + + hasMore = commits.pageInfo.hasNextPage + cursor = commits.pageInfo.endCursor + } + } + }) +} + +export const pullRequestCommitsSync: SyncDefinition = { + name: 'pull-request-commits', + cadenceMinutes: 120, + schema: githubActivitySchema, + run: runPullRequestCommitsSync, +} diff --git a/services/libs/connectors/src/connectors/github/syncs/pullRequestReviewComments.ts b/services/libs/connectors/src/connectors/github/syncs/pullRequestReviewComments.ts new file mode 100644 index 0000000000..937293f397 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/syncs/pullRequestReviewComments.ts @@ -0,0 +1,100 @@ +import { mapWithConcurrency } from '../../../concurrency' +import type { SyncContext, SyncDefinition, SyncOutcome } from '../../../types' +import { githubGraphql } from '../gql' +import type { + ReviewThreadNode, + ReviewThreadsBatchPage, + ThreadCommentNode, + ThreadCommentsBatchPage, +} from '../graphql/pullRequestChildren' +import { + COMMENTS_FOR_THREADS_QUERY, + REVIEW_THREADS_FOR_PRS_QUERY, +} from '../graphql/pullRequestChildren' +import { toReviewThreadComment } from '../mappers/reviewThreadComment' +import { ITEM_FETCH_CONCURRENCY } from '../paging' +import { runDualPhasePrSync } from '../prWalk' +import { githubActivitySchema } from '../schemas' + +const THREADS_PAGE_SIZE = 50 +const COMMENTS_PAGE_SIZE = 50 + +async function fetchThreads(ctx: SyncContext, prId: string): Promise { + const threads: ReviewThreadNode[] = [] + let cursor: string | null = null + do { + const data = await githubGraphql( + ctx.http, + REVIEW_THREADS_FOR_PRS_QUERY, + { ids: [prId], first: THREADS_PAGE_SIZE, after: cursor }, + ) + const connection = data.nodes[0]?.reviewThreads + if (!connection?.edges) { + break + } + threads.push( + ...connection.edges + .map((edge) => edge?.node) + .filter((node): node is ReviewThreadNode => Boolean(node?.id)), + ) + cursor = connection.pageInfo.hasNextPage ? connection.pageInfo.endCursor : null + } while (cursor) + return threads +} + +async function fetchThreadComments( + ctx: SyncContext, + threadId: string, +): Promise { + const comments: ThreadCommentNode[] = [] + let cursor: string | null = null + do { + const data = await githubGraphql( + ctx.http, + COMMENTS_FOR_THREADS_QUERY, + { ids: [threadId], first: COMMENTS_PAGE_SIZE, after: cursor }, + ) + const connection = data.nodes[0]?.comments + if (!connection?.edges) { + break + } + comments.push( + ...connection.edges + .map((edge) => edge?.node) + .filter((node): node is ThreadCommentNode => Boolean(node?.id)), + ) + cursor = connection.pageInfo.hasNextPage ? connection.pageInfo.endCursor : null + } while (cursor) + return comments +} + +async function runPullRequestReviewCommentsSync(ctx: SyncContext): Promise { + return runDualPhasePrSync(ctx, async (prs, sinceDate) => { + const threadsPerPr = await mapWithConcurrency(prs, ITEM_FETCH_CONCURRENCY, (pr) => + fetchThreads(ctx, pr.id), + ) + const threads = prs.flatMap((pullRequest, index) => + threadsPerPr[index].map((thread) => ({ thread, pullRequest })), + ) + + const commentsPerThread = await mapWithConcurrency(threads, ITEM_FETCH_CONCURRENCY, (entry) => + fetchThreadComments(ctx, entry.thread.id), + ) + + const activities = threads.flatMap(({ thread, pullRequest }, index) => + commentsPerThread[index] + .filter((comment) => !sinceDate || new Date(comment.createdAt) >= sinceDate) + .map((comment) => toReviewThreadComment(comment, thread, pullRequest)), + ) + if (activities.length > 0) { + await ctx.emit(activities) + } + }) +} + +export const pullRequestReviewCommentsSync: SyncDefinition = { + name: 'pull-request-review-comments', + cadenceMinutes: 120, + schema: githubActivitySchema, + run: runPullRequestReviewCommentsSync, +} diff --git a/services/libs/connectors/src/connectors/github/syncs/pullRequests.ts b/services/libs/connectors/src/connectors/github/syncs/pullRequests.ts new file mode 100644 index 0000000000..b5cc4a4333 --- /dev/null +++ b/services/libs/connectors/src/connectors/github/syncs/pullRequests.ts @@ -0,0 +1,59 @@ +import { mapWithConcurrency } from '../../../concurrency' +import type { SyncContext, SyncDefinition, SyncOutcome } from '../../../types' +import { githubGraphql } from '../gql' +import type { PrTimelineItem, PrTimelinePage, PullRequestNode } from '../graphql/pullRequests' +import { PR_TIMELINE_QUERY } from '../graphql/pullRequests' +import { toPullRequestActivities } from '../mappers/pullRequest' +import { ITEM_FETCH_CONCURRENCY } from '../paging' +import { runDualPhasePrSync } from '../prWalk' +import { githubActivitySchema } from '../schemas' + +const TIMELINE_PAGE_SIZE = 50 + +async function fetchTimeline(ctx: SyncContext, prId: string): Promise<(PrTimelineItem | null)[]> { + const items: (PrTimelineItem | null)[] = [] + let cursor: string | null = null + do { + const data = await githubGraphql(ctx.http, PR_TIMELINE_QUERY, { + ids: [prId], + first: TIMELINE_PAGE_SIZE, + after: cursor, + }) + const timeline = data.nodes[0]?.timelineItems + if (!timeline) { + break + } + items.push(...timeline.nodes) + cursor = timeline.pageInfo.hasNextPage ? timeline.pageInfo.endCursor : null + } while (cursor) + return items +} + +async function emitPullRequests(ctx: SyncContext, pullRequests: PullRequestNode[]): Promise { + const timelines = await mapWithConcurrency(pullRequests, ITEM_FETCH_CONCURRENCY, (pr) => + fetchTimeline(ctx, pr.id), + ) + + pullRequests.forEach((pr, index) => { + const hasMergedEvent = timelines[index].some((item) => item?.__typename === 'MergedEvent') + if (pr.state === 'MERGED' && !hasMergedEvent) { + ctx.log.warn( + { prId: pr.id, prNumber: pr.number, itemCount: timelines[index].length }, + 'merged pr timeline missing MergedEvent', + ) + } + }) + + await ctx.emit(pullRequests.flatMap((pr, index) => toPullRequestActivities(pr, timelines[index]))) +} + +async function runPullRequestsSync(ctx: SyncContext): Promise { + return runDualPhasePrSync(ctx, (prs) => emitPullRequests(ctx, prs)) +} + +export const pullRequestsSync: SyncDefinition = { + name: 'pull-requests', + cadenceMinutes: 60, + schema: githubActivitySchema, + run: runPullRequestsSync, +} diff --git a/services/libs/connectors/src/connectors/github/syncs/stars.ts b/services/libs/connectors/src/connectors/github/syncs/stars.ts new file mode 100644 index 0000000000..4b5940ed5d --- /dev/null +++ b/services/libs/connectors/src/connectors/github/syncs/stars.ts @@ -0,0 +1,50 @@ +import type { SyncContext, SyncDefinition, SyncOutcome } from '../../../types' +import { githubGraphql } from '../gql' +import type { StargazerEdge, StargazersPage } from '../graphql/stars' +import { STARGAZERS_QUERY } from '../graphql/stars' +import { toStar } from '../mappers/star' +import { PAGE_SIZE, parseRepoChannel, readWatermark } from '../paging' +import { githubActivitySchema } from '../schemas' + +async function runStarsSync(ctx: SyncContext): Promise { + const { owner, repo } = parseRepoChannel(ctx.channel.channelName) + const watermark = readWatermark(ctx.watermark) + + let since = watermark.since + let cursor = watermark.cursor + + while (ctx.hasRunBudget()) { + const data = await githubGraphql(ctx.http, STARGAZERS_QUERY, { + owner, + repo, + first: PAGE_SIZE, + cursor, + }) + + const { pageInfo, edges } = data.repository.stargazers + const stargazers = edges + .filter((edge): edge is StargazerEdge => edge !== null) + .filter((edge) => !since || edge.starredAt > since) + + if (stargazers.length > 0) { + await ctx.emit(stargazers.map(toStar)) + since = stargazers[stargazers.length - 1].starredAt + } + + cursor = pageInfo.endCursor ?? cursor + await ctx.commitWatermark({ phase: 'incremental', since, cursor }) + + if (!pageInfo.hasNextPage) { + return { complete: true } + } + } + + return { complete: false } +} + +export const starsSync: SyncDefinition = { + name: 'stars', + cadenceMinutes: 1440, + schema: githubActivitySchema, + run: runStarsSync, +} diff --git a/services/libs/connectors/src/credentials.ts b/services/libs/connectors/src/credentials.ts new file mode 100644 index 0000000000..d5daa68405 --- /dev/null +++ b/services/libs/connectors/src/credentials.ts @@ -0,0 +1,46 @@ +import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' + +import type { Credential } from './types' + +export async function getCredential(qx: QueryExecutor, integrationId: string): Promise { + const integration: { platform: string } | null = await qx.selectOneOrNone( + `SELECT platform + FROM integrations + WHERE id = $(integrationId) AND "deletedAt" IS NULL`, + { integrationId }, + ) + + if (!integration) { + throw new Error(`integration ${integrationId} not found`) + } + + // POC scope: GitHub only; each migrated connector adds its platform case here + switch (integration.platform) { + case 'github': + case 'github-nango': + return githubAppCredential() + default: + throw new Error(`unsupported platform ${integration.platform}`) + } +} + +// POC only: secrets come from env; secret-manager adoption (OCI Vault) swaps this +// body without touching callers — getCredential stays the single entry point +function githubAppCredential(): Credential { + const appId = process.env.CROWD_GITHUB_APP_ID + const rawPrivateKey = process.env.CROWD_GITHUB_PRIVATE_KEY + + if (!appId || !rawPrivateKey) { + throw new Error('missing CROWD_GITHUB_APP_ID or CROWD_GITHUB_PRIVATE_KEY environment variables') + } + + const privateKey = rawPrivateKey.startsWith('-----') + ? rawPrivateKey + : Buffer.from(rawPrivateKey, 'base64').toString('ascii') + + return { + platform: 'github', + kind: 'github-app', + data: { appId, privateKey }, + } +} diff --git a/services/libs/connectors/src/emit.ts b/services/libs/connectors/src/emit.ts new file mode 100644 index 0000000000..4a955f8c4c --- /dev/null +++ b/services/libs/connectors/src/emit.ts @@ -0,0 +1,64 @@ +import { ZodError, ZodType } from 'zod' + +import type { DataSinkWorkerEmitter } from '@crowd/common_services' +import type { ISyncUnit } from '@crowd/data-access-layer/src/connectors' +import type { Logger } from '@crowd/logging' +import { IIntegrationResult, IntegrationResultType } from '@crowd/types' + +import { ConnectorError } from './http/errors' + +export interface EmitterDeps { + publishResult: (integrationId: string, result: IIntegrationResult) => Promise + sinkEmitter: DataSinkWorkerEmitter + unit: ISyncUnit + segmentId: string + schema: ZodType> + log: Logger +} + +export interface Emitter { + emit: (records: unknown[]) => Promise + emittedCount: () => number +} + +export function createEmit(deps: EmitterDeps): Emitter { + let emitted = 0 + + const emit = async (records: unknown[]): Promise => { + for (const record of records) { + let parsed: Record + try { + parsed = deps.schema.parse(record) + } catch (err) { + if (err instanceof ZodError) { + throw new ConnectorError('connector.code', 'record failed schema validation', { + cause: err, + }) + } + throw err + } + + const payload = { ...parsed, channel: deps.unit.channelName } + + try { + const resultId = await deps.publishResult(deps.unit.integrationId, { + type: IntegrationResultType.ACTIVITY, + segmentId: deps.segmentId, + data: payload, + }) + await deps.sinkEmitter.triggerResultProcessing(resultId, resultId, false) + } catch (err) { + if (err instanceof ConnectorError) { + throw err + } + throw new ConnectorError('sink.rejected', 'failed to hand record to sink', { cause: err }) + } + + emitted += 1 + } + + deps.log.debug({ count: records.length, total: emitted }, 'emitted records') + } + + return { emit, emittedCount: () => emitted } +} diff --git a/services/libs/connectors/src/http/client.ts b/services/libs/connectors/src/http/client.ts new file mode 100644 index 0000000000..d006a9ac0d --- /dev/null +++ b/services/libs/connectors/src/http/client.ts @@ -0,0 +1,245 @@ +import axios, { AxiosHeaders, AxiosRequestConfig, AxiosResponse } from 'axios' + +import { timeout } from '@crowd/common' +import type { Logger } from '@crowd/logging' + +import { + ConnectorError, + ProviderUnavailableError, + RateLimitError, + errorFromHttpStatus, +} from './errors' + +export interface IPooledToken { + id: string + value: string +} + +export interface HttpResponse { + status: number + headers: Record + data: unknown +} + +export type TokenApplier = (config: AxiosRequestConfig, token: IPooledToken) => AxiosRequestConfig + +export type ResponseInterpreter = (response: HttpResponse) => ConnectorError | null + +export interface HttpClientDeps { + acquireToken: () => Promise + parkToken: (tokenId: string, resumeAt: Date) => Promise + invalidateToken: (tokenId: string) => Promise + log: Logger + applyToken?: TokenApplier + interpretResponse?: ResponseInterpreter +} + +export interface ConnectorHttp { + request(config: AxiosRequestConfig): Promise + requestCount(): number +} + +type CountingHttpClientDeps = HttpClientDeps & { countRequest: () => void } + +const MAX_ATTEMPTS = 3 +const BACKOFF_BASE_MS = 1000 +const RATE_LIMIT_FALLBACK_MS = 60_000 +const DEFAULT_TIMEOUT_MS = 60_000 + +export function createHttpClient(deps: HttpClientDeps): ConnectorHttp { + let requests = 0 + const countingDeps: CountingHttpClientDeps = { + ...deps, + countRequest: () => { + requests += 1 + }, + } + return { + request: (config: AxiosRequestConfig) => requestWithRetry(countingDeps, config), + requestCount: () => requests, + } +} + +async function requestWithRetry( + deps: CountingHttpClientDeps, + config: AxiosRequestConfig, +): Promise { + let lastError: ConnectorError = new ProviderUnavailableError() + for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { + try { + return await attemptRequest(deps, config) + } catch (err) { + if (!(err instanceof ConnectorError) || err.errorClass !== 'provider.unavailable') { + throw err + } + lastError = err + if (attempt < MAX_ATTEMPTS) { + const delay = BACKOFF_BASE_MS * 2 ** (attempt - 1) + deps.log.warn({ attempt, delay, reason: err.message }, 'provider unavailable, backing off') + await timeout(delay) + } + } + } + throw lastError +} + +async function attemptRequest( + deps: CountingHttpClientDeps, + config: AxiosRequestConfig, + authRetried = false, +): Promise { + const token = await deps.acquireToken() + const response = await send(deps, config, token) + const headers = normalizeHeaders(response.headers) + const error = classifyResponse(deps, response.status, headers, response.data) + + if (!error) { + return response.data + } + + if (error.errorClass === 'provider.rate_limit') { + const resumeAt = error.options?.resumeAt ?? computeResumeAt(headers) + // parking makes acquire() skip this entry, so rotation ends when the pool + // itself reports exhaustion rather than after a fixed number of tries + await deps.parkToken(token.id, resumeAt) + // TODO(CM-1372): temporary — drop once the staging secondary-limit parks are explained + deps.log.info( + { + event: 'pool_rate_limit_detail', + tokenId: token.id, + resumeAt, + resumeSource: error.options?.resumeAt ? 'error' : 'headers', + status: response.status, + reason: error.message, + resource: headers['x-ratelimit-resource'], + limit: headers['x-ratelimit-limit'], + remaining: headers['x-ratelimit-remaining'], + used: headers['x-ratelimit-used'], + reset: headers['x-ratelimit-reset'], + retryAfter: headers['retry-after'], + body: rateLimitBodySummary(response.data), + }, + 'token rate limited, retrying with fresh token', + ) + return attemptRequest(deps, config, authRetried) + } + + if (error.errorClass === 'provider.auth') { + await deps.invalidateToken(token.id) + deps.log.warn( + { tokenId: token.id, status: response.status }, + 'token invalidated on auth failure', + ) + if (!authRetried) { + return attemptRequest(deps, config, true) + } + } + + throw error +} + +const BODY_SUMMARY_MAX_LENGTH = 200 + +function rateLimitBodySummary(data: unknown): string | undefined { + if (!data || typeof data !== 'object') { + return undefined + } + const body = data as { message?: string; errors?: { type?: string; message?: string }[] } + if (body.message) { + return body.message.slice(0, BODY_SUMMARY_MAX_LENGTH) + } + const first = body.errors?.[0] + if (!first) { + return undefined + } + return `${first.type ?? 'ERROR'}: ${first.message ?? ''}`.slice(0, BODY_SUMMARY_MAX_LENGTH) +} + +async function send( + deps: CountingHttpClientDeps, + config: AxiosRequestConfig, + token: IPooledToken, +): Promise> { + const applyToken = deps.applyToken ?? applyBearerToken + const authenticatedConfig = { + timeout: DEFAULT_TIMEOUT_MS, + ...applyToken(config, token), + validateStatus: () => true, + } + deps.countRequest() + try { + return await axios.request(authenticatedConfig) + } catch (err) { + throw new ProviderUnavailableError('no response from provider', { cause: err }) + } +} + +function applyBearerToken(config: AxiosRequestConfig, token: IPooledToken): AxiosRequestConfig { + const headers = AxiosHeaders.from(config.headers) + headers.set('Authorization', `Bearer ${token.value}`) + return { ...config, headers } +} + +function classifyResponse( + deps: HttpClientDeps, + status: number, + headers: Record, + data: unknown, +): ConnectorError | null { + const custom = deps.interpretResponse?.({ status, headers, data }) + if (custom) { + return custom + } + if (isRateLimited(status, headers)) { + return new RateLimitError(`provider rate limited (status ${status})`, { + status, + resumeAt: computeResumeAt(headers), + }) + } + if (status >= 300) { + return errorFromHttpStatus(status) + } + return null +} + +function isRateLimited(status: number, headers: Record): boolean { + if (status === 429) { + return true + } + return ( + status === 403 && + (headers['x-ratelimit-remaining'] === '0' || headers['retry-after'] !== undefined) + ) +} + +function computeResumeAt(headers: Record): Date { + const retryAfter = headers['retry-after'] + const retryAfterSeconds = Number(retryAfter) + if (Number.isFinite(retryAfterSeconds) && retryAfterSeconds > 0) { + return new Date(Date.now() + retryAfterSeconds * 1000) + } + const retryAfterDateMs = Date.parse(retryAfter ?? '') + if (Number.isFinite(retryAfterDateMs) && retryAfterDateMs > Date.now()) { + return new Date(retryAfterDateMs) + } + // x-ratelimit-reset describes the primary bucket only. Honouring it on a + // secondary/abuse block parks the token for the rest of the hour instead of + // the minute or so the block actually lasts. + if (headers['x-ratelimit-remaining'] === '0') { + const resetEpochSeconds = Number(headers['x-ratelimit-reset']) + if (Number.isFinite(resetEpochSeconds) && resetEpochSeconds * 1000 > Date.now()) { + return new Date(resetEpochSeconds * 1000) + } + } + return new Date(Date.now() + RATE_LIMIT_FALLBACK_MS) +} + +function normalizeHeaders(raw: AxiosResponse['headers']): Record { + const headers: Record = {} + for (const [key, value] of Object.entries(raw)) { + if (value !== undefined && value !== null) { + headers[key.toLowerCase()] = Array.isArray(value) ? value.join(', ') : String(value) + } + } + return headers +} diff --git a/services/libs/connectors/src/http/errors.ts b/services/libs/connectors/src/http/errors.ts new file mode 100644 index 0000000000..bce5f79591 --- /dev/null +++ b/services/libs/connectors/src/http/errors.ts @@ -0,0 +1,84 @@ +export type ErrorClass = + | 'provider.unavailable' + | 'provider.rate_limit' + | 'provider.auth' + | 'provider.contract' + | 'connector.code' + | 'sink.rejected' + | 'unknown' + +export interface ConnectorErrorOptions { + status?: number + resumeAt?: Date + cause?: unknown +} + +export class ConnectorError extends Error { + constructor( + readonly errorClass: ErrorClass, + message: string, + readonly options?: ConnectorErrorOptions, + ) { + super(message) + this.name = 'ConnectorError' + } +} + +export class ProviderUnavailableError extends ConnectorError { + constructor(message = 'provider unavailable', options?: ConnectorErrorOptions) { + super('provider.unavailable', message, options) + this.name = 'ProviderUnavailableError' + } +} + +export class RateLimitError extends ConnectorError { + constructor(message = 'rate limited by provider', options?: ConnectorErrorOptions) { + super('provider.rate_limit', message, options) + this.name = 'RateLimitError' + } +} + +export class ProviderAuthError extends ConnectorError { + constructor(message = 'provider authentication failed', options?: ConnectorErrorOptions) { + super('provider.auth', message, options) + this.name = 'ProviderAuthError' + } +} + +export class ProviderContractError extends ConnectorError { + constructor(message = 'unexpected provider response', options?: ConnectorErrorOptions) { + super('provider.contract', message, options) + this.name = 'ProviderContractError' + } +} + +export class ConnectorCodeError extends ConnectorError { + constructor(message = 'connector code error', options?: ConnectorErrorOptions) { + super('connector.code', message, options) + this.name = 'ConnectorCodeError' + } +} + +export function errorFromHttpStatus( + status: number | undefined, + message?: string, + options?: ConnectorErrorOptions, +): ConnectorError { + const opts = { ...options, status } + if (status === undefined) { + return new ProviderUnavailableError(message ?? 'no response from provider', opts) + } + if (status === 401 || status === 403) { + return new ProviderAuthError(message ?? `provider returned status ${status}`, opts) + } + if (status === 429) { + return new RateLimitError(message ?? `provider returned status ${status}`, opts) + } + if (status >= 500) { + return new ProviderUnavailableError(message ?? `provider returned status ${status}`, opts) + } + if (status >= 300) { + return new ProviderContractError(message ?? `provider returned status ${status}`, opts) + } + return new ConnectorError('unknown', message ?? `unexpected status ${status}`, opts) +} diff --git a/services/libs/connectors/src/index.ts b/services/libs/connectors/src/index.ts new file mode 100644 index 0000000000..520162b5c2 --- /dev/null +++ b/services/libs/connectors/src/index.ts @@ -0,0 +1,8 @@ +export * from './concurrency' +export * from './credentials' +export * from './emit' +export * from './http/client' +export * from './http/errors' +export * from './pool/tokenPool' +export * from './registry' +export * from './types' diff --git a/services/libs/connectors/src/pool/tokenPool.ts b/services/libs/connectors/src/pool/tokenPool.ts new file mode 100644 index 0000000000..53fe397eda --- /dev/null +++ b/services/libs/connectors/src/pool/tokenPool.ts @@ -0,0 +1,376 @@ +import type { Logger } from '@crowd/logging' +import type { RedisClient } from '@crowd/redis' + +import type { IPooledToken } from '../http/client' +import { ConnectorError, ProviderAuthError, RateLimitError } from '../http/errors' + +const PROBE_STALENESS_MS = 90_000 +const TOKEN_EXPIRY_MARGIN_MS = 120_000 + +export interface BudgetSnapshot { + limit: number + remaining: number + resetAt: Date +} + +export type BudgetProbe = (platform: string, token: IPooledToken) => Promise + +export type TokenMinter = (entryId: string) => Promise<{ token: string; expiresAt: string }> + +// POC only: the probe is the single source of truth for budgets (github /rate_limit is free and +// limits are per installation, surviving token re-mints); other platforms are a later decision. +export interface TokenPoolOptions { + probeBudget?: BudgetProbe + mintToken?: TokenMinter + log?: Logger +} + +export interface TokenPool { + acquire(preferredEntryId?: string): Promise + hasHeadroom(estimate: number): Promise + park(entryId: string, resumeAt: Date): Promise + invalidate(entryId: string): Promise + quarantine(entryId: string): Promise + seed(entryIds: string[]): Promise + needsSeed(maxAgeMs: number): Promise + earliestResumeAt(): Promise +} + +interface IEntryState { + token?: string + tokenExpiresAtMs?: number + parkedUntil?: string + quarantined?: boolean +} + +type EntryStateUpdate = { [K in keyof IEntryState]?: IEntryState[K] | null } + +// atomic get-merge-set per entry; JSON null in the update deletes the field +const MERGE_ENTRY_STATE_SCRIPT = ` +local json = redis.call('HGET', KEYS[1], ARGV[1]) +if not json then + return 0 +end +local state = cjson.decode(json) +local update = cjson.decode(ARGV[2]) +for field, value in pairs(update) do + if value == cjson.null then + state[field] = nil + else + state[field] = value + end +end +redis.call('HSET', KEYS[1], ARGV[1], cjson.encode(state)) +return 1 +` + +interface IBucket { + limit: number + remaining: number + resetAtMs: number + probedAtMs: number +} + +export function createTokenPool( + redis: RedisClient, + platform: string, + options?: TokenPoolOptions, +): TokenPool { + const entriesKey = `connectors:pool:${platform}:entries` + const lruKey = `connectors:pool:${platform}:lru` + const seededAtKey = `connectors:pool:${platform}:seededAt` + const log = options?.log + + const loggedSkips = new Set() + let lastAcquiredEntryId: string | null = null + + function logSkip(entryId: string, reason: string, fields: Record = {}): void { + const key = `${entryId}:${reason}` + if (!log || loggedSkips.has(key)) { + return + } + loggedSkips.add(key) + log.info({ event: 'pool_entry_skipped', entryId, reason, ...fields }, 'pool entry skipped') + } + + function logAcquired(entryId: string, preferredEntryId?: string): void { + if (!log || lastAcquiredEntryId === entryId) { + return + } + lastAcquiredEntryId = entryId + log.info( + { + event: 'pool_token_acquired', + entryId, + preferredEntryId, + borrowed: preferredEntryId ? entryId !== preferredEntryId : false, + }, + 'pool token acquired', + ) + } + const bucketKey = (entryId: string) => `connectors:pool:${platform}:budget:${entryId}` + + async function readStates(): Promise> { + const raw = await redis.hGetAll(entriesKey) + const states = new Map() + for (const [id, json] of Object.entries(raw)) { + states.set(id, JSON.parse(json) as IEntryState) + } + return states + } + + function isHealthy(state: IEntryState, nowMs: number): boolean { + if (state.quarantined) { + return false + } + if (state.parkedUntil && new Date(state.parkedUntil).getTime() > nowMs) { + return false + } + return true + } + + function usableToken(state: IEntryState, nowMs: number): string | null { + if (!state.token || !state.tokenExpiresAtMs) { + return null + } + if (state.tokenExpiresAtMs - TOKEN_EXPIRY_MARGIN_MS <= nowMs) { + return null + } + return state.token + } + + function earliestParkedUntil(states: Map, nowMs: number): Date | null { + let earliest: Date | null = null + for (const state of states.values()) { + if (state.quarantined || !state.parkedUntil) { + continue + } + const parkedUntil = new Date(state.parkedUntil) + if (parkedUntil.getTime() <= nowMs) { + continue + } + if (!earliest || parkedUntil < earliest) { + earliest = parkedUntil + } + } + return earliest + } + + async function updateState(entryId: string, update: EntryStateUpdate): Promise { + await redis.eval(MERGE_ENTRY_STATE_SCRIPT, { + keys: [entriesKey], + arguments: [entryId, JSON.stringify(update)], + }) + } + + async function ensureUsableToken( + entryId: string, + state: IEntryState, + nowMs: number, + ): Promise { + const cached = usableToken(state, nowMs) + if (cached) { + return cached + } + const mint = options?.mintToken + if (!mint) { + return null + } + try { + const { token, expiresAt } = await mint(entryId) + await updateState(entryId, { token, tokenExpiresAtMs: new Date(expiresAt).getTime() }) + log?.info({ event: 'pool_token_minted', entryId, expiresAt }, 'pool token minted') + return token + } catch (err) { + if (err instanceof ConnectorError) { + if (err.errorClass === 'provider.auth' || err.errorClass === 'provider.contract') { + await updateState(entryId, { quarantined: true }) + log?.warn( + { + event: 'pool_entry_quarantined', + entryId, + errorClass: err.errorClass, + errMsg: err.message, + }, + 'pool entry quarantined on mint failure', + ) + return null + } + log?.warn( + { event: 'pool_mint_failed', entryId, errorClass: err.errorClass, errMsg: err.message }, + 'token mint failed, skipping entry', + ) + return null + } + throw err + } + } + + async function readBucket(entryId: string): Promise { + const raw = await redis.hGetAll(bucketKey(entryId)) + if (!raw.probedAt) { + return null + } + return { + limit: Number(raw.limit), + remaining: Number(raw.remaining), + resetAtMs: Number(raw.resetAt), + probedAtMs: Number(raw.probedAt), + } + } + + function needsProbe(bucket: IBucket | null, nowMs: number): boolean { + return !bucket || nowMs - bucket.probedAtMs > PROBE_STALENESS_MS || nowMs >= bucket.resetAtMs + } + + async function loadBucket( + probe: BudgetProbe, + entryId: string, + token: string | null, + nowMs: number, + ): Promise { + const bucket = await readBucket(entryId) + if (!needsProbe(bucket, nowMs)) { + return bucket + } + if (!token) { + return null + } + const snapshot = await probe(platform, { id: entryId, value: token }) + if (!snapshot) { + return null + } + const probed = { + limit: snapshot.limit, + remaining: snapshot.remaining, + resetAtMs: snapshot.resetAt.getTime(), + probedAtMs: nowMs, + } + await redis.hSet(bucketKey(entryId), { + limit: String(probed.limit), + remaining: String(probed.remaining), + resetAt: String(probed.resetAtMs), + probedAt: String(probed.probedAtMs), + }) + return probed + } + + return { + async acquire(preferredEntryId?: string): Promise { + const nowMs = Date.now() + const states = await readStates() + const lruOrder = await redis.zRange(lruKey, 0, -1) + const ordered = + preferredEntryId && states.has(preferredEntryId) + ? [preferredEntryId, ...lruOrder.filter((id) => id !== preferredEntryId)] + : lruOrder + const probe = options?.probeBudget + let earliestBudgetResetAt: Date | null = null + for (const id of ordered) { + const state = states.get(id) + if (!state) { + continue + } + if (state.quarantined) { + logSkip(id, 'quarantined') + continue + } + if (state.parkedUntil && new Date(state.parkedUntil).getTime() > nowMs) { + logSkip(id, 'parked', { parkedUntil: state.parkedUntil }) + continue + } + const token = await ensureUsableToken(id, state, nowMs) + if (!token) { + continue + } + if (probe) { + const bucket = await loadBucket(probe, id, token, nowMs) + if (bucket && bucket.remaining <= 0) { + const resetAt = new Date(bucket.resetAtMs) + if (!earliestBudgetResetAt || resetAt < earliestBudgetResetAt) { + earliestBudgetResetAt = resetAt + } + logSkip(id, 'budget_exhausted', { resetAt }) + continue + } + if (bucket) { + await redis.hIncrBy(bucketKey(id), 'remaining', -1) + } + } + await redis.zAdd(lruKey, { score: nowMs, value: id }) + logAcquired(id, preferredEntryId) + return { id, value: token } + } + const parkedResumeAt = earliestParkedUntil(states, nowMs) + const resumeAt = + parkedResumeAt && earliestBudgetResetAt + ? new Date(Math.min(parkedResumeAt.getTime(), earliestBudgetResetAt.getTime())) + : (parkedResumeAt ?? earliestBudgetResetAt) + if (resumeAt) { + throw new RateLimitError('token pool exhausted', { resumeAt }) + } + throw new ProviderAuthError('token pool empty') + }, + + async hasHeadroom(estimate: number): Promise { + const probe = options?.probeBudget + if (!probe) { + return true + } + const nowMs = Date.now() + const states = await readStates() + const healthy = [...states.entries()].filter(([, state]) => isHealthy(state, nowMs)) + if (healthy.length === 0) { + return states.size === 0 + } + let pooledRemaining = 0 + for (const [id, state] of healthy) { + const bucket = await loadBucket(probe, id, usableToken(state, nowMs), nowMs) + if (!bucket) { + return true + } + pooledRemaining += bucket.remaining + if (pooledRemaining >= estimate) { + return true + } + } + return false + }, + + async park(entryId: string, resumeAt: Date): Promise { + await updateState(entryId, { parkedUntil: resumeAt.toISOString() }) + }, + + async invalidate(entryId: string): Promise { + await updateState(entryId, { token: null, tokenExpiresAtMs: null }) + }, + + // POC only: quarantined entries are kept for inspection and never revived automatically + async quarantine(entryId: string): Promise { + await updateState(entryId, { quarantined: true }) + }, + + async seed(entryIds: string[]): Promise { + if (entryIds.length === 0) { + return + } + const multi = redis.multi() + for (const id of entryIds) { + multi.hSetNX(entriesKey, id, '{}') + multi.zAdd(lruKey, { score: 0, value: id }, { NX: true }) + } + multi.set(seededAtKey, String(Date.now())) + await multi.exec() + }, + + async needsSeed(maxAgeMs: number): Promise { + const seededAt = await redis.get(seededAtKey) + return !seededAt || Date.now() - Number(seededAt) > maxAgeMs + }, + + async earliestResumeAt(): Promise { + const states = await readStates() + return earliestParkedUntil(states, Date.now()) + }, + } +} diff --git a/services/libs/connectors/src/registry.ts b/services/libs/connectors/src/registry.ts new file mode 100644 index 0000000000..d88ddb68db --- /dev/null +++ b/services/libs/connectors/src/registry.ts @@ -0,0 +1,31 @@ +import type { Manifest, SyncDefinition } from './types' + +const manifests = new Map() + +export function registerConnector(manifest: Manifest): void { + manifests.set(manifest.platform, manifest) +} + +export function findManifest(platform: string): Manifest | undefined { + return manifests.get(platform) +} + +export function getManifest(platform: string): Manifest { + const manifest = manifests.get(platform) + if (!manifest) { + throw new Error(`unknown platform ${platform}`) + } + return manifest +} + +export function findSync(platform: string, syncName: string): SyncDefinition | undefined { + return findManifest(platform)?.syncs.find((s) => s.name === syncName) +} + +export function getSync(platform: string, syncName: string): SyncDefinition { + const sync = getManifest(platform).syncs.find((s) => s.name === syncName) + if (!sync) { + throw new Error(`unknown sync ${platform}/${syncName}`) + } + return sync +} diff --git a/services/libs/connectors/src/testing/dummyConnector.ts b/services/libs/connectors/src/testing/dummyConnector.ts new file mode 100644 index 0000000000..93ef346ec9 --- /dev/null +++ b/services/libs/connectors/src/testing/dummyConnector.ts @@ -0,0 +1,22 @@ +import { z } from 'zod' + +import type { Manifest, SyncContext } from '../types' + +const TICK_COUNT = 3 + +export const dummyConnector: Manifest = { + platform: 'dummy', + syncs: [ + { + name: 'ticks', + cadenceMinutes: 60, + schema: z.record(z.unknown()), + run: async (ctx: SyncContext) => { + await ctx.emit(Array.from({ length: TICK_COUNT }, (_, index) => ({ tick: index }))) + await ctx.commitWatermark({ since: new Date().toISOString() }) + return { complete: true } + }, + }, + ], + discover: async () => [{ channelId: 'dummy-channel', channelName: 'dummy/channel' }], +} diff --git a/services/libs/connectors/src/types.ts b/services/libs/connectors/src/types.ts new file mode 100644 index 0000000000..d5cd5f8c0a --- /dev/null +++ b/services/libs/connectors/src/types.ts @@ -0,0 +1,59 @@ +import type { ZodType } from 'zod' + +import type { Logger } from '@crowd/logging' + +import type { ConnectorHttp, ResponseInterpreter } from './http/client' +import type { BudgetProbe, TokenMinter, TokenPool } from './pool/tokenPool' + +export interface Channel { + channelId: string + channelName: string +} + +export interface GithubAppCredentialData { + appId: string + privateKey: string +} + +// POC only: single variant; becomes a discriminated union (token, oauth2, ...) +// as more connectors land +export interface Credential { + platform: string + kind: 'github-app' + data: GithubAppCredentialData +} + +export interface SyncContext { + channel: Channel + watermark: Record | null + emit: (records: unknown[]) => Promise + commitWatermark: (watermark: Record) => Promise + hasRunBudget: () => boolean + http: ConnectorHttp + log: Logger +} + +export interface SyncOutcome { + complete: boolean +} + +export interface SyncDefinition { + name: string + cadenceMinutes: number + schema: ZodType> + run: (ctx: SyncContext) => Promise +} + +export interface PoolPreparation { + preferredEntryId?: string +} + +export interface Manifest { + platform: string + syncs: SyncDefinition[] + discover: (credential: Credential) => Promise + preparePool?: (credential: Credential, pool: TokenPool) => Promise + mintToken?: (credential: Credential) => TokenMinter + probeBudget?: BudgetProbe + interpretResponse?: ResponseInterpreter +} diff --git a/services/libs/connectors/tsconfig.json b/services/libs/connectors/tsconfig.json new file mode 100644 index 0000000000..bf7f183850 --- /dev/null +++ b/services/libs/connectors/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../base.tsconfig.json", + "include": ["src/**/*"] +} diff --git a/services/libs/data-access-layer/src/connectors/index.ts b/services/libs/data-access-layer/src/connectors/index.ts new file mode 100644 index 0000000000..ee1244f4c2 --- /dev/null +++ b/services/libs/data-access-layer/src/connectors/index.ts @@ -0,0 +1,2 @@ +export * from './syncUnits' +export * from './types' diff --git a/services/libs/data-access-layer/src/connectors/syncUnits.ts b/services/libs/data-access-layer/src/connectors/syncUnits.ts new file mode 100644 index 0000000000..22f9634335 --- /dev/null +++ b/services/libs/data-access-layer/src/connectors/syncUnits.ts @@ -0,0 +1,211 @@ +import type { QueryExecutor } from '../queryExecutor' + +import type { + IClaimedUnit, + ISyncRunProgress, + ISyncRunSuccess, + ISyncUnit, + SyncUnitUpsert, +} from './types' + +const MIN_INITIAL_DELAY_SECONDS = 10 +const MAX_INITIAL_DELAY_SECONDS = 900 +const CLAIM_LEASE_MINUTES = 5 +const ERROR_MESSAGE_MAX_LENGTH = 500 + +function truncateErrorMessage(message: string | null): string | null { + if (!message) { + return null + } + return message.slice(0, ERROR_MESSAGE_MAX_LENGTH) +} + +export async function upsertSyncUnits(qx: QueryExecutor, units: SyncUnitUpsert[]): Promise { + if (units.length === 0) { + return 0 + } + + return qx.result( + `INSERT INTO integration.sync_units + ("integrationId", platform, "channelId", "channelName", "syncName", "nextRunAt") + SELECT u.*, now() + ($(minDelaySeconds) + random() * $(delaySpanSeconds)) * interval '1 second' + FROM unnest( + $(integrationIds)::uuid[], + $(platforms)::text[], + $(channelIds)::text[], + $(channelNames)::text[], + $(syncNames)::text[] + ) u + ON CONFLICT ("integrationId", "channelId", "syncName") + DO UPDATE SET "channelName" = EXCLUDED."channelName", "updatedAt" = now()`, + { + integrationIds: units.map((u) => u.integrationId), + platforms: units.map((u) => u.platform), + channelIds: units.map((u) => u.channelId), + channelNames: units.map((u) => u.channelName), + syncNames: units.map((u) => u.syncName), + minDelaySeconds: MIN_INITIAL_DELAY_SECONDS, + delaySpanSeconds: MAX_INITIAL_DELAY_SECONDS - MIN_INITIAL_DELAY_SECONDS, + }, + ) +} + +export async function claimDueUnits(qx: QueryExecutor, limit: number): Promise { + return qx.select( + `UPDATE integration.sync_units su + SET "lockedAt" = now(), "updatedAt" = now() + WHERE su.id IN ( + SELECT su2.id + FROM integration.sync_units su2 + WHERE su2.status = 'active' + AND su2."nextRunAt" <= now() + AND (su2."lockedAt" IS NULL OR su2."lockedAt" < now() - $(leaseMinutes) * interval '1 minute') + AND EXISTS ( + SELECT 1 + FROM public.integrations i + WHERE i.id = su2."integrationId" AND i."deletedAt" IS NULL + ) + ORDER BY su2."nextRunAt" + LIMIT $(limit) + FOR UPDATE SKIP LOCKED + ) + RETURNING su.id, su."integrationId", su.platform, su."syncName", su."channelId", su."channelName"`, + { limit, leaseMinutes: CLAIM_LEASE_MINUTES }, + ) +} + +export async function rescheduleUnit( + qx: QueryExecutor, + id: string, + nextRunAt: Date, +): Promise { + await qx.result( + `UPDATE integration.sync_units + SET "nextRunAt" = $(nextRunAt), "lockedAt" = NULL, "updatedAt" = now() + WHERE id = $(id)`, + { id, nextRunAt }, + ) +} + +export async function recordRunSuccess( + qx: QueryExecutor, + id: string, + data: ISyncRunSuccess, + nextRunAt: Date, +): Promise { + await qx.result( + `UPDATE integration.sync_units + SET watermark = $(watermark)::jsonb, + "emittedCount" = $(emittedCount), + "nextRunAt" = $(nextRunAt), + "lastRunAt" = now(), + "lastSuccessAt" = now(), + "consecutiveFailures" = 0, + "lastErrorClass" = NULL, + "lastErrorMessage" = NULL, + "lastRunComplete" = $(complete), + "lockedAt" = NULL, + "updatedAt" = now() + WHERE id = $(id)`, + { + id, + watermark: JSON.stringify(data.watermark), + emittedCount: data.emittedCount, + complete: data.complete, + nextRunAt, + }, + ) +} + +export async function recordRunPartial( + qx: QueryExecutor, + id: string, + progress: ISyncRunProgress, + resumeAt: Date, + errorClass: string, + errorMessage: string | null, +): Promise { + await qx.result( + `UPDATE integration.sync_units + SET watermark = $(watermark)::jsonb, + "emittedCount" = $(emittedCount), + "nextRunAt" = $(resumeAt), + "lastRunAt" = now(), + "lastErrorClass" = $(errorClass), + "lastErrorMessage" = $(errorMessage), + "lastRunComplete" = false, + "lockedAt" = NULL, + "updatedAt" = now() + WHERE id = $(id)`, + { + id, + watermark: JSON.stringify(progress.watermark), + emittedCount: progress.emittedCount, + resumeAt, + errorClass, + errorMessage: truncateErrorMessage(errorMessage), + }, + ) +} + +export async function parkUnit( + qx: QueryExecutor, + id: string, + resumeAt: Date, + errorClass: string, + errorMessage: string | null, +): Promise { + await qx.result( + `UPDATE integration.sync_units + SET "nextRunAt" = $(resumeAt), + "lastRunAt" = now(), + "lastErrorClass" = $(errorClass), + "lastErrorMessage" = $(errorMessage), + "lastRunComplete" = false, + "lockedAt" = NULL, + "updatedAt" = now() + WHERE id = $(id)`, + { id, resumeAt, errorClass, errorMessage: truncateErrorMessage(errorMessage) }, + ) +} + +export async function recordRunFailure( + qx: QueryExecutor, + id: string, + errorClass: string, + errorMessage: string | null, + deadLetterAfter: number | null, + nextRunAt: Date, +): Promise { + await qx.result( + `UPDATE integration.sync_units + SET "consecutiveFailures" = "consecutiveFailures" + 1, + "lastErrorClass" = $(errorClass), + "lastErrorMessage" = $(errorMessage), + "lastRunComplete" = false, + "lastRunAt" = now(), + "nextRunAt" = $(nextRunAt), + "lockedAt" = NULL, + status = CASE WHEN $(deadLetterAfter)::int IS NOT NULL + AND "consecutiveFailures" + 1 >= $(deadLetterAfter) + THEN 'dead_letter' ELSE status END, + "updatedAt" = now() + WHERE id = $(id)`, + { + id, + errorClass, + errorMessage: truncateErrorMessage(errorMessage), + deadLetterAfter, + nextRunAt, + }, + ) +} + +export async function getUnitById(qx: QueryExecutor, id: string): Promise { + return qx.selectOneOrNone( + `SELECT * + FROM integration.sync_units + WHERE id = $(id)`, + { id }, + ) +} diff --git a/services/libs/data-access-layer/src/connectors/types.ts b/services/libs/data-access-layer/src/connectors/types.ts new file mode 100644 index 0000000000..483e8b0bd3 --- /dev/null +++ b/services/libs/data-access-layer/src/connectors/types.ts @@ -0,0 +1,40 @@ +export type SyncUnitStatus = 'active' | 'paused' | 'dead_letter' | 'decommissioned' + +export interface ISyncUnit { + id: string + integrationId: string + platform: string + channelId: string + channelName: string + syncName: string + status: SyncUnitStatus + nextRunAt: string + lockedAt: string | null + lastRunAt: string | null + lastSuccessAt: string | null + consecutiveFailures: number + lastErrorClass: string | null + lastErrorMessage: string | null + lastRunComplete: boolean | null + watermark: Record | null + emittedCount: number | null +} + +export type SyncUnitUpsert = Pick< + ISyncUnit, + 'integrationId' | 'platform' | 'channelId' | 'channelName' | 'syncName' +> + +export type IClaimedUnit = Pick< + ISyncUnit, + 'id' | 'integrationId' | 'platform' | 'syncName' | 'channelId' | 'channelName' +> + +export interface ISyncRunProgress { + watermark: Record + emittedCount: number +} + +export interface ISyncRunSuccess extends ISyncRunProgress { + complete: boolean +} diff --git a/services/libs/data-access-layer/src/index.ts b/services/libs/data-access-layer/src/index.ts index 9079622755..7424ddcdd4 100644 --- a/services/libs/data-access-layer/src/index.ts +++ b/services/libs/data-access-layer/src/index.ts @@ -13,6 +13,7 @@ export * from './repositories' export * from './security_insights' export * from './segments' export * from './systemSettings' +export * from './connectors' export * from './integrations' export * from './auditLogs' export * from './maintainers'