[STRATCONN-5657] - Add batch keys to webhook action destination (#2878)

* [STRATCONN-5657] - Add batch keys to webhook action destination

* Add stats

* Use histogram

* Add test for batch keys

Author: varadarajan-tw

packages/destination-actions/src/destinations/webhook-audiences/index.ts

@@ -139,9 +139,18 @@ const destination: AudienceDestinationDefinition<Settings, AudienceSettings> = {
           settings
         })
       },
-      performBatch: (request, { payload, settings, audienceSettings }) => {
+      performBatch: (request, { payload, settings, audienceSettings, statsContext }) => {
         const extras = parseExtraSettingsJson(audienceSettings?.extras)
 
+        if (statsContext) {
+          const { statsClient, tags } = statsContext
+          const set = new Set()
+          for (const p of payload) {
+            set.add(`${p.url} ${p.method} ${JSON.stringify(p.headers)}`)
+          }
+          statsClient?.histogram('webhook.configurable_batch_keys.unique_keys', set.size, tags)
+        }
+
         // Call the same performBatch function from the regular webhook destination
         // and add in our extraSettings
         return send.performBatch!(request, {

packages/destination-actions/src/destinations/webhook-audiences/send.types.ts

@@ -25,4 +25,8 @@ export interface Payload {
   data?: {
     [k: string]: unknown
   }
+  /**
+   * The mapping keys to batch events together by.
+   */
+  batch_keys?: string[]
 }

packages/destination-actions/src/destinations/webhook/__test__/webhook.test.ts

@@ -177,6 +177,77 @@ export const baseWebhookTests = (def: DestinationDefinition<any>) => {
           })
         ).rejects.toThrow(PayloadValidationError)
       })
+
+      it('should incr stats by unique url, headers and method count', async () => {
+        const testEvent1 = {
+          properties: {
+            url: 'https://example.build',
+            method: 'POST',
+            headers: {
+              'sample-header': 'value1'
+            }
+          }
+        }
+        const testEvent2 = {
+          properties: {
+            url: 'https://example.build',
+            method: 'PUT',
+            headers: {
+              'sample-header': 'value1'
+            }
+          }
+        }
+        const events = [createTestEvent(testEvent1), createTestEvent(testEvent2)]
+
+        nock(testEvent1.properties.url).post('/').reply(200, { success: true })
+
+        const statsClient = {
+          incr: jest.fn(),
+          observe: jest.fn(),
+          _name: jest.fn(),
+          _tags: jest.fn(),
+          set: jest.fn,
+          histogram: jest.fn()
+        }
+        const statsContext = {
+          statsClient,
+          tags: ['test:tag']
+        }
+
+        const responses = await testDestination.testBatchAction('send', {
+          events,
+          mapping: {
+            url: {
+              '@path': '$.properties.url'
+            },
+            method: {
+              '@path': '$.properties.method'
+            },
+            headers: {
+              'sample-header': {
+                '@path': '$.properties.sample-header'
+              }
+            }
+          },
+          useDefaultMappings: true,
+          statsContext
+        })
+
+        expect(responses.length).toBe(1)
+        expect(responses[0].status).toBe(200)
+        expect(statsClient.histogram).toHaveBeenCalledWith(
+          'webhook.configurable_batch_keys.unique_keys',
+          2,
+          statsContext.tags
+        )
+      })
+    })
+
+    it('should have batch keys default to url, method, headers', () => {
+      const action = testDestination.definition.actions.send
+      expect(action.fields.batch_keys).toBeDefined()
+      expect(action.fields.batch_keys?.default).toBeDefined()
+      expect(action.fields.batch_keys?.default).toEqual(['url', 'method', 'headers'])
     })
   })
 }

packages/destination-actions/src/destinations/webhook/send/generated-types.ts

@@ -45,6 +45,15 @@ const action: ActionDefinition<Settings, Payload> = {
       description: 'Payload to deliver to webhook URL (JSON-encoded).',
       type: 'object',
       default: { '@path': '$.' }
+    },
+    batch_keys: {
+      label: 'Batch Keys',
+      description: 'The mapping keys to batch events together by.',
+      type: 'string',
+      multiple: true,
+      required: false,
+      unsafe_hidden: true,
+      default: ['url', 'method', 'headers']
     }
   },
   perform: (request, { payload }) => {
@@ -59,9 +68,19 @@ const action: ActionDefinition<Settings, Payload> = {
       throw error
     }
   },
-  performBatch: (request, { payload }) => {
+  performBatch: (request, { payload, statsContext }) => {
     // Expect these to be the same across the payloads
     const { url, method, headers } = payload[0]
+
+    if (statsContext) {
+      const { statsClient, tags } = statsContext
+      const set = new Set()
+      for (const p of payload) {
+        set.add(`${p.url} ${p.method} ${JSON.stringify(p.headers)}`)
+      }
+      statsClient?.histogram('webhook.configurable_batch_keys.unique_keys', set.size, tags)
+    }
+
     try {
       return request(url, {
         method: method as RequestMethod,