Process on-chain-received DApi batches

src/condition-check/condition-check.test.ts

@@ -2,6 +2,7 @@ import { BigNumber, ethers } from 'ethers';
 
 import { getUnixTimestamp } from '../../test/utils';
 import { HUNDRED_PERCENT } from '../constants';
+import { getDeviationThresholdAsBigNumber } from '../utils';
 
 import {
   calculateMedian,
@@ -16,25 +17,39 @@ describe('checkUpdateCondition', () => {
   const onChainValue = ethers.BigNumber.from(500);
 
   it('returns true when api value is higher and deviation threshold is reached', () => {
-    const shouldUpdate = checkDeviationThresholdExceeded(onChainValue, 10, ethers.BigNumber.from(560));
+    const shouldUpdate = checkDeviationThresholdExceeded(
+      onChainValue,
+      getDeviationThresholdAsBigNumber(10),
+      ethers.BigNumber.from(560)
+    );
 
     expect(shouldUpdate).toBe(true);
   });
 
   it('returns true when api value is lower and deviation threshold is reached', () => {
-    const shouldUpdate = checkDeviationThresholdExceeded(onChainValue, 10, ethers.BigNumber.from(440));
+    const shouldUpdate = checkDeviationThresholdExceeded(
+      onChainValue,
+      getDeviationThresholdAsBigNumber(10),
+      ethers.BigNumber.from(440)
+    );
 
     expect(shouldUpdate).toBe(true);
   });
 
   it('returns false when deviation threshold is not reached', () => {
-    const shouldUpdate = checkDeviationThresholdExceeded(onChainValue, 10, ethers.BigNumber.from(480));
+    const shouldUpdate = checkDeviationThresholdExceeded(
+      onChainValue,
+      getDeviationThresholdAsBigNumber(10),
+      ethers.BigNumber.from(480)
+    );
 
     expect(shouldUpdate).toBe(false);
   });
 
   it('handles correctly bad JS math', () => {
-    expect(() => checkDeviationThresholdExceeded(onChainValue, 0.14, ethers.BigNumber.from(560))).not.toThrow();
+    expect(() =>
+      checkDeviationThresholdExceeded(onChainValue, getDeviationThresholdAsBigNumber(0.14), ethers.BigNumber.from(560))
+    ).not.toThrow();
   });
 
   it('checks all update conditions | heartbeat exceeded', () => {
@@ -44,7 +59,7 @@ describe('checkUpdateCondition', () => {
       BigNumber.from(10),
       Date.now() / 1000,
       60 * 60 * 23,
-      2
+      getDeviationThresholdAsBigNumber(2)
     );
 
     expect(result).toBe(true);
@@ -57,7 +72,7 @@ describe('checkUpdateCondition', () => {
       BigNumber.from(10),
       Date.now() + 60 * 60 * 23,
       86_400,
-      2
+      getDeviationThresholdAsBigNumber(2)
     );
 
     expect(result).toBe(false);

src/condition-check/condition-check.ts

@@ -1,4 +1,4 @@
-import { ethers } from 'ethers';
+import { type BigNumber, ethers } from 'ethers';
 
 import { HUNDRED_PERCENT } from '../constants';
 import { logger } from '../logger';
@@ -25,17 +25,21 @@ export const calculateMedian = (arr: ethers.BigNumber[]) => {
   return arr.length % 2 === 0 ? nums[mid - 1]!.add(nums[mid]!).div(2) : nums[mid];
 };
 
+/**
+ * Checks if the deviation threshold has been exceeded.
+ *
+ * @param onChainValue
+ * @param deviationThreshold Refer to getDeviationThresholdAsBigNumber()
+ * @param apiValue
+ */
 export const checkDeviationThresholdExceeded = (
   onChainValue: ethers.BigNumber,
-  deviationThreshold: number,
+  deviationThreshold: ethers.BigNumber,
   apiValue: ethers.BigNumber
 ) => {
   const updateInPercentage = calculateUpdateInPercentage(onChainValue, apiValue);
-  const threshold = ethers.BigNumber.from(Math.trunc(deviationThreshold * HUNDRED_PERCENT)).div(
-    ethers.BigNumber.from(100)
-  );
 
-  return updateInPercentage.gt(threshold);
+  return updateInPercentage.gt(deviationThreshold);
 };
 
 /**
@@ -60,7 +64,7 @@ export const checkUpdateConditions = (
   offChainValue: ethers.BigNumber,
   offChainTimestamp: number,
   heartbeatInterval: number,
-  deviationThreshold: number
+  deviationThreshold: BigNumber
 ): boolean => {
   // Check that fulfillment data is newer than on chain data
   const isFulfillmentDataFresh = checkFulfillmentDataTimestamp(onChainTimestamp, offChainTimestamp);

src/constants.ts

@@ -2,3 +2,7 @@ export const HTTP_SIGNED_DATA_API_ATTEMPT_TIMEOUT = 10_000;
 export const HTTP_SIGNED_DATA_API_HEADROOM = 1000;
 
 export const HUNDRED_PERCENT = 1e8;
+
+export const FEEDS_TO_UPDATE_CHUNK_SIZE = 10;
+
+export const SIGNED_URL_EXPIRY_IN_MINUTES = 5;

src/gas-price/gas-price.ts

@@ -1,5 +1,5 @@
 import { ethers } from 'ethers';
-import { remove } from 'lodash';
+import { get, remove } from 'lodash';
 
 import type { GasSettings } from '../config/schema';
 import { getState, updateState } from '../state';
@@ -70,7 +70,14 @@ export const clearSponsorLastUpdateTimestampMs = (
   sponsorWalletAddress: string
 ) =>
   updateState((draft) => {
-    delete draft.gasPriceStore[chainId]![providerName]!.sponsorLastUpdateTimestampMs[sponsorWalletAddress];
+    const sponsorLastUpdateTimestampMs = get(
+      draft,
+      `gasPriceStore[${chainId}][${providerName}].sponsorLastUpdateTimestampMs[${sponsorWalletAddress}]`
+    );
+
+    if (sponsorLastUpdateTimestampMs) {
+      delete draft.gasPriceStore[chainId]![providerName]!.sponsorLastUpdateTimestampMs[sponsorWalletAddress];
+    }
   });
 
 export const getPercentile = (percentile: number, array: ethers.BigNumber[]) => {

src/signed-api-fetch/data-fetcher.test.ts

@@ -16,12 +16,10 @@ describe('data fetcher', () => {
     localDataStore.clear();
     setState(
       produce(getState(), (draft) => {
-        draft.signedApiUrlStore = {
-          '31337': {
-            '0xbF3137b0a7574563a23a8fC8badC6537F98197CC': 'http://127.0.0.1:8090/',
-            '0xc52EeA00154B4fF1EbbF8Ba39FDe37F1AC3B9Fd4': 'https://pool.nodary.io',
-          },
-        };
+        draft.signedApiUrlStore = [
+          { url: 'http://127.0.0.1:8090/0xbF3137b0a7574563a23a8fC8badC6537F98197CC', lastReceived: 1 },
+          { url: 'https://pool.nodary.io/0xc52EeA00154B4fF1EbbF8Ba39FDe37F1AC3B9Fd4', lastReceived: 1 },
+        ];
       })
     );
   });

src/signed-api-fetch/data-fetcher.ts

@@ -2,7 +2,6 @@ import { clearInterval } from 'node:timers';
 
 import { go } from '@api3/promise-utils';
 import axios from 'axios';
-import { uniq } from 'lodash';
 
 import { HTTP_SIGNED_DATA_API_ATTEMPT_TIMEOUT, HTTP_SIGNED_DATA_API_HEADROOM } from '../constants';
 import * as localDataStore from '../signed-data-store';
@@ -64,14 +63,8 @@ export const runDataFetcher = async () => {
     });
   }
 
-  const urls = uniq(
-    Object.keys(config.chains).flatMap((chainId) =>
-      Object.entries(signedApiUrlStore[chainId]!).flatMap(([airnodeAddress, baseUrl]) => `${baseUrl}/${airnodeAddress}`)
-    )
-  );
-
   return Promise.allSettled(
-    urls.map(async (url) =>
+    signedApiUrlStore.map(async ({ url }) =>
       go(
         async () => {
           const payload = await callSignedDataApi(url);

src/signed-data-store/signed-data-store.test.ts

@@ -3,6 +3,7 @@ import { BigNumber, ethers } from 'ethers';
 import { init } from '../../test/fixtures/mock-config';
 import { generateRandomBytes32, signData } from '../../test/utils';
 import type { SignedData } from '../types';
+import { deriveBeaconId } from '../utils';
 
 import { verifySignedDataIntegrity } from './signed-data-store';
 import * as localDataStore from './signed-data-store';
@@ -34,11 +35,10 @@ describe('datastore', () => {
     const promisedStorage = localDataStore.setStoreDataPoint(testDataPoint);
     expect(promisedStorage).toBeFalsy();
 
-    const datapoint = localDataStore.getStoreDataPoint(testDataPoint.airnode, testDataPoint.templateId);
+    const dataFeedId = deriveBeaconId(testDataPoint.airnode, testDataPoint.templateId)!;
+    const datapoint = localDataStore.getStoreDataPoint(dataFeedId);
 
-    const { encodedValue, signature, timestamp } = testDataPoint;
-
-    expect(datapoint).toStrictEqual({ encodedValue, signature, timestamp });
+    expect(datapoint).toStrictEqual(testDataPoint);
   });
 
   it('checks that the timestamp on signed data is not in the future', async () => {

src/signed-data-store/signed-data-store.ts

@@ -3,7 +3,8 @@ import { ethers } from 'ethers';
 
 import { logger } from '../logger';
 import { getState, updateState } from '../state';
-import type { SignedData, AirnodeAddress, TemplateId } from '../types';
+import type { SignedData, AirnodeAddress, TemplateId, DataFeedId } from '../types';
+import { deriveBeaconId } from '../utils';
 
 export const verifySignedData = ({ airnode, templateId, timestamp, signature, encodedValue }: SignedData) => {
   // Verification is wrapped in goSync, because ethers methods can potentially throw on invalid input.
@@ -65,7 +66,10 @@ export const setStoreDataPoint = (signedData: SignedData) => {
   }
 
   const state = getState();
-  const existingValue = state.signedApiStore[airnode]?.[templateId];
+
+  const dataFeedId = deriveBeaconId(airnode, templateId)!;
+
+  const existingValue = state.signedApiStore[dataFeedId];
   if (existingValue && existingValue.timestamp >= timestamp) {
     logger.debug('Skipping store update. The existing store value is fresher.');
     return;
@@ -79,16 +83,14 @@ export const setStoreDataPoint = (signedData: SignedData) => {
     encodedValue,
   });
   updateState((draft) => {
-    if (!draft.signedApiStore[airnode]) {
-      draft.signedApiStore[airnode] = {};
-    }
-
-    draft.signedApiStore[airnode]![templateId] = { signature, timestamp, encodedValue };
+    draft.signedApiStore[dataFeedId] = signedData;
   });
 };
 
-export const getStoreDataPoint = (airnode: AirnodeAddress, templateId: TemplateId) =>
-  getState().signedApiStore[airnode]?.[templateId];
+export const getStoreDataPointByAirnodeAndTemplate = (airnode: AirnodeAddress, template: TemplateId) =>
+  getState().signedApiStore[deriveBeaconId(airnode, template)!];
+
+export const getStoreDataPoint = (dataFeedId: DataFeedId) => getState().signedApiStore[dataFeedId];
 
 export const clear = () => {
   updateState((draft) => {

src/state/state.test.ts

@@ -1,12 +1,14 @@
 import { BigNumber } from 'ethers';
 
 import { getConfig } from '../../test/fixtures/mock-config';
+import { deriveBeaconId } from '../utils';
 
 import { updateState, getState, setState } from './state';
 
 const timestampMock = 1_696_930_907_351;
 const stateMock = {
   config: getConfig(),
+
   gasPriceStore: {
     '31337': {
       localhost: {
@@ -16,40 +18,49 @@ const stateMock = {
     },
   },
   signedApiStore: {
-    '0xC04575A2773Da9Cd23853A69694e02111b2c4182': {
-      '0x154c34adf151cf4d91b7abe7eb6dcd193104ef2a29738ddc88020a58d6cf6183': {
-        encodedValue: '0x000000000000000000000000000000000000000000000065954b143faff77440',
-        signature:
-          '0x0fe25ad7debe4d018aa53acfe56d84f35c8bedf58574611f5569a8d4415e342311c093bfe0648d54e0a02f13987ac4b033b24220880638df9103a60d4f74090b1c',
-        timestamp: '1687850583',
-      },
-    },
-  },
-  signedApiUrlStore: {
-    '31337': {
-      '0xbF3137b0a7574563a23a8fC8badC6537F98197CC': 'http://127.0.0.1:8090/',
-      '0xc52EeA00154B4fF1EbbF8Ba39FDe37F1AC3B9Fd4': 'https://pool.nodary.io',
+    [deriveBeaconId(
+      '0xC04575A2773Da9Cd23853A69694e02111b2c4182',
+      '0x154c34adf151cf4d91b7abe7eb6dcd193104ef2a29738ddc88020a58d6cf6183'
+    )!]: {
+      airnode: '0xC04575A2773Da9Cd23853A69694e02111b2c4182',
+      encodedValue: '0x000000000000000000000000000000000000000000000065954b143faff77440',
+      templateId: '0x154c34adf151cf4d91b7abe7eb6dcd193104ef2a29738ddc88020a58d6cf6183',
+      signature:
+        '0x0fe25ad7debe4d018aa53acfe56d84f35c8bedf58574611f5569a8d4415e342311c093bfe0648d54e0a02f13987ac4b033b24220880638df9103a60d4f74090b1c',
+      timestamp: 'something-silly',
     },
   },
+  signedApiUrlStore: [
+    { url: 'http://127.0.0.1:8090/0xbF3137b0a7574563a23a8fC8badC6537F98197CC', lastReceived: 1 },
+    { url: 'https://pool.nodary.io/0xc52EeA00154B4fF1EbbF8Ba39FDe37F1AC3B9Fd4', lastReceived: 1 },
+  ],
+  dapis: {},
 };
 
 describe('state', () => {
   beforeEach(() => {
     setState(stateMock);
   });
 
+  const beaconId = deriveBeaconId(
+    '0xC04575A2773Da9Cd23853A69694e02111b2c4182',
+    '0x154c34adf151cf4d91b7abe7eb6dcd193104ef2a29738ddc88020a58d6cf6183'
+  )!;
+
+  const signedDataSample = {
+    airnode: '0xC04575A2773Da9Cd23853A69694e02111b2c4182',
+    encodedValue: '0x000000000000000000000000000000000000000000000065954b143faff77440',
+    templateId: '0x154c34adf151cf4d91b7abe7eb6dcd193104ef2a29738ddc88020a58d6cf6183',
+    signature:
+      '0x0fe25ad7debe4d018aa53acfe56d84f35c8bedf58574611f5569a8d4415e342311c093bfe0648d54e0a02f13987ac4b033b24220880638df9103a60d4f74090b1c',
+    timestamp: '1687850583',
+  };
+
   it('should update the state correctly', () => {
     const stateBefore = getState();
     updateState((draft) => {
-      draft.signedApiStore['0xc52EeA00154B4fF1EbbF8Ba39FDe37F1AC3B9Fd4'] = {};
-      draft.signedApiStore['0xc52EeA00154B4fF1EbbF8Ba39FDe37F1AC3B9Fd4'][
-        '0x96504241fb9ae9a5941f97c9561dcfcd7cee77ee9486a58c8e78551c1268ddec'
-      ] = {
-        encodedValue: '0x000000000000000000000000000000000000000000000065954b143faff77440',
-        signature:
-          '0x0fe25ad7debe4d018aa53acfe56d84f35c8bedf58574611f5569a8d4415e342311c093bfe0648d54e0a02f13987ac4b033b24220880638df9103a60d4f74090b1c',
-        timestamp: '1687850583',
-      };
+      delete draft.signedApiStore[beaconId];
+      draft.signedApiStore[beaconId] = signedDataSample;
     });
 
     const stateAfter = getState();
@@ -60,14 +71,7 @@ describe('state', () => {
       ...stateBefore,
       signedApiStore: {
         ...stateBefore.signedApiStore,
-        '0xc52EeA00154B4fF1EbbF8Ba39FDe37F1AC3B9Fd4': {
-          '0x96504241fb9ae9a5941f97c9561dcfcd7cee77ee9486a58c8e78551c1268ddec': {
-            encodedValue: '0x000000000000000000000000000000000000000000000065954b143faff77440',
-            signature:
-              '0x0fe25ad7debe4d018aa53acfe56d84f35c8bedf58574611f5569a8d4415e342311c093bfe0648d54e0a02f13987ac4b033b24220880638df9103a60d4f74090b1c',
-            timestamp: '1687850583',
-          },
-        },
+        [beaconId]: signedDataSample,
       },
     });
   });