Use multicall to fetch dAPI batches (#56)

* Implement multicall fetching

* Create contracts setup

* Implement e2e test, fix bugs

* Self review

* Fix tests and CI

Author: Siegrift

.github/workflows/main.yml

@@ -61,6 +61,8 @@ jobs:
           cache: 'pnpm'
       - name: Install Dependencies
         run: pnpm install --frozen-lockfile
+      - name: Compile contracts
+        run: pnpm run contracts:compile
       - name: Start Hardhat
         run: pnpm dev:eth-node&
       - name: Test E2E

contracts/contract-imports.sol

@@ -2,3 +2,4 @@
 pragma solidity 0.8.18;
 
 import "@api3/dapi-management/contracts/DapiDataRegistry.sol";
+import "@api3/dapi-management/contracts/HashRegistry.sol";

jest-e2e.config.js

@@ -15,5 +15,6 @@ module.exports = {
   testEnvironment: 'jest-environment-node',
   testMatch: ['**/?(*.)+(feature).[t]s?(x)'],
   testPathIgnorePatterns: ['<rootDir>/.build', '<rootDir>/dist/', '<rootDir>/build/'],
+  testTimeout: 15_000,
   verbose: true,
 };

package.json

@@ -29,11 +29,13 @@
     "tsc": "tsc --project ."
   },
   "devDependencies": {
+    "@api3/airnode-abi": "^0.12.0",
     "@api3/ois": "^2.2.0",
     "@nomicfoundation/hardhat-network-helpers": "^1.0.9",
     "@nomicfoundation/hardhat-toolbox": "^2.0.2",
     "@nomiclabs/hardhat-ethers": "^2.2.3",
     "@openzeppelin/contracts": "4.9.3",
+    "@openzeppelin/merkle-tree": "^1.0.5",
     "@typechain/ethers-v5": "^10.1.0",
     "@typechain/hardhat": "^6.1.2",
     "@types/jest": "^29.5.5",

pnpm-lock.yaml

@@ -1,6 +1,6 @@
 import { BigNumber, ethers } from 'ethers';
 
-import { getUnixTimestamp } from '../../test/fixtures/utils';
+import { getUnixTimestamp } from '../../test/utils';
 import { HUNDRED_PERCENT } from '../constants';
 
 import {

src/condition-check/condition-check.test.ts

@@ -9,7 +9,7 @@ let env: EnvConfig | undefined;
 export const loadEnv = () => {
   if (env) return env;
 
-  dotenv.config({ path: join(__dirname, '../.env') });
+  dotenv.config({ path: join(__dirname, '../../.env') });
 
   const parseResult = envConfigSchema.safeParse(process.env);
   if (!parseResult.success) {

src/env/env.ts

@@ -6,4 +6,48 @@ import { type DapiDataRegistry, DapiDataRegistry__factory } from '../../typechai
 export const getDapiDataRegistry = (address: string, provider: ethers.providers.StaticJsonRpcProvider) =>
   DapiDataRegistry__factory.connect(address, provider);
 
-export type ReadDapisResponse = Awaited<ReturnType<DapiDataRegistry['readDapis']>>;
+export const verifyMulticallResponse = (
+  response: Awaited<ReturnType<DapiDataRegistry['callStatic']['tryMulticall']>>
+) => {
+  const { successes, returndata } = response;
+
+  if (!successes.every(Boolean)) throw new Error('One of the multicalls failed');
+  return returndata;
+};
+
+export const decodeDapisCountResponse = (dapiDataRegistry: DapiDataRegistry, dapisCountReturndata: string) => {
+  const dapisCount = dapiDataRegistry.interface.decodeFunctionResult('dapisCount', dapisCountReturndata)[0] as Awaited<
+    ReturnType<DapiDataRegistry['dapisCount']>
+  >;
+  return dapisCount.toNumber();
+};
+
+export type DapisCountResponse = ReturnType<typeof decodeDapisCountResponse>;
+
+export const decodeReadDapiWithIndexResponse = (
+  dapiDataRegistry: DapiDataRegistry,
+  readDapiWithIndexReturndata: string
+) => {
+  const { dapiName, updateParameters, dataFeedValue, dataFeed, signedApiUrls } =
+    dapiDataRegistry.interface.decodeFunctionResult('readDapiWithIndex', readDapiWithIndexReturndata) as Awaited<
+      ReturnType<DapiDataRegistry['readDapiWithIndex']>
+    >;
+
+  // Ethers responses are returned as a combination of array and object. When such object is logged, only the array part
+  // is logged. To make the logs more readable, we convert the object part to a plain object.
+  const { deviationReference, deviationThresholdInPercentage, heartbeatInterval } = updateParameters;
+  const { value, timestamp } = dataFeedValue;
+  return {
+    dapiName,
+    updateParameters: {
+      deviationReference,
+      deviationThresholdInPercentage,
+      heartbeatInterval,
+    },
+    dataFeedValue: { value, timestamp },
+    dataFeed,
+    signedApiUrls,
+  };
+};
+
+export type ReadDapiWithIndexResponse = ReturnType<typeof decodeReadDapiWithIndexResponse>;

src/update-feeds/dapi-data-registry.ts

@@ -1,6 +1,9 @@
 import { ethers } from 'ethers';
 
-import { generateMockDapiDataRegistry, generateReadDapisResponse } from '../../test/fixtures/dapi-data-registry';
+import {
+  generateMockDapiDataRegistry,
+  generateReadDapiWithIndexResponse,
+} from '../../test/fixtures/dapi-data-registry';
 import { allowPartial } from '../../test/utils';
 import type { DapiDataRegistry } from '../../typechain-types';
 import type { Chain } from '../config/schema';
@@ -38,7 +41,7 @@ describe(startUpdateFeedLoops.name, () => {
 
     // Expect the intervals to be called with the correct stagger time.
     expect(setInterval).toHaveBeenCalledTimes(2);
-    expect(intervalCalls[1]! - intervalCalls[0]!).toBeGreaterThanOrEqual(40); // Reserving 10s as the buffer for computing stagger time.
+    expect(intervalCalls[1]! - intervalCalls[0]!).toBeGreaterThanOrEqual(40); // Reserving 10ms as the buffer for computing stagger time.
 
     // Expect the logs to be called with the correct context.
     expect(logger.debug).toHaveBeenCalledTimes(3);
@@ -92,21 +95,21 @@ describe(startUpdateFeedLoops.name, () => {
 
     // Expect the logs to be called with the correct context.
     expect(logger.debug).toHaveBeenCalledTimes(4);
-    expect(logger.debug).toHaveBeenCalledWith('Starting update loops for chain', {
+    expect(logger.debug).toHaveBeenNthCalledWith(1, 'Starting update loops for chain', {
       chainId: '123',
       staggerTime: 100,
       providerNames: ['first-provider'],
     });
-    expect(logger.debug).toHaveBeenCalledWith('Starting update loops for chain', {
+    expect(logger.debug).toHaveBeenNthCalledWith(2, 'Starting update feed loop', {
+      chainId: '123',
+      providerName: 'first-provider',
+    });
+    expect(logger.debug).toHaveBeenNthCalledWith(3, 'Starting update loops for chain', {
       chainId: '456',
       staggerTime: 100,
       providerNames: ['another-provider'],
     });
-    expect(logger.debug).toHaveBeenCalledWith('Starting update feed loop', {
-      chainId: '123',
-      providerName: 'first-provider',
-    });
-    expect(logger.debug).toHaveBeenCalledWith('Starting update feed loop', {
+    expect(logger.debug).toHaveBeenNthCalledWith(4, 'Starting update feed loop', {
       chainId: '456',
       providerName: 'another-provider',
     });
@@ -119,7 +122,7 @@ describe(runUpdateFeed.name, () => {
     jest
       .spyOn(dapiDataRegistryModule, 'getDapiDataRegistry')
       .mockReturnValue(dapiDataRegistry as unknown as DapiDataRegistry);
-    dapiDataRegistry.readDapis.mockRejectedValueOnce(new Error('provider-error'));
+    dapiDataRegistry.callStatic.tryMulticall.mockRejectedValueOnce(new Error('provider-error'));
     jest.spyOn(logger, 'error');
 
     await runUpdateFeed(
@@ -145,16 +148,19 @@ describe(runUpdateFeed.name, () => {
 
   it('fetches other batches in a staggered way and logs errors', async () => {
     // Prepare the mocked contract so it returns three batches (of size 1) of dAPIs and the second batch fails to load.
-    const firstBatch = generateReadDapisResponse();
-    const thirdBatch = generateReadDapisResponse();
+    const firstBatch = generateReadDapiWithIndexResponse();
+    const thirdBatch = generateReadDapiWithIndexResponse();
     const dapiDataRegistry = generateMockDapiDataRegistry();
     jest
       .spyOn(dapiDataRegistryModule, 'getDapiDataRegistry')
       .mockReturnValue(dapiDataRegistry as unknown as DapiDataRegistry);
-    dapiDataRegistry.readDapis.mockResolvedValueOnce(firstBatch);
-    dapiDataRegistry.readDapis.mockRejectedValueOnce(new Error('provider-error'));
-    dapiDataRegistry.readDapis.mockResolvedValueOnce(thirdBatch);
-    dapiDataRegistry.dapisCount.mockResolvedValueOnce(ethers.BigNumber.from(3));
+    dapiDataRegistry.interface.decodeFunctionResult.mockImplementation((_fn, value) => value);
+    dapiDataRegistry.callStatic.tryMulticall.mockResolvedValueOnce({
+      successes: [true, true],
+      returndata: [[ethers.BigNumber.from(3)], firstBatch],
+    });
+    dapiDataRegistry.callStatic.tryMulticall.mockResolvedValueOnce({ successes: [false], returndata: [] });
+    dapiDataRegistry.callStatic.tryMulticall.mockResolvedValueOnce({ successes: [true], returndata: [thirdBatch] });
     const sleepCalls = [] as number[];
     const originalSleep = utilsModule.sleep;
     jest.spyOn(utilsModule, 'sleep').mockImplementation(async (ms) => {
@@ -179,36 +185,42 @@ describe(runUpdateFeed.name, () => {
 
     // Expect the contract to fetch the batches to be called with the correct stagger time.
     expect(utilsModule.sleep).toHaveBeenCalledTimes(3);
-    expect(sleepCalls[0]).toBeGreaterThanOrEqual(40); // Reserving 10s as the buffer for computing stagger time.
+    expect(sleepCalls[0]).toBeGreaterThanOrEqual(40); // Reserving 10ms as the buffer for computing stagger time.
     expect(sleepCalls[1]).toBeGreaterThanOrEqual(0);
     expect(sleepCalls[2]).toBe(49.999_999_999_999_99); // Stagger time is actually 150 / 3 = 50, but there is an rounding error.
 
     // Expect the logs to be called with the correct context.
     expect(logger.error).toHaveBeenCalledTimes(1);
-    expect(logger.error).toHaveBeenCalledWith('Failed to get active dAPIs batch', new Error('provider-error'), {
-      chainId: '123',
-      providerName: 'provider-name',
-    });
-    expect(logger.debug).toHaveBeenCalledTimes(4);
-    expect(logger.debug).toHaveBeenCalledWith('Fetching first batch of dAPIs batches', {
+    expect(logger.error).toHaveBeenCalledWith(
+      'Failed to get active dAPIs batch',
+      new Error('One of the multicalls failed'),
+      {
+        chainId: '123',
+        providerName: 'provider-name',
+      }
+    );
+    expect(logger.debug).toHaveBeenCalledTimes(6);
+    expect(logger.debug).toHaveBeenNthCalledWith(1, 'Fetching first batch of dAPIs batches', {
       chainId: '123',
       providerName: 'provider-name',
     });
-    expect(logger.debug).toHaveBeenCalledWith('Fetching batches of active dAPIs', {
+    expect(logger.debug).toHaveBeenNthCalledWith(2, 'Processing batch of active dAPIs', expect.anything());
+    expect(logger.debug).toHaveBeenNthCalledWith(3, 'Fetching batches of active dAPIs', {
       batchesCount: 3,
-      staggerTime: 49.999_999_999_999_99,
       chainId: '123',
       providerName: 'provider-name',
+      staggerTime: 49.999_999_999_999_99,
     });
-    expect(logger.debug).toHaveBeenCalledWith('Fetching batch of active dAPIs', {
+    expect(logger.debug).toHaveBeenNthCalledWith(4, 'Fetching batch of active dAPIs', {
       batchIndex: 1,
       chainId: '123',
       providerName: 'provider-name',
     });
-    expect(logger.debug).toHaveBeenCalledWith('Fetching batch of active dAPIs', {
+    expect(logger.debug).toHaveBeenNthCalledWith(5, 'Fetching batch of active dAPIs', {
       batchIndex: 2,
       chainId: '123',
       providerName: 'provider-name',
     });
+    expect(logger.debug).toHaveBeenNthCalledWith(6, 'Processing batch of active dAPIs', expect.anything());
   });
 });