Create and submit update transactions

hardhat.config.ts

@@ -3,9 +3,7 @@ import type { HardhatUserConfig } from 'hardhat/types';
 import '@nomicfoundation/hardhat-toolbox';
 
 const config: HardhatUserConfig = {
-  solidity: {
-    version: '0.8.18',
-  },
+  solidity: { version: '0.8.18' },
   networks: {
     localhost: {
       url: 'http://127.0.0.1:8545/',

jest-e2e.config.js

@@ -15,6 +15,6 @@ module.exports = {
   testEnvironment: 'jest-environment-node',
   testMatch: ['**/?(*.)+(feature).[t]s?(x)'],
   testPathIgnorePatterns: ['<rootDir>/.build', '<rootDir>/dist/', '<rootDir>/build/'],
-  testTimeout: 15_000,
+  testTimeout: 25_000,
   verbose: true,
 };

src/constants.ts

@@ -2,3 +2,5 @@ export const HTTP_SIGNED_DATA_API_ATTEMPT_TIMEOUT = 10_000;
 export const HTTP_SIGNED_DATA_API_HEADROOM = 1000;
 
 export const HUNDRED_PERCENT = 1e8;
+
+export const AIRSEEKER_PROTOCOL_ID = '5'; // From: https://github.com/api3dao/airnode/blob/ef16c54f33d455a1794e7886242567fc47ee14ef/packages/airnode-protocol/src/index.ts#L46

src/update-feeds/api3-server-v1.ts

@@ -0,0 +1,5 @@
+import { Api3ServerV1__factory } from '@api3/airnode-protocol-v1';
+import type { ethers } from 'ethers';
+
+export const getApi3ServerV1 = (address: string, provider: ethers.providers.StaticJsonRpcProvider) =>
+  Api3ServerV1__factory.connect(address, provider);

src/update-feeds/update-feeds.test.ts

@@ -1,10 +1,7 @@
 import { ethers } from 'ethers';
 
-import {
-  generateMockDapiDataRegistry,
-  generateReadDapiWithIndexResponse,
-} from '../../test/fixtures/dapi-data-registry';
 import { generateTestConfig } from '../../test/fixtures/mock-config';
+import { generateMockDapiDataRegistry, generateReadDapiWithIndexResponse } from '../../test/fixtures/mock-contract';
 import { allowPartial } from '../../test/utils';
 import type { DapiDataRegistry } from '../../typechain-types';
 import type { Chain } from '../config/schema';

src/update-feeds/update-feeds.ts

@@ -11,11 +11,11 @@ import type { ChainId, Provider } from '../types';
 import { isFulfilled, sleep } from '../utils';
 
 import {
+  decodeDapisCountResponse,
+  decodeReadDapiWithIndexResponse,
   getDapiDataRegistry,
-  type ReadDapiWithIndexResponse,
   verifyMulticallResponse,
-  decodeReadDapiWithIndexResponse,
-  decodeDapisCountResponse,
+  type ReadDapiWithIndexResponse,
 } from './dapi-data-registry';
 
 export const startUpdateFeedLoops = async () => {
@@ -55,12 +55,12 @@ export const runUpdateFeed = async (providerName: Provider, chain: Chain, chainI
     logger.debug(`Fetching first batch of dAPIs batches`);
     const firstBatchStartTime = Date.now();
     const goFirstBatch = await go(async () => {
-      const dapisCountCall = dapiDataRegistry.interface.encodeFunctionData('dapisCount');
-      const readDapiWithIndexCalls = range(0, dataFeedBatchSize).map((dapiIndex) =>
+      const dapisCountCalldata = dapiDataRegistry.interface.encodeFunctionData('dapisCount');
+      const readDapiWithIndexCalldatas = range(0, dataFeedBatchSize).map((dapiIndex) =>
         dapiDataRegistry.interface.encodeFunctionData('readDapiWithIndex', [dapiIndex])
       );
       const [dapisCountReturndata, ...readDapiWithIndexCallsReturndata] = verifyMulticallResponse(
-        await dapiDataRegistry.callStatic.tryMulticall([dapisCountCall, ...readDapiWithIndexCalls])
+        await dapiDataRegistry.callStatic.tryMulticall([dapisCountCalldata, ...readDapiWithIndexCalldatas])
       );
 
       const dapisCount = decodeDapisCountResponse(dapiDataRegistry, dapisCountReturndata!);
@@ -99,11 +99,11 @@ export const runUpdateFeed = async (providerName: Provider, chain: Chain, chainI
         logger.debug(`Fetching batch of active dAPIs`, { batchIndex });
         const dapiBatchIndexStart = batchIndex * dataFeedBatchSize;
         const dapiBatchIndexEnd = Math.min(dapisCount, dapiBatchIndexStart + dataFeedBatchSize);
-        const readDapiWithIndexCalls = range(dapiBatchIndexStart, dapiBatchIndexEnd).map((dapiIndex) =>
+        const readDapiWithIndexCalldatas = range(dapiBatchIndexStart, dapiBatchIndexEnd).map((dapiIndex) =>
           dapiDataRegistry.interface.encodeFunctionData('readDapiWithIndex', [dapiIndex])
         );
         const returndata = verifyMulticallResponse(
-          await dapiDataRegistry.callStatic.tryMulticall(readDapiWithIndexCalls)
+          await dapiDataRegistry.callStatic.tryMulticall(readDapiWithIndexCalldatas)
         );
 
         return returndata.map((returndata) => decodeReadDapiWithIndexResponse(dapiDataRegistry, returndata));

src/update-feeds/update-transactions.test.ts

@@ -0,0 +1,81 @@
+import type { Api3ServerV1 } from '@api3/airnode-protocol-v1';
+import { ethers } from 'ethers';
+
+import { generateMockApi3ServerV1 } from '../../test/fixtures/mock-contract';
+
+import {
+  deriveSponsorWallet,
+  deriveWalletPathFromSponsorAddress,
+  estimateMulticallGasLimit,
+} from './update-transactions';
+
+describe(estimateMulticallGasLimit.name, () => {
+  it('estimates the gas limit for a multicall', async () => {
+    const mockApi3ServerV1 = generateMockApi3ServerV1();
+    mockApi3ServerV1.estimateGas.multicall.mockResolvedValueOnce(ethers.BigNumber.from(500_000));
+
+    const gasLimit = await estimateMulticallGasLimit(
+      mockApi3ServerV1 as unknown as Api3ServerV1,
+      ['0xbeaconId1Calldata', '0xbeaconId2Calldata', '0xbeaconSetCalldata'],
+      ['beaconId1', 'beaconId2']
+    );
+
+    expect(gasLimit).toStrictEqual(ethers.BigNumber.from(550_000)); // Note that the gas limit is increased by 10%.
+  });
+
+  it('uses dummy data estimation when multicall estimate fails', async () => {
+    const mockApi3ServerV1 = generateMockApi3ServerV1();
+    mockApi3ServerV1.estimateGas.multicall.mockRejectedValue(
+      new Error('e.g. one of the beacons has on chain value with higher timestamp')
+    );
+    mockApi3ServerV1.estimateGas.updateBeaconWithSignedData.mockResolvedValueOnce(ethers.BigNumber.from(50_000));
+    mockApi3ServerV1.estimateGas.updateBeaconSetWithBeacons.mockResolvedValueOnce(ethers.BigNumber.from(30_000));
+
+    const gasLimit = await estimateMulticallGasLimit(
+      mockApi3ServerV1 as unknown as Api3ServerV1,
+      ['0xbeaconId1Calldata', '0xbeaconId2Calldata', '0xbeaconSetCalldata'],
+      ['beaconId1', 'beaconId2']
+    );
+
+    expect(gasLimit).toStrictEqual(ethers.BigNumber.from(130_000));
+  });
+
+  it('uses fixed gas limit when dummy data estimation fails', async () => {
+    const mockApi3ServerV1 = generateMockApi3ServerV1();
+    mockApi3ServerV1.estimateGas.multicall.mockRejectedValue(
+      new Error('e.g. one of the beacons has on chain value with higher timestamp')
+    );
+    mockApi3ServerV1.estimateGas.updateBeaconWithSignedData.mockRejectedValue(new Error('provider-error'));
+    mockApi3ServerV1.estimateGas.updateBeaconSetWithBeacons.mockRejectedValue(new Error('provider-error'));
+
+    const gasLimit = await estimateMulticallGasLimit(
+      mockApi3ServerV1 as unknown as Api3ServerV1,
+      ['0xbeaconId1Calldata', '0xbeaconId2Calldata', '0xbeaconSetCalldata'],
+      ['beaconId1', 'beaconId2']
+    );
+
+    expect(gasLimit).toStrictEqual(ethers.BigNumber.from(2_000_000));
+  });
+});
+
+describe(deriveSponsorWallet.name, () => {
+  it('derives sponsor wallets for a dAPI', () => {
+    const btcEthDapiName = ethers.utils.formatBytes32String('BTC/ETH');
+    const sponsorWalletMnemonic = 'diamond result history offer forest diagram crop armed stumble orchard stage glance';
+
+    const btcEthSponsorWallet = deriveSponsorWallet(sponsorWalletMnemonic, btcEthDapiName);
+
+    expect(btcEthSponsorWallet.address).toBe('0xDa8b0388F435F609C8cdA6cf73C890D90205c863');
+  });
+});
+
+describe(deriveWalletPathFromSponsorAddress.name, () => {
+  it('derives the correct wallet path from the sponsor address', () => {
+    expect(deriveWalletPathFromSponsorAddress('0xE2c582D05126E09734cAFABea8A0E56E9B827629')).toBe(
+      '5/461534761/1363266269/1395387130/154600633/743976197/28'
+    );
+    expect(deriveWalletPathFromSponsorAddress('0x86D3763039cA6BABe616755Ceb58e19f3388D9AB')).toBe(
+      '5/864606635/1454490430/408540531/1314086239/1832346371/16'
+    );
+  });
+});

src/update-feeds/update-transactions.ts

@@ -0,0 +1,169 @@
+import type { Api3ServerV1 } from '@api3/airnode-protocol-v1';
+import { go } from '@api3/promise-utils';
+import { ethers } from 'ethers';
+
+import { AIRSEEKER_PROTOCOL_ID } from '../constants';
+import { logger } from '../logger';
+import { getState } from '../state';
+import type { SignedData } from '../types';
+
+import type { ReadDapiWithIndexResponse } from './dapi-data-registry';
+
+export interface UpdateableBeacon {
+  beaconId: string;
+  signedData: SignedData;
+}
+
+export interface UpdateableDapi {
+  dapiInfo: ReadDapiWithIndexResponse;
+  updateableBeacons: UpdateableBeacon[];
+}
+
+export const updateFeeds = async (
+  api3ServerV1: Api3ServerV1,
+  gasPrice: ethers.BigNumber,
+  updateableDapis: UpdateableDapi[]
+) => {
+  const state = getState();
+  const {
+    config: { sponsorWalletMnemonic },
+  } = state;
+
+  // Update all of the dAPIs in parallel.
+  await Promise.all(
+    updateableDapis.map(async (dapi) => {
+      const { dapiInfo, updateableBeacons } = dapi;
+      const {
+        dapiName,
+        decodedDataFeed: { dataFeedId },
+      } = dapiInfo;
+
+      await logger.runWithContext({ dapiName, dataFeedId }, async () => {
+        const goUpdate = await go(async () => {
+          // Create calldata for all beacons of the particular data feed the dAPI points to.
+          const beaconUpdateCalls = updateableBeacons.map((beacon) => {
+            const { signedData } = beacon;
+
+            return api3ServerV1.interface.encodeFunctionData('updateBeaconWithSignedData', [
+              signedData.airnode,
+              signedData.templateId,
+              signedData.timestamp,
+              signedData.encodedValue,
+              signedData.signature,
+            ]);
+          });
+
+          // If there are multiple beacons in the data feed it's a beacons set which we need to update as well.
+          const dataFeedUpdateCalldatas =
+            beaconUpdateCalls.length > 1
+              ? [
+                  ...beaconUpdateCalls,
+                  api3ServerV1.interface.encodeFunctionData('updateBeaconSetWithBeacons', [
+                    updateableBeacons.map(({ beaconId }) => beaconId),
+                  ]),
+                ]
+              : beaconUpdateCalls;
+
+          logger.debug('Estimating gas limit');
+          const gasLimit = await estimateMulticallGasLimit(
+            api3ServerV1,
+            dataFeedUpdateCalldatas,
+            updateableBeacons.map((beacon) => beacon.beaconId)
+          );
+
+          logger.debug('Deriving sponsor wallet');
+          // TODO: These wallets could be persisted as a performance optimization.
+          const sponsorWallet = deriveSponsorWallet(sponsorWalletMnemonic, dapiName);
+
+          logger.debug('Updating dAPI', { gasPrice, gasLimit });
+          await api3ServerV1
+            // When we add the sponsor wallet (signer) without connecting it to the provider, the provider of the
+            // contract will be set to "null". We need to connect the sponsor wallet to the provider of the contract.
+            .connect(sponsorWallet.connect(api3ServerV1.provider))
+            .tryMulticall(dataFeedUpdateCalldatas, { gasPrice, gasLimit });
+          logger.debug('Successfully updated dAPI');
+        });
+
+        if (!goUpdate.success) {
+          logger.error(`Failed to update a dAPI`, goUpdate.error);
+        }
+      });
+    })
+  );
+};
+
+export const estimateMulticallGasLimit = async (
+  api3ServerV1: Api3ServerV1,
+  calldatas: string[],
+  beaconIds: string[]
+) => {
+  const goEstimateGas = await go(async () => api3ServerV1.estimateGas.multicall(calldatas));
+  if (goEstimateGas.success) {
+    // Adding a extra 10% because multicall consumes less gas than tryMulticall
+    return goEstimateGas.data.mul(ethers.BigNumber.from(Math.round(1.1 * 100))).div(ethers.BigNumber.from(100));
+  }
+  logger.warn(`Unable to estimate gas for multicall`, goEstimateGas.error);
+
+  const goEstimateDummyBeaconUpdateGas = await go(async () => {
+    const { dummyAirnode, dummyBeaconTemplateId, dummyBeaconTimestamp, dummyBeaconData, dummyBeaconSignature } =
+      await createDummyBeaconUpdateData();
+    return [
+      await api3ServerV1.estimateGas.updateBeaconWithSignedData(
+        dummyAirnode.address,
+        dummyBeaconTemplateId,
+        dummyBeaconTimestamp,
+        dummyBeaconData,
+        dummyBeaconSignature
+      ),
+      await api3ServerV1.estimateGas.updateBeaconSetWithBeacons(beaconIds),
+    ] as const;
+  });
+  if (goEstimateDummyBeaconUpdateGas.success) {
+    const [updateBeaconWithSignedDataGas, updateBeaconSetWithBeaconsGas] = goEstimateDummyBeaconUpdateGas.data;
+    return updateBeaconWithSignedDataGas.mul(beaconIds.length).add(updateBeaconSetWithBeaconsGas);
+  }
+
+  return ethers.BigNumber.from(2_000_000);
+};
+
+export const deriveSponsorWallet = (sponsorWalletMnemonic: string, dapiName: string) => {
+  // Take first 20 bytes of dapiName as sponsor address together with the "0x" prefix.
+  const sponsorAddress = ethers.utils.getAddress(dapiName.slice(0, 42));
+  const sponsorWallet = ethers.Wallet.fromMnemonic(
+    sponsorWalletMnemonic,
+    `m/44'/60'/0'/${deriveWalletPathFromSponsorAddress(sponsorAddress)}`
+  );
+  logger.debug('Derived sponsor wallet', { sponsorAddress, sponsorWalletAddress: sponsorWallet.address });
+
+  return sponsorWallet;
+};
+
+export function deriveWalletPathFromSponsorAddress(sponsorAddress: string) {
+  const sponsorAddressBN = ethers.BigNumber.from(sponsorAddress);
+  const paths = [];
+  for (let i = 0; i < 6; i++) {
+    const shiftedSponsorAddressBN = sponsorAddressBN.shr(31 * i);
+    paths.push(shiftedSponsorAddressBN.mask(31).toString());
+  }
+  return `${AIRSEEKER_PROTOCOL_ID}/${paths.join('/')}`;
+}
+
+export const createDummyBeaconUpdateData = async (dummyAirnode: ethers.Wallet = ethers.Wallet.createRandom()) => {
+  const dummyBeaconTemplateId = ethers.utils.hexlify(ethers.utils.randomBytes(32));
+  const dummyBeaconTimestamp = Math.floor(Date.now() / 1000);
+  const randomBytes = ethers.utils.randomBytes(Math.floor(Math.random() * 27) + 1);
+  const dummyBeaconData = ethers.utils.defaultAbiCoder.encode(
+    ['int224'],
+    // Any random number that fits into an int224
+    [ethers.BigNumber.from(randomBytes)]
+  );
+  const dummyBeaconSignature = await dummyAirnode.signMessage(
+    ethers.utils.arrayify(
+      ethers.utils.solidityKeccak256(
+        ['bytes32', 'uint256', 'bytes'],
+        [dummyBeaconTemplateId, dummyBeaconTimestamp, dummyBeaconData]
+      )
+    )
+  );
+  return { dummyAirnode, dummyBeaconTemplateId, dummyBeaconTimestamp, dummyBeaconData, dummyBeaconSignature };
+};