Merge pull request #244 from lidofinance/feature/si-1580-add-validator-info-log-and-endpoint

Feature/si 1580 add validator info log and endpoint

Author: Jeday

src/common/prometheus/prometheus.service.ts

@@ -35,6 +35,12 @@ export class PrometheusService {
     labelNames: ENV_KEYS,
   });
 
+  public validatorsState = this.getOrCreateMetric('Gauge', {
+    name: METRICS_PREFIX + 'validators_state',
+    help: 'balances of Lido validators with withdrawable_epoch by frames',
+    labelNames: ['frame', 'balance'],
+  });
+
   public clApiRequestDuration = this.getOrCreateMetric('Histogram', {
     name: METRICS_PREFIX + 'cl_api_requests_duration_seconds',
     help: 'CL API request duration',

src/common/validators/strigify-frame-balances.ts

@@ -0,0 +1,9 @@
+import { BigNumber } from '@ethersproject/bignumber';
+
+export function stringifyFrameBalances(frameBalances: Record<string, BigNumber>) {
+  return JSON.stringify(
+    Object.keys(frameBalances).reduce((acc, key) => {
+      return { ...acc, [key]: frameBalances[key].toString() };
+    }, {}),
+  );
+}

src/http/http.constants.ts

@@ -1,6 +1,7 @@
 export const HTTP_PATHS = {
   1: {
     nft: 'nft',
+    'validators-info': 'validators-info',
     'request-time': 'request-time',
     'estimate-gas': 'estimate-gas',
   },

src/http/http.module.ts

@@ -12,9 +12,10 @@ import { CacheModule, CacheControlHeadersInterceptor } from './common/cache';
 import { RequestTimeModule } from './request-time';
 import { NFTModule } from './nft';
 import { EstimateModule } from './estimate';
+import { ValidatorsModule } from './validators';
 
 @Module({
-  imports: [RequestTimeModule, NFTModule, EstimateModule, CacheModule, ThrottlerModule],
+  imports: [RequestTimeModule, NFTModule, EstimateModule, ValidatorsModule, CacheModule, ThrottlerModule],
   providers: [
     { provide: APP_GUARD, useClass: ThrottlerBehindProxyGuard },
     { provide: APP_INTERCEPTOR, useClass: CacheControlHeadersInterceptor },

src/http/validators/dto/index.ts

@@ -0,0 +1 @@
+export * from './validators.dto';

src/http/validators/dto/validators.dto.ts

@@ -0,0 +1,33 @@
+import { ApiProperty } from '@nestjs/swagger';
+
+export class ValidatorsDto {
+  @ApiProperty({
+    example: 1658650005,
+    description: 'ms time when data was last updated at',
+  })
+  lastUpdatedAt: number;
+
+  @ApiProperty({
+    example: 1724856617,
+    description: 'max exit epoch over all CL network',
+  })
+  maxExitEpoch: number;
+
+  @ApiProperty({
+    example: '{}',
+    description: 'sum of balances Lido validators with withdrawable_epoch by frame',
+  })
+  frameBalances: Record<string, string>;
+
+  @ApiProperty({
+    example: 100000,
+    description: 'total number of validators in network',
+  })
+  totalValidators: number;
+
+  @ApiProperty({
+    example: 100000,
+    description: 'current frame',
+  })
+  currentFrame: number;
+}

src/http/validators/index.ts

@@ -0,0 +1,3 @@
+export * from './validators.controller';
+export * from './validators.module';
+export * from './validators.service';

src/http/validators/validators.controller.ts

@@ -0,0 +1,28 @@
+import {
+  ClassSerializerInterceptor,
+  Controller,
+  Get,
+  HttpStatus,
+  UseInterceptors,
+  Version,
+  CacheTTL,
+} from '@nestjs/common';
+import { ApiResponse, ApiTags } from '@nestjs/swagger';
+import { HTTP_PATHS } from 'http/http.constants';
+import { ValidatorsService } from './validators.service';
+import { ValidatorsDto } from './dto';
+
+@Controller()
+@ApiTags('Validators')
+@UseInterceptors(ClassSerializerInterceptor)
+export class ValidatorsController {
+  constructor(protected readonly validatorsService: ValidatorsService) {}
+
+  @Version('1')
+  @Get(HTTP_PATHS[1]['validators-info'])
+  @CacheTTL(20 * 1000)
+  @ApiResponse({ status: HttpStatus.OK, type: ValidatorsDto })
+  async validatorsV1(): Promise<ValidatorsDto> {
+    return this.validatorsService.getValidatorsInfo();
+  }
+}

src/http/validators/validators.module.ts

@@ -0,0 +1,13 @@
+import { Module } from '@nestjs/common';
+import { ConfigModule } from 'common/config';
+import { ValidatorsController } from './validators.controller';
+import { ValidatorsService } from './validators.service';
+import { ValidatorsStorageModule } from '../../storage';
+import { GenesisTimeModule } from '../../common/genesis-time';
+
+@Module({
+  imports: [ConfigModule, ValidatorsStorageModule, GenesisTimeModule],
+  controllers: [ValidatorsController],
+  providers: [ValidatorsService],
+})
+export class ValidatorsModule {}

src/http/validators/validators.service.ts

@@ -0,0 +1,34 @@
+import { Injectable } from '@nestjs/common';
+import { ConfigService } from 'common/config';
+import { ValidatorsStorageService } from '../../storage';
+import { GenesisTimeService } from '../../common/genesis-time';
+
+@Injectable()
+export class ValidatorsService {
+  constructor(
+    protected readonly configService: ConfigService,
+    protected readonly validatorsServiceStorage: ValidatorsStorageService,
+    protected readonly genesisTimeService: GenesisTimeService,
+  ) {}
+
+  getValidatorsInfo() {
+    const lastUpdatedAt = this.validatorsServiceStorage.getLastUpdate();
+    const maxExitEpoch = Number(this.validatorsServiceStorage.getMaxExitEpoch());
+    const frameBalancesBigNumber = this.validatorsServiceStorage.getFrameBalances();
+    const totalValidators = this.validatorsServiceStorage.getTotal();
+    const currentFrame = this.genesisTimeService.getFrameOfEpoch(this.genesisTimeService.getCurrentEpoch());
+
+    const frameBalances = Object.keys(frameBalancesBigNumber).reduce((acc, item) => {
+      acc[item] = frameBalancesBigNumber[item].toString();
+      return acc;
+    }, {} as Record<string, string>);
+
+    return {
+      lastUpdatedAt,
+      maxExitEpoch,
+      frameBalances,
+      totalValidators,
+      currentFrame,
+    };
+  }
+}

src/jobs/validators/validators.service.ts

@@ -16,13 +16,16 @@ import { ResponseValidatorsData, Validator } from './validators.types';
 import { parseGweiToWei } from '../../common/utils/parse-gwei-to-big-number';
 import { ValidatorsCacheService } from 'storage/validators/validators-cache.service';
 import { CronExpression } from '@nestjs/schedule';
+import { PrometheusService } from '../../common/prometheus';
+import { stringifyFrameBalances } from '../../common/validators/strigify-frame-balances';
 
 export class ValidatorsService {
   static SERVICE_LOG_NAME = 'validators';
 
   constructor(
     @Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService,
 
+    protected readonly prometheusService: PrometheusService,
     protected readonly consensusProviderService: ConsensusProviderService,
     protected readonly configService: ConfigService,
     protected readonly jobService: JobService,
@@ -78,23 +81,36 @@ export class ValidatorsService {
 
           await unblock();
         }
-        await this.setLidoValidatorsWithdrawableBalances(data);
         this.validatorsStorageService.setTotal(totalValidators);
         this.validatorsStorageService.setMaxExitEpoch(latestEpoch);
         this.validatorsStorageService.setLastUpdate(Math.floor(Date.now() / 1000));
 
+        const frameBalances = await this.getLidoValidatorsWithdrawableBalances(data);
+        this.validatorsStorageService.setFrameBalances(frameBalances);
         await this.validatorsCacheService.saveDataToCache();
 
+        const currentFrame = this.genesisTimeService.getFrameOfEpoch(this.genesisTimeService.getCurrentEpoch());
         this.logger.log('End update validators', {
           service: ValidatorsService.SERVICE_LOG_NAME,
           totalValidators,
           latestEpoch,
+          frameBalances: stringifyFrameBalances(frameBalances),
+          currentFrame,
+        });
+
+        Object.keys(frameBalances).forEach((frame) => {
+          this.prometheusService.validatorsState
+            .labels({
+              frame,
+              balance: frameBalances[frame],
+            })
+            .inc();
         });
       },
     );
   }
 
-  protected async setLidoValidatorsWithdrawableBalances(validators: Validator[]) {
+  protected async getLidoValidatorsWithdrawableBalances(validators: Validator[]) {
     const keysData = await this.lidoKeys.fetchLidoKeysData();
     const lidoValidators = await this.lidoKeys.getLidoValidatorsByKeys(keysData.data, validators);
 
@@ -111,6 +127,6 @@ export class ValidatorsService {
       await unblock();
     }
 
-    this.validatorsStorageService.setFrameBalances(frameBalances);
+    return frameBalances;
   }
 }

src/storage/validators/validators-cache.service.ts

@@ -4,7 +4,7 @@ import * as path from 'path';
 import { LOGGER_PROVIDER, LoggerService } from '../../common/logger';
 import { ValidatorsStorageService } from './validators.service';
 import { BigNumber } from '@ethersproject/bignumber';
-import { parseEther } from '@ethersproject/units';
+import { stringifyFrameBalances } from '../../common/validators/strigify-frame-balances';
 
 @Injectable()
 export class ValidatorsCacheService {
@@ -76,7 +76,7 @@ export class ValidatorsCacheService {
       this.validatorsStorage.getTotal(),
       this.validatorsStorage.getMaxExitEpoch(),
       this.validatorsStorage.getLastUpdate(),
-      this.stringifyFrameBalances(this.validatorsStorage.getFrameBalances()),
+      stringifyFrameBalances(this.validatorsStorage.getFrameBalances()),
     ].join(ValidatorsCacheService.CACHE_DATA_DIVIDER);
     await writeFile(cacheFileName, data);
     this.logger.log(`success save to file ${cacheFileName}`, { service: ValidatorsCacheService.SERVICE_LOG_NAME });
@@ -86,18 +86,10 @@ export class ValidatorsCacheService {
     return path.join(ValidatorsCacheService.CACHE_DIR, ValidatorsCacheService.CACHE_FILE_NAME);
   };
 
-  protected stringifyFrameBalances(frameBalances: Record<string, BigNumber>) {
-    return JSON.stringify(
-      Object.keys(frameBalances).reduce((acc, key) => {
-        return { ...acc, [key]: frameBalances[key].toString() };
-      }, {}),
-    );
-  }
-
   protected parseFrameBalances(frameBalancesStr: string) {
     const frameBalances = JSON.parse(frameBalancesStr);
     return Object.keys(frameBalances).reduce((acc, key) => {
-      return { ...acc, [key]: parseEther(frameBalances[key]) };
+      return { ...acc, [key]: BigNumber.from(frameBalances[key]) };
     }, {});
   }
 }

src/waiting-time/utils/calculate-frame-by-validator-balances.ts

@@ -17,7 +17,7 @@ export const calculateFrameByValidatorBalances = (args: calculateFrameByValidato
   let lastFrame = BigNumber.from(currentFrame);
 
   const frames = Object.keys(frameBalances);
-  let result = null;
+  let result: BigNumber = null;
 
   for (let i = 0; i < frames.length; i++) {
     const frame = frames[i];