dev to goerli

.dockerignore

@@ -2,9 +2,6 @@
 /dist
 /node_modules
 
-# Env
-.env
-
 # Logs
 logs
 *.log

.github/workflows/test.yml

@@ -3,6 +3,22 @@ on: pull_request
 jobs:
   test:
     runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: postgres:16-alpine
+        env:
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_USER: postgres
+          POSTGRES_DB: node_operator_keys_service_db
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
     steps:
       - name: Checkout repo
         uses: actions/[email protected]
@@ -19,9 +35,15 @@ jobs:
         run: yarn lint
       - name: Run tests
         run: yarn test
-      - name: Run tests
-        run: yarn test:e2e
+      - name: Run E2E tests
+        run: yarn test:e2e:docker
         env:
           PROVIDERS_URLS: ${{ secrets.PROVIDERS_URLS }}
+          CL_API_URLS: "https://e2e-test.lido.fi,"
           CHAIN_ID: ${{ secrets.CHAIN_ID }}
           CHRONIX_PROVIDER_MAINNET_URL: ${{ secrets.CHRONIX_PROVIDER_MAINNET_URL }}
+          DB_NAME: node_operator_keys_service_db
+          DB_PORT: 5432
+          DB_HOST: localhost
+          DB_USER: postgres
+          DB_PASSWORD: postgres

.vscode/launch.json

@@ -0,0 +1,55 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Debug Nest Framework",
+      "runtimeExecutable": "yarn",
+      "runtimeArgs": [
+        "start:debug",
+      ],
+      "cwd": "${workspaceFolder}/src",
+      "autoAttachChildProcesses": true,
+      "restart": true,
+      "sourceMaps": true,
+      "stopOnEntry": false,
+      "console": "integratedTerminal",
+    },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Debug Jest e2e",
+      "runtimeExecutable": "yarn",
+      "runtimeArgs": [
+        "test:e2e:docker:debug",
+      ],
+      "cwd": "${workspaceFolder}/src",
+      "autoAttachChildProcesses": true,
+      "restart": true,
+      "sourceMaps": true,
+      "stopOnEntry": false,
+      "console": "integratedTerminal",
+    },
+    {
+      "name": "Jest file",
+      "type": "node",
+      "request": "launch",
+      "runtimeExecutable": "${workspaceRoot}/node_modules/.bin/jest",
+      "args": [
+        "${fileBasenameNoExtension}",
+        "--runInBand",
+        "--watch",
+        "--coverage=false",
+        "--no-cache"
+      ],
+      "cwd": "${workspaceRoot}",
+      "console": "integratedTerminal",
+      "internalConsoleOptions": "neverOpen",
+      "sourceMaps": true,
+      "windows": {
+          "program": "${workspaceFolder}/node_modules/jest/bin/jest"
+      }
+    }
+  ]
+}

Dockerfile

@@ -4,7 +4,7 @@ RUN apk add --no-cache git=2.36.6-r0
 
 WORKDIR /app
 
-COPY package.json yarn.lock chronix.config.ts ./
+COPY package.json yarn.lock ./
 COPY ./tsconfig*.json ./
 COPY ./src ./src
 

Dockerfile.e2e

@@ -0,0 +1,15 @@
+FROM node:18.14.2-alpine3.16 as building
+
+RUN apk add --no-cache git=2.36.6-r0
+
+WORKDIR /app
+
+COPY package.json yarn.lock chronix.config.ts .env ./
+COPY jest* ./
+COPY ./tsconfig*.json ./
+
+RUN yarn install --frozen-lockfile --non-interactive && yarn cache clean
+COPY ./src ./src
+RUN yarn typechain
+
+CMD ["yarn", "test:e2e:docker"]

docker-compose.e2e.yml

@@ -0,0 +1,27 @@
+version: '3.7'
+
+services:
+  e2e_pgdb:
+    container_name: e2e_pgdb
+    image: postgres:14-alpine
+    restart: unless-stopped
+    environment:
+      - POSTGRES_DB=${DB_NAME}
+      - POSTGRES_USER=${DB_USER}
+      - POSTGRES_PASSWORD=${DB_PASSWORD}
+    expose:
+      - 5432:5432
+    volumes:
+      - ./.volumes/pgdata-${CHAIN_ID}/:/var/lib/postgresql/data
+
+  e2e_keys_api:
+    container_name: e2e_keys_api
+    build:
+      context: ./
+      dockerfile: Dockerfile.e2e
+    environment:
+      - NODE_ENV=production
+      - DB_PORT=5432
+      - DB_HOST=e2e_pgdb
+    depends_on:
+      - e2e_pgdb

nest-cli.json

@@ -1,4 +1,7 @@
 {
   "collection": "@nestjs/schematics",
-  "sourceRoot": "src"
+  "sourceRoot": "src",
+  "compilerOptions": {
+    "tsConfigPath": "./tsconfig.build.json"
+  }
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "lido-keys-api",
-  "version": "0.8.2",
+  "version": "0.10.1",
   "description": "Lido Node Operators keys service",
   "author": "Lido team",
   "private": true,
@@ -20,7 +20,7 @@
     "url": "https://github.com/lidofinance/lido-keys-api/issues"
   },
   "scripts": {
-    "postinstall": "rimraf ./node_modules/@ethersproject/abstract-provider/src.ts/*.ts && yarn chronix:compile",
+    "postinstall": "rimraf ./node_modules/@ethersproject/abstract-provider/src.ts/*.ts",
     "mikro-orm": "./node_modules/mikro-orm",
     "prebuild": "rimraf dist",
     "build": "nest build",
@@ -34,7 +34,9 @@
     "test:watch": "jest --watch",
     "test:cov": "jest --coverage",
     "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",
-    "test:e2e": "jest --config jest-e2e.json && chronix test",
+    "test:e2e": "docker-compose --env-file=./.env -f docker-compose.e2e.yml up --build --abort-on-container-exit",
+    "test:e2e:docker": "mikro-orm schema:drop -r && mikro-orm migration:up && jest -i --config jest-e2e.json && chronix compile && chronix test",
+    "test:e2e:docker:debug": "mikro-orm schema:drop -r && mikro-orm migration:up && node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest -i --config jest-e2e.json && chronix compile && chronix test",
     "typechain": "typechain --target=ethers-v5 --out-dir ./src/generated ./src/staking-router-modules/contracts/abi/*.json",
     "chronix:compile": "chronix compile",
     "chronix:test": "chronix test",
@@ -73,6 +75,7 @@
     "ethers": "^5.5.4",
     "fastify-swagger": "^4.13.1",
     "jsonstream": "^1.0.3",
+    "pg-query-stream": "^4.5.3",
     "prom-client": "^14.0.1",
     "reflect-metadata": "^0.1.13",
     "rimraf": "^3.0.2",

rest-api.md

@@ -728,6 +728,51 @@ class NotFoundException implements HttpException {
 }
 ```
 
+#### List of modules, operators, keys and meta
+
+Path: /v2/modules/operators/keys
+
+Returns modules, operators, keys and meta.
+
+Request example:
+
+```bash
+curl 'http://localhost:3000/v2/modules/operators/keys
+```
+
+Response:
+
+```typescript
+type KApiModulesOperatorsKeysResponse = {
+  key: Key | null;
+  operator: Operator | null;
+  meta: { elBlockSnapshot: ELBlockSnapshot } | null;
+  module: StakingModule | null;
+}[];
+
+interface HttpException {
+  statusCode: number;
+  message: string;
+}
+
+class TooEarlyResponse implements HttpException {
+  statusCode: number = 425;
+  message: string = 'Too early response';
+}
+```
+
+For example:
+```json
+[
+  { "meta": { "elBlockSnapshot": {} }, "module": { "id": 1, "moduleAddress": "0x001" }, "operator": { "id": 1, "moduleAddress": "0x001" }, "key": { "id": 1, "moduleAddress": "0x001" } },
+  { "meta": null, "module": null, "operator": { "id": 2, "moduleAddress": "0x001" }, "key": { "id": 2, "moduleAddress": "0x001" } },
+  { "meta": null, "module": null, "operator": null, "key": { "id": 3, "moduleAddress": "0x001" } },
+  { "meta": null, "module": { "id": 2, "moduleAddress": "0x002" }, "operator": { "id": 1, "moduleAddress": "0x002" }, "key": { "id": 1, "moduleAddress": "0x002" } },
+  { "meta": null, "module": null, "operator": { "id": 2, "moduleAddress": "0x002" }, "key": { "id": 2, "moduleAddress": "0x002" } },
+  { "meta": null, "module": null, "operator": null, "key": { "id": 3, "moduleAddress": "0x002" } }
+]
+```
+
 ### KAPI status
 
 #### Return KAPI status

src/app/database-testing.module.ts

@@ -0,0 +1,29 @@
+import { Module } from '@nestjs/common';
+import { MikroOrmModule } from '@mikro-orm/nestjs';
+import config from 'mikro-orm.config';
+import { ConfigModule, ConfigService } from 'common/config';
+
+@Module({
+  imports: [
+    ConfigModule,
+    MikroOrmModule.forRootAsync({
+      async useFactory(configService: ConfigService) {
+        return {
+          ...config,
+          dbName: configService.get('DB_NAME'),
+          host: configService.get('DB_HOST'),
+          port: configService.get('DB_PORT'),
+          user: configService.get('DB_USER'),
+          password: configService.get('DB_PASSWORD'),
+          autoLoadEntities: false,
+          cache: { enabled: false },
+          debug: false,
+          registerRequestContext: true,
+          allowGlobalContext: true,
+        };
+      },
+      inject: [ConfigService],
+    }),
+  ],
+})
+export class DatabaseTestingModule {}

src/app/index.ts

@@ -1,3 +1,4 @@
 export * from './app.constants';
 export * from './app.module';
 export * from './app.service';
+export * from './database-testing.module';

src/app/simple-dvt-deploy.e2e-chain.ts

@@ -9,7 +9,7 @@ import { RegistryKeyStorageService } from '../common/registry';
 import { ElMetaStorageService } from '../storage/el-meta.storage';
 import { SRModuleStorageService } from '../storage/sr-module.storage';
 import { KeysUpdateService } from '../jobs/keys-update';
-import { ExecutionProvider, ExecutionProviderService } from '../common/execution-provider';
+import { ExecutionProvider } from '../common/execution-provider';
 import { ConfigService } from '../common/config';
 import { PrometheusService } from '../common/prometheus';
 import { StakingRouterService } from '../staking-router-modules/staking-router.service';
@@ -64,8 +64,6 @@ describe('Simple DVT deploy', () => {
     });
 
     moduleRef = await Test.createTestingModule({ imports: [AppModule] })
-      .overrideProvider(ExecutionProviderService)
-      .useValue(session.provider)
       .overrideProvider(SimpleFallbackJsonRpcBatchProvider)
       .useValue(session.provider)
       .overrideProvider(ExecutionProvider)
@@ -89,7 +87,9 @@ describe('Simple DVT deploy', () => {
       .overrideProvider(ConfigService)
       .useValue({
         get(path) {
-          const conf = { LIDO_LOCATOR_ADDRESS: '0xC1d0b3DE6792Bf6b4b37EccdcC24e45978Cfd2Eb' };
+          const conf = {
+            LIDO_LOCATOR_ADDRESS: '0xC1d0b3DE6792Bf6b4b37EccdcC24e45978Cfd2Eb',
+          };
           return conf[path];
         },
       })

src/common/execution-provider/execution-provider.service.ts

@@ -46,4 +46,13 @@ export class ExecutionProviderService {
     const block = await this.provider.getBlock(blockHashOrBlockTag);
     return { number: block.number, hash: block.hash, timestamp: block.timestamp };
   }
+
+  /**
+   *
+   * Returns full block info
+   */
+  public async getFullBlock(blockHashOrBlockTag: number | string) {
+    const block = await this.provider.getBlock(blockHashOrBlockTag);
+    return block;
+  }
 }

src/common/registry/fetch/interfaces/operator.interface.ts

@@ -8,4 +8,5 @@ export interface RegistryOperator {
   totalSigningKeys: number;
   usedSigningKeys: number;
   moduleAddress: string;
+  finalizedUsedSigningKeys: number;
 }