feat!: lighten retry logic for LightPush (#2182)

* feat: lighten retry logic for LightPush

* update tests

* remove base protocol sdk from light push, add unit tests for light push

* remove replaced test

* ensure numPeersToUse is respected

* skip tests

Author: weboko

packages/interfaces/src/light_push.ts

@@ -1,5 +1,4 @@
-import { IBaseProtocolCore, IBaseProtocolSDK } from "./protocols.js";
+import { IBaseProtocolCore } from "./protocols.js";
 import type { ISender } from "./sender.js";
 
-export type ILightPush = ISender &
-  IBaseProtocolSDK & { protocol: IBaseProtocolCore };
+export type ILightPush = ISender & { protocol: IBaseProtocolCore };

packages/interfaces/src/sender.ts

@@ -1,10 +1,23 @@
 import type { IEncoder, IMessage } from "./message.js";
-import { ProtocolUseOptions, SDKProtocolResult } from "./protocols.js";
+import { SDKProtocolResult } from "./protocols.js";
+
+export type ISenderOptions = {
+  /**
+   * Enables retry of a message that was failed to be sent.
+   * @default false
+   */
+  autoRetry?: boolean;
+  /**
+   * Sets number of attempts if `autoRetry` is enabled.
+   * @default 3
+   */
+  maxAttempts?: number;
+};
 
 export interface ISender {
   send: (
     encoder: IEncoder,
     message: IMessage,
-    sendOptions?: ProtocolUseOptions
+    sendOptions?: ISenderOptions
   ) => Promise<SDKProtocolResult>;
 }

packages/sdk/src/protocols/base_protocol.ts

@@ -11,7 +11,7 @@ interface Options {
   maintainPeersInterval?: number;
 }
 
-const DEFAULT_NUM_PEERS_TO_USE = 2;
+export const DEFAULT_NUM_PEERS_TO_USE = 2;
 const DEFAULT_MAINTAIN_PEERS_INTERVAL = 30_000;
 
 export class BaseProtocolSDK implements IBaseProtocolSDK {
@@ -29,20 +29,20 @@ export class BaseProtocolSDK implements IBaseProtocolSDK {
   ) {
     this.log = new Logger(`sdk:${core.multicodec}`);
 
-    this.peerManager = new PeerManager(connectionManager, core, this.log);
-
     this.numPeersToUse = options?.numPeersToUse ?? DEFAULT_NUM_PEERS_TO_USE;
     const maintainPeersInterval =
       options?.maintainPeersInterval ?? DEFAULT_MAINTAIN_PEERS_INTERVAL;
 
+    this.peerManager = new PeerManager(connectionManager, core, this.log);
+
     this.log.info(
       `Initializing BaseProtocolSDK with numPeersToUse: ${this.numPeersToUse}, maintainPeersInterval: ${maintainPeersInterval}ms`
     );
     void this.startMaintainPeersInterval(maintainPeersInterval);
   }
 
   public get connectedPeers(): Peer[] {
-    return this.peerManager.getPeers();
+    return this.peerManager.getPeers().slice(0, this.numPeersToUse);
   }
 
   /**

packages/sdk/src/protocols/light_push/light_push.spec.ts

@@ -0,0 +1,170 @@
+import { Peer } from "@libp2p/interface";
+import {
+  ConnectionManager,
+  createEncoder,
+  Encoder,
+  LightPushCodec
+} from "@waku/core";
+import { Libp2p, ProtocolError } from "@waku/interfaces";
+import { utf8ToBytes } from "@waku/utils/bytes";
+import { expect } from "chai";
+import sinon from "sinon";
+
+import { LightPush } from "./light_push.js";
+
+const PUBSUB_TOPIC = "/waku/2/rs/1/4";
+const CONTENT_TOPIC = "/test/1/waku-light-push/utf8";
+
+describe("LightPush SDK", () => {
+  let libp2p: Libp2p;
+  let encoder: Encoder;
+  let lightPush: LightPush;
+
+  beforeEach(() => {
+    libp2p = mockLibp2p();
+    encoder = createEncoder({ contentTopic: CONTENT_TOPIC });
+    lightPush = mockLightPush({ libp2p });
+  });
+
+  it("should fail to send if pubsub topics are misconfigured", async () => {
+    lightPush = mockLightPush({ libp2p, pubsubTopics: ["/wrong"] });
+
+    const result = await lightPush.send(encoder, {
+      payload: utf8ToBytes("test")
+    });
+    const failures = result.failures ?? [];
+
+    expect(failures.length).to.be.eq(1);
+    expect(failures.some((v) => v.error === ProtocolError.TOPIC_NOT_CONFIGURED))
+      .to.be.true;
+  });
+
+  it("should fail to send if no connected peers found", async () => {
+    const result = await lightPush.send(encoder, {
+      payload: utf8ToBytes("test")
+    });
+    const failures = result.failures ?? [];
+
+    expect(failures.length).to.be.eq(1);
+    expect(failures.some((v) => v.error === ProtocolError.NO_PEER_AVAILABLE)).to
+      .be.true;
+  });
+
+  it("should send to specified number of peers of used peers", async () => {
+    libp2p = mockLibp2p({
+      peers: [mockPeer("1"), mockPeer("2"), mockPeer("3"), mockPeer("4")]
+    });
+
+    // check default value that should be 2
+    lightPush = mockLightPush({ libp2p });
+    let sendSpy = sinon.spy(
+      (_encoder: any, _message: any, peer: Peer) =>
+        ({ success: peer.id }) as any
+    );
+    lightPush.protocol.send = sendSpy;
+
+    let result = await lightPush.send(encoder, {
+      payload: utf8ToBytes("test")
+    });
+
+    expect(sendSpy.calledTwice).to.be.true;
+    expect(result.successes?.length).to.be.eq(2);
+
+    // check if setting another value works
+    lightPush = mockLightPush({ libp2p, numPeersToUse: 3 });
+    sendSpy = sinon.spy(
+      (_encoder: any, _message: any, peer: Peer) =>
+        ({ success: peer.id }) as any
+    );
+    lightPush.protocol.send = sendSpy;
+
+    result = await lightPush.send(encoder, { payload: utf8ToBytes("test") });
+
+    expect(sendSpy.calledThrice).to.be.true;
+    expect(result.successes?.length).to.be.eq(3);
+  });
+
+  it("should retry on failure if specified", async () => {
+    libp2p = mockLibp2p({
+      peers: [mockPeer("1"), mockPeer("2")]
+    });
+
+    lightPush = mockLightPush({ libp2p });
+    let sendSpy = sinon.spy((_encoder: any, _message: any, peer: Peer) => {
+      if (peer.id.toString() === "1") {
+        return { success: peer.id };
+      }
+
+      return { failure: { error: "problem" } };
+    });
+    lightPush.protocol.send = sendSpy as any;
+    const attemptRetriesSpy = sinon.spy(lightPush["attemptRetries"]);
+    lightPush["attemptRetries"] = attemptRetriesSpy;
+
+    const result = await lightPush.send(
+      encoder,
+      { payload: utf8ToBytes("test") },
+      { autoRetry: true }
+    );
+
+    expect(attemptRetriesSpy.calledOnce).to.be.true;
+    expect(result.successes?.length).to.be.eq(1);
+    expect(result.failures?.length).to.be.eq(1);
+
+    sendSpy = sinon.spy(() => ({ failure: { error: "problem" } })) as any;
+    await lightPush["attemptRetries"](sendSpy as any);
+
+    expect(sendSpy.callCount).to.be.eq(3);
+
+    sendSpy = sinon.spy(() => ({ failure: { error: "problem" } })) as any;
+    await lightPush["attemptRetries"](sendSpy as any, 2);
+
+    expect(sendSpy.callCount).to.be.eq(2);
+  });
+});
+
+type MockLibp2pOptions = {
+  peers?: Peer[];
+};
+
+function mockLibp2p(options?: MockLibp2pOptions): Libp2p {
+  const peers = options?.peers || [];
+  const peerStore = {
+    get: (id: any) => Promise.resolve(peers.find((p) => p.id === id))
+  };
+
+  return {
+    peerStore,
+    getPeers: () => peers.map((p) => p.id),
+    components: {
+      events: new EventTarget(),
+      connectionManager: {
+        getConnections: () => []
+      } as any,
+      peerStore
+    }
+  } as unknown as Libp2p;
+}
+
+type MockLightPushOptions = {
+  libp2p: Libp2p;
+  pubsubTopics?: string[];
+  numPeersToUse?: number;
+};
+
+function mockLightPush(options: MockLightPushOptions): LightPush {
+  return new LightPush(
+    {
+      configuredPubsubTopics: options.pubsubTopics || [PUBSUB_TOPIC]
+    } as ConnectionManager,
+    options.libp2p,
+    { numPeersToUse: options.numPeersToUse }
+  );
+}
+
+function mockPeer(id: string): Peer {
+  return {
+    id,
+    protocols: [LightPushCodec]
+  } as unknown as Peer;
+}

packages/sdk/src/protocols/light_push/light_push.ts

@@ -6,53 +6,51 @@ import {
   LightPushCore
 } from "@waku/core";
 import {
+  type CoreProtocolResult,
   Failure,
   type IEncoder,
   ILightPush,
   type IMessage,
+  type ISenderOptions,
   type Libp2p,
   type ProtocolCreateOptions,
   ProtocolError,
-  ProtocolUseOptions,
   SDKProtocolResult
 } from "@waku/interfaces";
 import { ensurePubsubTopicIsConfigured, Logger } from "@waku/utils";
 
-import { ReliabilityMonitorManager } from "../../reliability_monitor/index.js";
-import { SenderReliabilityMonitor } from "../../reliability_monitor/sender.js";
-import { BaseProtocolSDK } from "../base_protocol.js";
+import { DEFAULT_NUM_PEERS_TO_USE } from "../base_protocol.js";
 
 const log = new Logger("sdk:light-push");
 
-class LightPush extends BaseProtocolSDK implements ILightPush {
-  public readonly protocol: LightPushCore;
+const DEFAULT_MAX_ATTEMPTS = 3;
+const DEFAULT_SEND_OPTIONS: ISenderOptions = {
+  autoRetry: false,
+  maxAttempts: DEFAULT_MAX_ATTEMPTS
+};
+
+type RetryCallback = (peer: Peer) => Promise<CoreProtocolResult>;
 
-  private readonly reliabilityMonitor: SenderReliabilityMonitor;
+export class LightPush implements ILightPush {
+  private numPeersToUse: number = DEFAULT_NUM_PEERS_TO_USE;
+  public readonly protocol: LightPushCore;
 
   public constructor(
     connectionManager: ConnectionManager,
     private libp2p: Libp2p,
     options?: ProtocolCreateOptions
   ) {
-    super(
-      new LightPushCore(connectionManager.configuredPubsubTopics, libp2p),
-      connectionManager,
-      {
-        numPeersToUse: options?.numPeersToUse
-      }
+    this.numPeersToUse = options?.numPeersToUse ?? DEFAULT_NUM_PEERS_TO_USE;
+    this.protocol = new LightPushCore(
+      connectionManager.configuredPubsubTopics,
+      libp2p
     );
-
-    this.reliabilityMonitor = ReliabilityMonitorManager.createSenderMonitor(
-      this.renewPeer.bind(this)
-    );
-
-    this.protocol = this.core as LightPushCore;
   }
 
   public async send(
     encoder: IEncoder,
     message: IMessage,
-    _options?: ProtocolUseOptions
+    options: ISenderOptions = DEFAULT_SEND_OPTIONS
   ): Promise<SDKProtocolResult> {
     const successes: PeerId[] = [];
     const failures: Failure[] = [];
@@ -105,14 +103,10 @@ class LightPush extends BaseProtocolSDK implements ILightPush {
       if (failure) {
         failures.push(failure);
 
-        const connectedPeer = this.connectedPeers.find((connectedPeer) =>
-          connectedPeer.id.equals(failure.peerId)
-        );
-
-        if (connectedPeer) {
-          void this.reliabilityMonitor.attemptRetriesOrRenew(
-            connectedPeer.id,
-            () => this.protocol.send(encoder, message, connectedPeer)
+        if (options?.autoRetry) {
+          void this.attemptRetries(
+            (peer: Peer) => this.protocol.send(encoder, message, peer),
+            options.maxAttempts
           );
         }
       }
@@ -129,6 +123,32 @@ class LightPush extends BaseProtocolSDK implements ILightPush {
     };
   }
 
+  private async attemptRetries(
+    fn: RetryCallback,
+    maxAttempts?: number
+  ): Promise<void> {
+    maxAttempts = maxAttempts || DEFAULT_MAX_ATTEMPTS;
+    const connectedPeers = await this.getConnectedPeers();
+
+    if (connectedPeers.length === 0) {
+      log.warn("Cannot retry with no connected peers.");
+      return;
+    }
+
+    for (let i = 0; i < maxAttempts; i++) {
+      const peer = connectedPeers[i % connectedPeers.length]; // always present as we checked for the length already
+      const response = await fn(peer);
+
+      if (response.success) {
+        return;
+      }
+
+      log.info(
+        `Attempted retry for peer:${peer.id} failed with:${response?.failure?.error}`
+      );
+    }
+  }
+
   private async getConnectedPeers(): Promise<Peer[]> {
     const peerIDs = this.libp2p.getPeers();