diff --git a/src/index.ts b/src/index.ts
index d9b0445b..d26482a0 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -296,7 +296,7 @@ export class GossipSub extends TypedEventEmitter<GossipsubEvents> implements Pub
 
   // State
 
-  public readonly peers = new Set<PeerIdStr>()
+  public readonly peers = new Map<PeerIdStr, PeerId>()
   public readonly streamsInbound = new Map<PeerIdStr, InboundStream>()
   public readonly streamsOutbound = new Map<PeerIdStr, OutboundStream>()
 
@@ -602,7 +602,7 @@ export class GossipSub extends TypedEventEmitter<GossipsubEvents> implements Pub
   ]
 
   getPeers (): PeerId[] {
-    return [...this.peers.keys()].map((str) => peerIdFromString(str))
+    return [...this.peers.values()]
   }
 
   isStarted (): boolean {
@@ -915,7 +915,7 @@ export class GossipSub extends TypedEventEmitter<GossipsubEvents> implements Pub
     if (!this.peers.has(id)) {
       this.log('new peer %p', peerId)
 
-      this.peers.add(id)
+      this.peers.set(id, peerId)
 
       // Add to peer scoring
       this.score.addPeer(id)
@@ -1016,7 +1016,7 @@ export class GossipSub extends TypedEventEmitter<GossipsubEvents> implements Pub
    */
   getSubscribers (topic: TopicStr): PeerId[] {
     const peersInTopic = this.topics.get(topic)
-    return ((peersInTopic != null) ? Array.from(peersInTopic) : []).map((str) => peerIdFromString(str))
+    return ((peersInTopic != null) ? Array.from(peersInTopic) : []).map((str) => this.peers.get(str) ?? peerIdFromString(str))
   }
 
   /**
@@ -2316,7 +2316,7 @@ export class GossipSub extends TypedEventEmitter<GossipsubEvents> implements Pub
     }
 
     return {
-      recipients: Array.from(tosend.values()).map((str) => peerIdFromString(str))
+      recipients: Array.from(tosend.values()).map((str) => this.peers.get(str) ?? peerIdFromString(str))
     }
   }
 
@@ -2705,7 +2705,7 @@ export class GossipSub extends TypedEventEmitter<GossipsubEvents> implements Pub
         // the peer ID and let the pruned peer find them in the DHT -- we can't trust
         // unsigned address records through PX anyways
         // Finding signed records in the DHT is not supported at the time of writing in js-libp2p
-        const id = peerIdFromString(peerId)
+        const id = this.peers.get(peerId) ?? peerIdFromString(peerId)
         let peerInfo: Peer | undefined
 
         try {
@@ -3229,7 +3229,7 @@ export class GossipSub extends TypedEventEmitter<GossipsubEvents> implements Pub
 
   private readonly tagMeshPeer = (evt: CustomEvent<MeshPeer>): void => {
     const { peerId, topic } = evt.detail
-    this.components.peerStore.merge(peerIdFromString(peerId), {
+    this.components.peerStore.merge(this.peers.get(peerId) ?? peerIdFromString(peerId), {
       tags: {
         [topic]: {
           value: 100
@@ -3240,7 +3240,7 @@ export class GossipSub extends TypedEventEmitter<GossipsubEvents> implements Pub
 
   private readonly untagMeshPeer = (evt: CustomEvent<MeshPeer>): void => {
     const { peerId, topic } = evt.detail
-    this.components.peerStore.merge(peerIdFromString(peerId), {
+    this.components.peerStore.merge(this.peers.get(peerId) ?? peerIdFromString(peerId), {
       tags: {
         [topic]: undefined
       }