changelog

guillaumemichel · guillaumemichel · commit 23b728f0647f · 2025-04-15T17:25:16.000+02:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -29,6 +29,7 @@ The following emojis are used to highlight certain changes:
 - upgrade to `go-libp2p` [v0.41.1](https://github.com/libp2p/go-libp2p/releases/tag/v0.41.1)
 - `bitswap/network`: Add a new `requests_in_flight` metric gauge that measures how many bitswap streams are being written or read at a given time.
 - improve speed of data onboarding by batching/bufering provider queue writes [#888](https://github.com/ipfs/boxo/pull/888)
+- `provider/queue`: change queue internal buffer from a `Deque` to a `lru.Cache` to deduplicate cids as they are added to the provider queue [#909](https://github.com/ipfs/boxo/pull/909)
 
 ### Removed
 
diff --git a/provider/internal/queue/queue.go b/provider/internal/queue/queue.go
@@ -277,7 +277,7 @@ func (q *Queue) commitInput(ctx context.Context, counter uint64, cids *lru.Cache
 		return fmt.Errorf("failed to create batch: %w", err)
 	}
 
-	oldestCid, _, ok := cids.GetOldest()
+	oldestCid, _, ok := cids.RemoveOldest()
 	cstr := makeCidString(oldestCid)
 	for c := oldestCid; ok; c, _, ok = cids.RemoveOldest() {
 		key := datastore.NewKey(fmt.Sprintf("%020d/%s", counter, cstr))

Original file line number	Diff line number	Diff line change
`@@ -277,7 +277,7 @@ func (q Queue) commitInput(ctx context.Context, counter uint64, cids lru.Cache`
`277`	`277`	`return fmt.Errorf("failed to create batch: %w", err)`
`278`	`278`	`}`
`279`	`279`
`280`		`- oldestCid, _, ok := cids.GetOldest()`
	`280`	`+ oldestCid, _, ok := cids.RemoveOldest()`
`281`	`281`	`cstr := makeCidString(oldestCid)`
`282`	`282`	`for c := oldestCid; ok; c, _, ok = cids.RemoveOldest() {`
`283`	`283`	`key := datastore.NewKey(fmt.Sprintf("%020d/%s", counter, cstr))`