Skip to content

Commit 28f4bc8

Browse files
authored
Merge branch 'main' into gw-ipld-notfound
2 parents 589f56f + 42c0c86 commit 28f4bc8

File tree

27 files changed

+567
-202
lines changed

27 files changed

+567
-202
lines changed

CHANGELOG.md

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,16 @@ The following emojis are used to highlight certain changes:
1818

1919
### Changed
2020

21+
- `bitswap/server` minor memory use and performance improvements
22+
- `bitswap` unify logger names to use uniform format bitswap/path/pkgname
23+
- `gateway` now always returns meaningful cache-control headers for generated HTML listings of UnixFS directories
24+
2125
### Removed
2226

2327
### Fixed
2428

2529
- `boxo/gateway` now returns 404 Status Not Found instead of 500 when the requested data cannot be found, without a fallback on bitswap or similar restriction.
30+
- `bitswap/client` fix memory leak in BlockPresenceManager due to unlimited map growth.
2631

2732
### Security
2833

@@ -31,7 +36,7 @@ The following emojis are used to highlight certain changes:
3136
### Changed
3237

3338
- `boxo/gateway` is now tested against [gateway-conformance v6](https://github.com/ipfs/gateway-conformance/releases/tag/v0.6.0)
34-
- `bitswap/client` supports additional tracing
39+
- `bitswap/client` supports additional tracing
3540

3641
### Removed
3742

@@ -41,6 +46,7 @@ The following emojis are used to highlight certain changes:
4146

4247
- `routing/http`: the `FindPeer` now returns `routing.ErrNotFound` when no addresses are found
4348
- `routing/http`: the `FindProvidersAsync` no longer causes a goroutine buildup
49+
- `bitswap`: wantlist overflow handling now cancels existing entries to make room for newer entries. This fix prevents the wantlist from filling up with CIDs that the server does not have.
4450

4551
## [v0.20.0]
4652

README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ Boxo includes high-quality components useful for interacting with IPFS protocols
8585
- Interacting with public and private IPFS networks
8686
- Working with content-addressed data
8787

88-
Boxo aims to provide a cohesive interface into these components. Note that not all of the underlying components necessarily reside in this respository.
88+
Boxo aims to provide a cohesive interface into these components. Note that not all of the underlying components necessarily reside in this repository.
8989

9090
### Does Boxo == IPFS?
9191

bitswap/client/client.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ import (
3838
"go.opentelemetry.io/otel/trace"
3939
)
4040

41-
var log = logging.Logger("bitswap-client")
41+
var log = logging.Logger("bitswap/client")
4242

4343
// Option defines the functional option type that can be used to configure
4444
// bitswap instances

bitswap/client/internal/blockpresencemanager/blockpresencemanager.go

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,7 @@ type BlockPresenceManager struct {
1515
}
1616

1717
func New() *BlockPresenceManager {
18-
return &BlockPresenceManager{
19-
presence: make(map[cid.Cid]map[peer.ID]bool),
20-
}
18+
return &BlockPresenceManager{}
2119
}
2220

2321
// ReceiveFrom is called when a peer sends us information about which blocks
@@ -26,6 +24,10 @@ func (bpm *BlockPresenceManager) ReceiveFrom(p peer.ID, haves []cid.Cid, dontHav
2624
bpm.Lock()
2725
defer bpm.Unlock()
2826

27+
if bpm.presence == nil {
28+
bpm.presence = make(map[cid.Cid]map[peer.ID]bool)
29+
}
30+
2931
for _, c := range haves {
3032
bpm.updateBlockPresence(p, c, true)
3133
}
@@ -75,6 +77,10 @@ func (bpm *BlockPresenceManager) AllPeersDoNotHaveBlock(peers []peer.ID, ks []ci
7577
bpm.RLock()
7678
defer bpm.RUnlock()
7779

80+
if len(bpm.presence) == 0 {
81+
return nil
82+
}
83+
7884
var res []cid.Cid
7985
for _, c := range ks {
8086
if bpm.allDontHave(peers, c) {
@@ -90,6 +96,9 @@ func (bpm *BlockPresenceManager) allDontHave(peers []peer.ID, c cid.Cid) bool {
9096
if !cok {
9197
return false
9298
}
99+
if len(ps) == 0 {
100+
return false
101+
}
93102

94103
// Check if we explicitly know that all the given peers do not have the cid
95104
for _, p := range peers {
@@ -108,6 +117,25 @@ func (bpm *BlockPresenceManager) RemoveKeys(ks []cid.Cid) {
108117
for _, c := range ks {
109118
delete(bpm.presence, c)
110119
}
120+
if len(bpm.presence) == 0 {
121+
bpm.presence = nil
122+
}
123+
}
124+
125+
// RemovePeer removes the given peer from every cid key in the presence map.
126+
func (bpm *BlockPresenceManager) RemovePeer(p peer.ID) {
127+
bpm.Lock()
128+
defer bpm.Unlock()
129+
130+
for c, pm := range bpm.presence {
131+
delete(pm, p)
132+
if len(pm) == 0 {
133+
delete(bpm.presence, c)
134+
}
135+
}
136+
if len(bpm.presence) == 0 {
137+
bpm.presence = nil
138+
}
111139
}
112140

113141
// HasKey indicates whether the BlockPresenceManager is tracking the given key

bitswap/client/internal/blockpresencemanager/blockpresencemanager_test.go

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,27 @@ func TestBlockPresenceManager(t *testing.T) {
9393
if bpm.PeerDoesNotHaveBlock(p, c1) {
9494
t.Fatal(expDoesNotHaveFalseMsg)
9595
}
96+
97+
bpm.ReceiveFrom(p, []cid.Cid{c0}, []cid.Cid{c1})
98+
if !bpm.PeerHasBlock(p, c0) {
99+
t.Fatal(expHasTrueMsg)
100+
}
101+
if !bpm.PeerDoesNotHaveBlock(p, c1) {
102+
t.Fatal(expDoesNotHaveTrueMsg)
103+
}
104+
bpm.RemovePeer(p)
105+
if bpm.PeerHasBlock(p, c0) {
106+
t.Fatal(expHasFalseMsg)
107+
}
108+
if bpm.PeerDoesNotHaveBlock(p, c0) {
109+
t.Fatal(expDoesNotHaveFalseMsg)
110+
}
111+
if bpm.PeerHasBlock(p, c1) {
112+
t.Fatal(expHasFalseMsg)
113+
}
114+
if bpm.PeerDoesNotHaveBlock(p, c1) {
115+
t.Fatal(expDoesNotHaveFalseMsg)
116+
}
96117
}
97118

98119
func TestAddRemoveMulti(t *testing.T) {

bitswap/client/internal/getter/getter.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import (
1313
ipld "github.com/ipfs/go-ipld-format"
1414
)
1515

16-
var log = logging.Logger("bitswap")
16+
var log = logging.Logger("bitswap/client/getter")
1717

1818
// GetBlocksFunc is any function that can take an array of CIDs and return a
1919
// channel of incoming blocks.

bitswap/client/internal/messagequeue/messagequeue.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ import (
1919
)
2020

2121
var (
22-
log = logging.Logger("bitswap")
22+
log = logging.Logger("bitswap/client/msgq")
2323
sflog = log.Desugar()
2424
)
2525

bitswap/client/internal/messagequeue/messagequeue_test.go

Lines changed: 17 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,13 @@ import (
1414
bsmsg "github.com/ipfs/boxo/bitswap/message"
1515
pb "github.com/ipfs/boxo/bitswap/message/pb"
1616
bsnet "github.com/ipfs/boxo/bitswap/network"
17-
"github.com/ipfs/boxo/internal/test"
1817
cid "github.com/ipfs/go-cid"
1918
peer "github.com/libp2p/go-libp2p/core/peer"
2019
"github.com/libp2p/go-libp2p/p2p/protocol/ping"
2120
)
2221

22+
const collectTimeout = 200 * time.Millisecond
23+
2324
type fakeMessageNetwork struct {
2425
connectError error
2526
messageSenderError error
@@ -172,7 +173,7 @@ func TestStartupAndShutdown(t *testing.T) {
172173

173174
messageQueue.Startup()
174175
messageQueue.AddBroadcastWantHaves(bcstwh)
175-
messages := collectMessages(ctx, t, messagesSent, 100*time.Millisecond)
176+
messages := collectMessages(ctx, t, messagesSent, collectTimeout)
176177
if len(messages) != 1 {
177178
t.Fatal("wrong number of messages were sent for broadcast want-haves")
178179
}
@@ -212,16 +213,14 @@ func TestSendingMessagesDeduped(t *testing.T) {
212213
messageQueue.Startup()
213214
messageQueue.AddWants(wantBlocks, wantHaves)
214215
messageQueue.AddWants(wantBlocks, wantHaves)
215-
messages := collectMessages(ctx, t, messagesSent, 100*time.Millisecond)
216+
messages := collectMessages(ctx, t, messagesSent, collectTimeout)
216217

217218
if totalEntriesLength(messages) != len(wantHaves)+len(wantBlocks) {
218219
t.Fatal("Messages were not deduped")
219220
}
220221
}
221222

222223
func TestSendingMessagesPartialDupe(t *testing.T) {
223-
test.Flaky(t)
224-
225224
ctx := context.Background()
226225
messagesSent := make(chan []bsmsg.Entry)
227226
resetChan := make(chan struct{}, 1)
@@ -235,16 +234,14 @@ func TestSendingMessagesPartialDupe(t *testing.T) {
235234
messageQueue.Startup()
236235
messageQueue.AddWants(wantBlocks[:8], wantHaves[:8])
237236
messageQueue.AddWants(wantBlocks[3:], wantHaves[3:])
238-
messages := collectMessages(ctx, t, messagesSent, 20*time.Millisecond)
237+
messages := collectMessages(ctx, t, messagesSent, 5*collectTimeout)
239238

240239
if totalEntriesLength(messages) != len(wantHaves)+len(wantBlocks) {
241240
t.Fatal("messages were not correctly deduped")
242241
}
243242
}
244243

245244
func TestSendingMessagesPriority(t *testing.T) {
246-
test.Flaky(t)
247-
248245
ctx := context.Background()
249246
messagesSent := make(chan []bsmsg.Entry)
250247
resetChan := make(chan struct{}, 1)
@@ -262,7 +259,7 @@ func TestSendingMessagesPriority(t *testing.T) {
262259
messageQueue.Startup()
263260
messageQueue.AddWants(wantBlocks1, wantHaves1)
264261
messageQueue.AddWants(wantBlocks2, wantHaves2)
265-
messages := collectMessages(ctx, t, messagesSent, 20*time.Millisecond)
262+
messages := collectMessages(ctx, t, messagesSent, 5*collectTimeout)
266263

267264
if totalEntriesLength(messages) != len(wantHaves)+len(wantBlocks) {
268265
t.Fatal("wrong number of wants")
@@ -327,7 +324,7 @@ func TestCancelOverridesPendingWants(t *testing.T) {
327324
messageQueue.Startup()
328325
messageQueue.AddWants(wantBlocks, wantHaves)
329326
messageQueue.AddCancels(cancels)
330-
messages := collectMessages(ctx, t, messagesSent, 100*time.Millisecond)
327+
messages := collectMessages(ctx, t, messagesSent, collectTimeout)
331328

332329
if totalEntriesLength(messages) != len(wantHaves)+len(wantBlocks)-len(cancels) {
333330
t.Fatal("Wrong message count")
@@ -351,7 +348,7 @@ func TestCancelOverridesPendingWants(t *testing.T) {
351348
// Cancel the remaining want-blocks and want-haves
352349
cancels = append(wantHaves, wantBlocks...)
353350
messageQueue.AddCancels(cancels)
354-
messages = collectMessages(ctx, t, messagesSent, 100*time.Millisecond)
351+
messages = collectMessages(ctx, t, messagesSent, collectTimeout)
355352

356353
// The remaining 2 cancels should be sent to the network as they are for
357354
// wants that were sent to the network
@@ -379,7 +376,7 @@ func TestWantOverridesPendingCancels(t *testing.T) {
379376
// Add 1 want-block and 2 want-haves
380377
messageQueue.AddWants(wantBlocks, wantHaves)
381378

382-
messages := collectMessages(ctx, t, messagesSent, 100*time.Millisecond)
379+
messages := collectMessages(ctx, t, messagesSent, collectTimeout)
383380
if totalEntriesLength(messages) != len(wantBlocks)+len(wantHaves) {
384381
t.Fatal("Wrong message count", totalEntriesLength(messages))
385382
}
@@ -389,7 +386,7 @@ func TestWantOverridesPendingCancels(t *testing.T) {
389386
// Override one cancel with a want-block (before cancel is sent to network)
390387
messageQueue.AddWants(cids[:1], []cid.Cid{})
391388

392-
messages = collectMessages(ctx, t, messagesSent, 100*time.Millisecond)
389+
messages = collectMessages(ctx, t, messagesSent, collectTimeout)
393390
if totalEntriesLength(messages) != 3 {
394391
t.Fatal("Wrong message count", totalEntriesLength(messages))
395392
}
@@ -531,7 +528,7 @@ func TestSendingLargeMessages(t *testing.T) {
531528

532529
messageQueue.Startup()
533530
messageQueue.AddWants(wantBlocks, []cid.Cid{})
534-
messages := collectMessages(ctx, t, messagesSent, 100*time.Millisecond)
531+
messages := collectMessages(ctx, t, messagesSent, 5*collectTimeout)
535532

536533
// want-block has size 44, so with maxMsgSize 44 * 3 (3 want-blocks), then if
537534
// we send 10 want-blocks we should expect 4 messages:
@@ -563,7 +560,7 @@ func TestSendToPeerThatDoesntSupportHave(t *testing.T) {
563560
// Check broadcast want-haves
564561
bcwh := testutil.GenerateCids(10)
565562
messageQueue.AddBroadcastWantHaves(bcwh)
566-
messages := collectMessages(ctx, t, messagesSent, 100*time.Millisecond)
563+
messages := collectMessages(ctx, t, messagesSent, collectTimeout)
567564

568565
if len(messages) != 1 {
569566
t.Fatal("wrong number of messages were sent", len(messages))
@@ -582,7 +579,7 @@ func TestSendToPeerThatDoesntSupportHave(t *testing.T) {
582579
wbs := testutil.GenerateCids(10)
583580
whs := testutil.GenerateCids(10)
584581
messageQueue.AddWants(wbs, whs)
585-
messages = collectMessages(ctx, t, messagesSent, 100*time.Millisecond)
582+
messages = collectMessages(ctx, t, messagesSent, collectTimeout)
586583

587584
if len(messages) != 1 {
588585
t.Fatal("wrong number of messages were sent", len(messages))
@@ -612,7 +609,7 @@ func TestSendToPeerThatDoesntSupportHaveMonitorsTimeouts(t *testing.T) {
612609

613610
wbs := testutil.GenerateCids(10)
614611
messageQueue.AddWants(wbs, nil)
615-
collectMessages(ctx, t, messagesSent, 100*time.Millisecond)
612+
collectMessages(ctx, t, messagesSent, collectTimeout)
616613

617614
// Check want-blocks are added to DontHaveTimeoutMgr
618615
if dhtm.pendingCount() != len(wbs) {
@@ -621,7 +618,7 @@ func TestSendToPeerThatDoesntSupportHaveMonitorsTimeouts(t *testing.T) {
621618

622619
cancelCount := 2
623620
messageQueue.AddCancels(wbs[:cancelCount])
624-
collectMessages(ctx, t, messagesSent, 100*time.Millisecond)
621+
collectMessages(ctx, t, messagesSent, collectTimeout)
625622

626623
// Check want-blocks are removed from DontHaveTimeoutMgr
627624
if dhtm.pendingCount() != len(wbs)-cancelCount {
@@ -692,9 +689,9 @@ func TestResponseReceivedAppliesForFirstResponseOnly(t *testing.T) {
692689

693690
cids := testutil.GenerateCids(2)
694691

695-
// Add some wants and wait 10ms
692+
// Add some wants and wait
696693
messageQueue.AddWants(cids, nil)
697-
collectMessages(ctx, t, messagesSent, 100*time.Millisecond)
694+
collectMessages(ctx, t, messagesSent, collectTimeout)
698695

699696
// Receive a response for the wants
700697
messageQueue.ResponseReceived(cids)

bitswap/client/internal/peermanager/peermanager.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import (
1111
peer "github.com/libp2p/go-libp2p/core/peer"
1212
)
1313

14-
var log = logging.Logger("bs:peermgr")
14+
var log = logging.Logger("bitswap/client/peermgr")
1515

1616
// PeerQueue provides a queue of messages to be sent for a single peer.
1717
type PeerQueue interface {

bitswap/client/internal/providerquerymanager/providerquerymanager.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import (
1313
"go.opentelemetry.io/otel/trace"
1414
)
1515

16-
var log = logging.Logger("bitswap")
16+
var log = logging.Logger("bitswap/client/provqrymgr")
1717

1818
const (
1919
maxProviders = 10

bitswap/client/internal/session/session.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ import (
2020
)
2121

2222
var (
23-
log = logging.Logger("bs:sess")
23+
log = logging.Logger("bitswap/session")
2424
sflog = log.Desugar()
2525
)
2626

bitswap/client/internal/session/sessionwantsender.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -455,6 +455,7 @@ func (sws *sessionWantSender) processUpdates(updates []update) []cid.Cid {
455455
go func() {
456456
for p := range prunePeers {
457457
// Peer doesn't have anything we want, so remove it
458+
sws.bpm.RemovePeer(p)
458459
log.Infof("peer %s sent too many dont haves, removing from session %d", p, sws.ID())
459460
sws.SignalAvailability(p, false)
460461
}

bitswap/client/internal/sessionpeermanager/sessionpeermanager.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import (
99
peer "github.com/libp2p/go-libp2p/core/peer"
1010
)
1111

12-
var log = logging.Logger("bs:sprmgr")
12+
var log = logging.Logger("bitswap/client/sesspeermgr")
1313

1414
const (
1515
// Connection Manager tag value for session peers. Indicates to connection

0 commit comments

Comments
 (0)