Skip to content

Commit 5d43ebb

Browse files
jbenetjbenet
committed
Merge pull request #1274 from ipfs/refactor/importer
change pinning to happen in a callback
2 parents ecdf149 + a0aa07e commit 5d43ebb

File tree

12 files changed

+105
-48
lines changed

12 files changed

+105
-48
lines changed

‎core/commands/add.go

+15-4
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ import (
77
"strings"
88

99
"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/cheggaaa/pb"
10-
context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
1110

1211
cmds "github.com/ipfs/go-ipfs/commands"
1312
files "github.com/ipfs/go-ipfs/commands/files"
@@ -16,6 +15,7 @@ import (
1615
importer "github.com/ipfs/go-ipfs/importer"
1716
"github.com/ipfs/go-ipfs/importer/chunk"
1817
dag "github.com/ipfs/go-ipfs/merkledag"
18+
pin "github.com/ipfs/go-ipfs/pin"
1919
ft "github.com/ipfs/go-ipfs/unixfs"
2020
u "github.com/ipfs/go-ipfs/util"
2121
)
@@ -113,12 +113,16 @@ remains to be implemented.
113113
return
114114
}
115115

116-
err = n.Pinning.Pin(context.Background(), rootnd, true)
116+
rnk, err := rootnd.Key()
117117
if err != nil {
118118
res.SetError(err, cmds.ErrNormal)
119119
return
120120
}
121121

122+
mp := n.Pinning.GetManual()
123+
mp.RemovePinWithMode(rnk, pin.Indirect)
124+
mp.PinWithMode(rnk, pin.Recursive)
125+
122126
err = n.Pinning.Flush()
123127
if err != nil {
124128
res.SetError(err, cmds.ErrNormal)
@@ -214,7 +218,12 @@ remains to be implemented.
214218
}
215219

216220
func add(n *core.IpfsNode, reader io.Reader) (*dag.Node, error) {
217-
node, err := importer.BuildDagFromReader(reader, n.DAG, nil, chunk.DefaultSplitter)
221+
node, err := importer.BuildDagFromReader(
222+
reader,
223+
n.DAG,
224+
chunk.DefaultSplitter,
225+
importer.PinIndirectCB(n.Pinning.GetManual()),
226+
)
218227
if err != nil {
219228
return nil, err
220229
}
@@ -290,11 +299,13 @@ func addDir(n *core.IpfsNode, dir files.File, out chan interface{}, progress boo
290299
return nil, err
291300
}
292301

293-
_, err = n.DAG.Add(tree)
302+
k, err := n.DAG.Add(tree)
294303
if err != nil {
295304
return nil, err
296305
}
297306

307+
n.Pinning.GetManual().PinWithMode(k, pin.Indirect)
308+
298309
return tree, nil
299310
}
300311

‎core/corehttp/gateway_handler.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ func (i *gatewayHandler) newDagFromReader(r io.Reader) (*dag.Node, error) {
7272
// TODO(cryptix): change and remove this helper once PR1136 is merged
7373
// return ufs.AddFromReader(i.node, r.Body)
7474
return importer.BuildDagFromReader(
75-
r, i.node.DAG, i.node.Pinning.GetManual(), chunk.DefaultSplitter)
75+
r, i.node.DAG, chunk.DefaultSplitter, importer.BasicPinnerCB(i.node.Pinning.GetManual()))
7676
}
7777

7878
// TODO(btc): break this apart into separate handlers using a more expressive muxer

‎core/coreunix/add.go

+18-15
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package coreunix
22

33
import (
4-
"errors"
54
"io"
65
"io/ioutil"
76
"os"
@@ -29,15 +28,12 @@ func Add(n *core.IpfsNode, r io.Reader) (string, error) {
2928
dagNode, err := importer.BuildDagFromReader(
3029
r,
3130
n.DAG,
32-
n.Pinning.GetManual(), // Fix this interface
3331
chunk.DefaultSplitter,
32+
importer.BasicPinnerCB(n.Pinning.GetManual()),
3433
)
3534
if err != nil {
3635
return "", err
3736
}
38-
if err := n.Pinning.Flush(); err != nil {
39-
return "", err
40-
}
4137
k, err := dagNode.Key()
4238
if err != nil {
4339
return "", err
@@ -53,18 +49,28 @@ func AddR(n *core.IpfsNode, root string) (key string, err error) {
5349
return "", err
5450
}
5551
defer f.Close()
52+
5653
ff, err := files.NewSerialFile(root, f)
5754
if err != nil {
5855
return "", err
5956
}
57+
6058
dagnode, err := addFile(n, ff)
6159
if err != nil {
6260
return "", err
6361
}
62+
6463
k, err := dagnode.Key()
6564
if err != nil {
6665
return "", err
6766
}
67+
68+
n.Pinning.GetManual().RemovePinWithMode(k, pin.Indirect)
69+
err = n.Pinning.Flush()
70+
if err != nil {
71+
return "", err
72+
}
73+
6874
return k.String(), nil
6975
}
7076

@@ -87,17 +93,14 @@ func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, *merkle
8793
}
8894

8995
func add(n *core.IpfsNode, reader io.Reader) (*merkledag.Node, error) {
90-
mp, ok := n.Pinning.(pin.ManualPinner)
91-
if !ok {
92-
return nil, errors.New("invalid pinner type! expected manual pinner")
93-
}
94-
95-
node, err := importer.BuildDagFromReader(reader, n.DAG, mp, chunk.DefaultSplitter)
96-
if err != nil {
97-
return nil, err
98-
}
96+
mp := n.Pinning.GetManual()
9997

100-
err = n.Pinning.Flush()
98+
node, err := importer.BuildDagFromReader(
99+
reader,
100+
n.DAG,
101+
chunk.DefaultSplitter,
102+
importer.PinIndirectCB(mp),
103+
)
101104
if err != nil {
102105
return nil, err
103106
}

‎core/coreunix/metadata_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ func TestMetadata(t *testing.T) {
3737
data := make([]byte, 1000)
3838
u.NewTimeSeededRand().Read(data)
3939
r := bytes.NewReader(data)
40-
nd, err := importer.BuildDagFromReader(r, ds, nil, chunk.DefaultSplitter)
40+
nd, err := importer.BuildDagFromReader(r, ds, chunk.DefaultSplitter, nil)
4141
if err != nil {
4242
t.Fatal(err)
4343
}

‎fuse/readonly/ipfs_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ func randObj(t *testing.T, nd *core.IpfsNode, size int64) (*dag.Node, []byte) {
3434
buf := make([]byte, size)
3535
u.NewTimeSeededRand().Read(buf)
3636
read := bytes.NewReader(buf)
37-
obj, err := importer.BuildTrickleDagFromReader(read, nd.DAG, nil, chunk.DefaultSplitter)
37+
obj, err := importer.BuildTrickleDagFromReader(read, nd.DAG, chunk.DefaultSplitter, nil)
3838
if err != nil {
3939
t.Fatal(err)
4040
}

‎importer/helpers/dagbuilder.go

+22-10
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,14 @@ import (
55
"github.com/ipfs/go-ipfs/pin"
66
)
77

8+
// NodeCB is callback function for dag generation
9+
// the `last` flag signifies whether or not this is the last
10+
// (top-most root) node being added. useful for things like
11+
// only pinning the first node recursively.
12+
type NodeCB func(node *dag.Node, last bool) error
13+
14+
var nilFunc NodeCB = func(_ *dag.Node, _ bool) error { return nil }
15+
816
// DagBuilderHelper wraps together a bunch of objects needed to
917
// efficiently create unixfs dag trees
1018
type DagBuilderHelper struct {
@@ -13,6 +21,7 @@ type DagBuilderHelper struct {
1321
in <-chan []byte
1422
nextData []byte // the next item to return.
1523
maxlinks int
24+
ncb NodeCB
1625
}
1726

1827
type DagBuilderParams struct {
@@ -22,18 +31,23 @@ type DagBuilderParams struct {
2231
// DAGService to write blocks to (required)
2332
Dagserv dag.DAGService
2433

25-
// Pinner to use for pinning files (optionally nil)
26-
Pinner pin.ManualPinner
34+
// Callback for each block added
35+
NodeCB NodeCB
2736
}
2837

2938
// Generate a new DagBuilderHelper from the given params, using 'in' as a
3039
// data source
3140
func (dbp *DagBuilderParams) New(in <-chan []byte) *DagBuilderHelper {
41+
ncb := dbp.NodeCB
42+
if ncb == nil {
43+
ncb = nilFunc
44+
}
45+
3246
return &DagBuilderHelper{
3347
dserv: dbp.Dagserv,
34-
mp: dbp.Pinner,
3548
in: in,
3649
maxlinks: dbp.Maxlinks,
50+
ncb: ncb,
3751
}
3852
}
3953

@@ -125,17 +139,15 @@ func (db *DagBuilderHelper) Add(node *UnixfsNode) (*dag.Node, error) {
125139
return nil, err
126140
}
127141

128-
key, err := db.dserv.Add(dn)
142+
_, err = db.dserv.Add(dn)
129143
if err != nil {
130144
return nil, err
131145
}
132146

133-
if db.mp != nil {
134-
db.mp.PinWithMode(key, pin.Recursive)
135-
err := db.mp.Flush()
136-
if err != nil {
137-
return nil, err
138-
}
147+
// node callback
148+
err = db.ncb(dn, true)
149+
if err != nil {
150+
return nil, err
139151
}
140152

141153
return dn, nil

‎importer/helpers/helpers.go

+4-3
Original file line numberDiff line numberDiff line change
@@ -107,14 +107,15 @@ func (n *UnixfsNode) AddChild(child *UnixfsNode, db *DagBuilderHelper) error {
107107
return err
108108
}
109109

110-
childkey, err := db.dserv.Add(childnode)
110+
_, err = db.dserv.Add(childnode)
111111
if err != nil {
112112
return err
113113
}
114114

115115
// Pin the child node indirectly
116-
if db.mp != nil {
117-
db.mp.PinWithMode(childkey, pin.Indirect)
116+
err = db.ncb(childnode, false)
117+
if err != nil {
118+
return err
118119
}
119120

120121
return nil

‎importer/importer.go

+36-7
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,10 @@ import (
1313
trickle "github.com/ipfs/go-ipfs/importer/trickle"
1414
dag "github.com/ipfs/go-ipfs/merkledag"
1515
"github.com/ipfs/go-ipfs/pin"
16-
"github.com/ipfs/go-ipfs/util"
16+
u "github.com/ipfs/go-ipfs/util"
1717
)
1818

19-
var log = util.Logger("importer")
19+
var log = u.Logger("importer")
2020

2121
// Builds a DAG from the given file, writing created blocks to disk as they are
2222
// created
@@ -36,31 +36,60 @@ func BuildDagFromFile(fpath string, ds dag.DAGService, mp pin.ManualPinner) (*da
3636
}
3737
defer f.Close()
3838

39-
return BuildDagFromReader(f, ds, mp, chunk.DefaultSplitter)
39+
return BuildDagFromReader(f, ds, chunk.DefaultSplitter, BasicPinnerCB(mp))
4040
}
4141

42-
func BuildDagFromReader(r io.Reader, ds dag.DAGService, mp pin.ManualPinner, spl chunk.BlockSplitter) (*dag.Node, error) {
42+
func BuildDagFromReader(r io.Reader, ds dag.DAGService, spl chunk.BlockSplitter, ncb h.NodeCB) (*dag.Node, error) {
4343
// Start the splitter
4444
blkch := spl.Split(r)
4545

4646
dbp := h.DagBuilderParams{
4747
Dagserv: ds,
4848
Maxlinks: h.DefaultLinksPerBlock,
49-
Pinner: mp,
49+
NodeCB: ncb,
5050
}
5151

5252
return bal.BalancedLayout(dbp.New(blkch))
5353
}
5454

55-
func BuildTrickleDagFromReader(r io.Reader, ds dag.DAGService, mp pin.ManualPinner, spl chunk.BlockSplitter) (*dag.Node, error) {
55+
func BuildTrickleDagFromReader(r io.Reader, ds dag.DAGService, spl chunk.BlockSplitter, ncb h.NodeCB) (*dag.Node, error) {
5656
// Start the splitter
5757
blkch := spl.Split(r)
5858

5959
dbp := h.DagBuilderParams{
6060
Dagserv: ds,
6161
Maxlinks: h.DefaultLinksPerBlock,
62-
Pinner: mp,
62+
NodeCB: ncb,
6363
}
6464

6565
return trickle.TrickleLayout(dbp.New(blkch))
6666
}
67+
68+
func BasicPinnerCB(p pin.ManualPinner) h.NodeCB {
69+
return func(n *dag.Node, last bool) error {
70+
k, err := n.Key()
71+
if err != nil {
72+
return err
73+
}
74+
75+
if last {
76+
p.PinWithMode(k, pin.Recursive)
77+
return p.Flush()
78+
} else {
79+
p.PinWithMode(k, pin.Indirect)
80+
return nil
81+
}
82+
}
83+
}
84+
85+
func PinIndirectCB(p pin.ManualPinner) h.NodeCB {
86+
return func(n *dag.Node, last bool) error {
87+
k, err := n.Key()
88+
if err != nil {
89+
return err
90+
}
91+
92+
p.PinWithMode(k, pin.Indirect)
93+
return nil
94+
}
95+
}

‎importer/importer_test.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ import (
1717
func getBalancedDag(t testing.TB, size int64, blksize int) (*dag.Node, dag.DAGService) {
1818
ds := mdtest.Mock(t)
1919
r := io.LimitReader(u.NewTimeSeededRand(), size)
20-
nd, err := BuildDagFromReader(r, ds, nil, &chunk.SizeSplitter{blksize})
20+
nd, err := BuildDagFromReader(r, ds, &chunk.SizeSplitter{blksize}, nil)
2121
if err != nil {
2222
t.Fatal(err)
2323
}
@@ -27,7 +27,7 @@ func getBalancedDag(t testing.TB, size int64, blksize int) (*dag.Node, dag.DAGSe
2727
func getTrickleDag(t testing.TB, size int64, blksize int) (*dag.Node, dag.DAGService) {
2828
ds := mdtest.Mock(t)
2929
r := io.LimitReader(u.NewTimeSeededRand(), size)
30-
nd, err := BuildTrickleDagFromReader(r, ds, nil, &chunk.SizeSplitter{blksize})
30+
nd, err := BuildTrickleDagFromReader(r, ds, &chunk.SizeSplitter{blksize}, nil)
3131
if err != nil {
3232
t.Fatal(err)
3333
}
@@ -40,7 +40,7 @@ func TestBalancedDag(t *testing.T) {
4040
u.NewTimeSeededRand().Read(buf)
4141
r := bytes.NewReader(buf)
4242

43-
nd, err := BuildDagFromReader(r, ds, nil, chunk.DefaultSplitter)
43+
nd, err := BuildDagFromReader(r, ds, chunk.DefaultSplitter, nil)
4444
if err != nil {
4545
t.Fatal(err)
4646
}

‎merkledag/merkledag_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,7 @@ func runBatchFetchTest(t *testing.T, read io.Reader) {
156156

157157
spl := &chunk.SizeSplitter{512}
158158

159-
root, err := imp.BuildDagFromReader(read, dagservs[0], nil, spl)
159+
root, err := imp.BuildDagFromReader(read, dagservs[0], spl, nil)
160160
if err != nil {
161161
t.Fatal(err)
162162
}

‎unixfs/mod/dagmodifier.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
mh "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"
1212
context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
1313

14+
imp "github.com/ipfs/go-ipfs/importer"
1415
chunk "github.com/ipfs/go-ipfs/importer/chunk"
1516
help "github.com/ipfs/go-ipfs/importer/helpers"
1617
trickle "github.com/ipfs/go-ipfs/importer/trickle"
@@ -308,7 +309,7 @@ func (dm *DagModifier) appendData(node *mdag.Node, blks <-chan []byte) (*mdag.No
308309
dbp := &help.DagBuilderParams{
309310
Dagserv: dm.dagserv,
310311
Maxlinks: help.DefaultLinksPerBlock,
311-
Pinner: dm.mp,
312+
NodeCB: imp.BasicPinnerCB(dm.mp),
312313
}
313314

314315
return trickle.TrickleAppend(node, dbp.New(blks))

0 commit comments

Comments
 (0)