Skip to content

Commit 22dc275

Browse files
whyrusleepingwhyrusleeping
committed
copying in sparse files no longer breaks things
1 parent 8ed952c commit 22dc275

File tree

8 files changed

+156
-64
lines changed

8 files changed

+156
-64
lines changed

‎cmd/ipfs/init.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
cmds "github.com/jbenet/go-ipfs/commands"
1111
core "github.com/jbenet/go-ipfs/core"
1212
coreunix "github.com/jbenet/go-ipfs/core/coreunix"
13+
namesys "github.com/jbenet/go-ipfs/namesys"
1314
config "github.com/jbenet/go-ipfs/repo/config"
1415
fsrepo "github.com/jbenet/go-ipfs/repo/fsrepo"
1516
uio "github.com/jbenet/go-ipfs/unixfs/io"
@@ -171,5 +172,5 @@ func initializeIpnsKeyspace(repoRoot string) error {
171172
return err
172173
}
173174

174-
return nd.IpnsFs.InitializeKeyspace(ctx, nd.PrivateKey)
175+
return namesys.InitializeKeyspace(ctx, nd.DAG, nd.Namesys, nd.Pinning, nd.PrivateKey)
175176
}

‎core/core.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -142,8 +142,8 @@ func NewIPFSNode(parent context.Context, option ConfigOption) (*IpfsNode, error)
142142
}
143143
node.Resolver = &path.Resolver{DAG: node.DAG}
144144

145+
// Setup the mutable ipns filesystem structure
145146
if node.OnlineMode() {
146-
// Setup the mutable ipns filesystem structure
147147
fs, err := ipnsfs.NewFilesystem(ctx, node.DAG, node.Namesys, node.Pinning, node.PrivateKey)
148148
if err != nil {
149149
return nil, debugerror.Wrap(err)

‎ipnsfs/dir.go

+16-20
Original file line numberDiff line numberDiff line change
@@ -69,21 +69,11 @@ func (d *Directory) Open(tpath []string, mode int) (File, error) {
6969

7070
// consider combining into a single method...
7171
type childCloser interface {
72-
closeChild(string) error
72+
closeChild(string, *dag.Node) error
7373
}
7474

75-
func (d *Directory) closeChild(name string) error {
76-
child, err := d.Child(name)
77-
if err != nil {
78-
return err
79-
}
80-
81-
nd, err := child.GetNode()
82-
if err != nil {
83-
return err
84-
}
85-
86-
_, err = d.fs.dserv.Add(nd)
75+
func (d *Directory) closeChild(name string, nd *dag.Node) error {
76+
_, err := d.fs.dserv.Add(nd)
8777
if err != nil {
8878
return err
8979
}
@@ -102,7 +92,7 @@ func (d *Directory) closeChild(name string) error {
10292
}
10393
d.lock.Unlock()
10494

105-
return d.parent.closeChild(d.name)
95+
return d.parent.closeChild(d.name, d.node)
10696
}
10797

10898
func (d *Directory) Type() NodeType {
@@ -231,7 +221,7 @@ func (d *Directory) Mkdir(name string) (*Directory, error) {
231221
}
232222
d.lock.Unlock()
233223

234-
err = d.parent.closeChild(d.name)
224+
err = d.parent.closeChild(d.name, d.node)
235225
if err != nil {
236226
return nil, err
237227
}
@@ -254,7 +244,7 @@ func (d *Directory) Unlink(name string) error {
254244
}
255245
d.lock.Unlock()
256246

257-
return d.parent.closeChild(d.name)
247+
return d.parent.closeChild(d.name, d.node)
258248
}
259249

260250
func (d *Directory) RenameEntry(oldname, newname string) error {
@@ -273,7 +263,7 @@ func (d *Directory) RenameEntry(oldname, newname string) error {
273263

274264
delete(d.childDirs, oldname)
275265
d.childDirs[newname] = dir
276-
return d.parent.closeChild(d.name)
266+
return d.parent.closeChild(d.name, d.node)
277267
}
278268

279269
fi, err := d.childFile(oldname)
@@ -284,14 +274,20 @@ func (d *Directory) RenameEntry(oldname, newname string) error {
284274
if err != nil {
285275
return err
286276
}
287-
err = d.node.AddNodeLinkClean(newname, fi.node)
277+
278+
nd, err := fi.GetNode()
279+
if err != nil {
280+
return err
281+
}
282+
283+
err = d.node.AddNodeLinkClean(newname, nd)
288284
if err != nil {
289285
return err
290286
}
291287

292288
delete(d.childDirs, oldname)
293289
d.files[newname] = fi
294-
return d.parent.closeChild(d.name)
290+
return d.parent.closeChild(d.name, d.node)
295291
}
296292
return ErrNoSuch
297293
}
@@ -324,7 +320,7 @@ func (d *Directory) AddChild(name string, nd *dag.Node) error {
324320
default:
325321
panic("invalid unixfs node")
326322
}
327-
return d.parent.closeChild(d.name)
323+
return d.parent.closeChild(d.name, d.node)
328324
}
329325

330326
func (d *Directory) GetNode() (*dag.Node, error) {

‎ipnsfs/file.go

+27-9
Original file line numberDiff line numberDiff line change
@@ -25,17 +25,13 @@ type File interface {
2525

2626
type file struct {
2727
parent childCloser
28-
node *dag.Node
2928
fs *Filesystem
3029

31-
name string
32-
33-
refLk sync.Mutex
34-
ref int
35-
wref bool
36-
30+
name string
3731
hasChanges bool
3832

33+
// TODO: determine whether or not locking here is actually required...
34+
lk sync.Mutex
3935
mod *mod.DagModifier
4036
}
4137

@@ -48,29 +44,39 @@ func NewFile(name string, node *dag.Node, parent childCloser, fs *Filesystem) (*
4844
return &file{
4945
fs: fs,
5046
parent: parent,
51-
node: node,
5247
name: name,
5348
mod: dmod,
5449
}, nil
5550
}
5651

5752
func (fi *file) Write(b []byte) (int, error) {
53+
fi.lk.Lock()
54+
defer fi.lk.Unlock()
5855
fi.hasChanges = true
5956
return fi.mod.Write(b)
6057
}
6158

6259
func (fi *file) Read(b []byte) (int, error) {
60+
fi.lk.Lock()
61+
defer fi.lk.Unlock()
6362
return fi.mod.Read(b)
6463
}
6564

6665
func (fi *file) Close() error {
66+
fi.lk.Lock()
67+
defer fi.lk.Unlock()
6768
if fi.hasChanges {
6869
err := fi.mod.Flush()
6970
if err != nil {
7071
return err
7172
}
7273

73-
err = fi.parent.closeChild(fi.name)
74+
nd, err := fi.mod.GetNode()
75+
if err != nil {
76+
return err
77+
}
78+
79+
err = fi.parent.closeChild(fi.name, nd)
7480
if err != nil {
7581
return err
7682
}
@@ -82,6 +88,8 @@ func (fi *file) Close() error {
8288
}
8389

8490
func (fi *file) Flush() error {
91+
fi.lk.Lock()
92+
defer fi.lk.Unlock()
8593
return fi.mod.Flush()
8694
}
8795

@@ -93,23 +101,33 @@ func (fi *file) withMode(mode int) File {
93101
}
94102

95103
func (fi *file) Seek(offset int64, whence int) (int64, error) {
104+
fi.lk.Lock()
105+
defer fi.lk.Unlock()
96106
return fi.mod.Seek(offset, whence)
97107
}
98108

99109
func (fi *file) WriteAt(b []byte, at int64) (int, error) {
110+
fi.lk.Lock()
111+
defer fi.lk.Unlock()
100112
fi.hasChanges = true
101113
return fi.mod.WriteAt(b, at)
102114
}
103115

104116
func (fi *file) Size() (int64, error) {
117+
fi.lk.Lock()
118+
defer fi.lk.Unlock()
105119
return fi.mod.Size()
106120
}
107121

108122
func (fi *file) GetNode() (*dag.Node, error) {
123+
fi.lk.Lock()
124+
defer fi.lk.Unlock()
109125
return fi.mod.GetNode()
110126
}
111127

112128
func (fi *file) Truncate(size int64) error {
129+
fi.lk.Lock()
130+
defer fi.lk.Unlock()
113131
fi.hasChanges = true
114132
return fi.mod.Truncate(size)
115133
}

‎ipnsfs/system.go

+2-30
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ func (fs *Filesystem) NewKeyRoot(parent context.Context, k ci.PrivKey) (*KeyRoot
131131

132132
pointsTo, err := fs.nsys.Resolve(ctx, name)
133133
if err != nil {
134-
err = fs.InitializeKeyspace(ctx, k)
134+
err = namesys.InitializeKeyspace(ctx, fs.dserv, fs.nsys, fs.pins, k)
135135
if err != nil {
136136
return nil, err
137137
}
@@ -173,34 +173,6 @@ func (fs *Filesystem) NewKeyRoot(parent context.Context, k ci.PrivKey) (*KeyRoot
173173
return root, nil
174174
}
175175

176-
// InitializeKeyspace sets the ipns record for the given key to
177-
// point to an empty directory.
178-
// TODO: this doesnt feel like it belongs here
179-
func (fs *Filesystem) InitializeKeyspace(ctx context.Context, key ci.PrivKey) error {
180-
emptyDir := &dag.Node{Data: ft.FolderPBData()}
181-
nodek, err := fs.dserv.Add(emptyDir)
182-
if err != nil {
183-
return err
184-
}
185-
186-
err = fs.pins.Pin(emptyDir, false)
187-
if err != nil {
188-
return err
189-
}
190-
191-
err = fs.pins.Flush()
192-
if err != nil {
193-
return err
194-
}
195-
196-
err = fs.nsys.Publish(ctx, key, nodek)
197-
if err != nil {
198-
return err
199-
}
200-
201-
return nil
202-
}
203-
204176
func (kr *KeyRoot) GetValue() FSNode {
205177
return kr.val
206178
}
@@ -232,7 +204,7 @@ func (kr *KeyRoot) Open(tpath []string, mode int) (File, error) {
232204

233205
// closeChild implements the childCloser interface, and signals to the publisher that
234206
// there are changes ready to be published
235-
func (kr *KeyRoot) closeChild(name string) error {
207+
func (kr *KeyRoot) closeChild(name string, nd *dag.Node) error {
236208
kr.repub.Touch()
237209
return nil
238210
}

‎namesys/publisher.go

+31
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,13 @@ import (
1010
mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"
1111
context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
1212

13+
dag "github.com/jbenet/go-ipfs/merkledag"
1314
pb "github.com/jbenet/go-ipfs/namesys/internal/pb"
1415
ci "github.com/jbenet/go-ipfs/p2p/crypto"
16+
pin "github.com/jbenet/go-ipfs/pin"
1517
routing "github.com/jbenet/go-ipfs/routing"
1618
record "github.com/jbenet/go-ipfs/routing/record"
19+
ft "github.com/jbenet/go-ipfs/unixfs"
1720
u "github.com/jbenet/go-ipfs/util"
1821
)
1922

@@ -134,3 +137,31 @@ func ValidateIpnsRecord(k u.Key, val []byte) error {
134137
}
135138
return nil
136139
}
140+
141+
// InitializeKeyspace sets the ipns record for the given key to
142+
// point to an empty directory.
143+
// TODO: this doesnt feel like it belongs here
144+
func InitializeKeyspace(ctx context.Context, ds dag.DAGService, pub Publisher, pins pin.Pinner, key ci.PrivKey) error {
145+
emptyDir := &dag.Node{Data: ft.FolderPBData()}
146+
nodek, err := ds.Add(emptyDir)
147+
if err != nil {
148+
return err
149+
}
150+
151+
err = pins.Pin(emptyDir, false)
152+
if err != nil {
153+
return err
154+
}
155+
156+
err = pins.Flush()
157+
if err != nil {
158+
return err
159+
}
160+
161+
err = pub.Publish(ctx, key, nodek)
162+
if err != nil {
163+
return err
164+
}
165+
166+
return nil
167+
}

‎unixfs/mod/dagmodifier.go

+38-3
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,18 @@ func NewDagModifier(ctx context.Context, from *mdag.Node, serv mdag.DAGService,
5858
func (dm *DagModifier) WriteAt(b []byte, offset int64) (int, error) {
5959
// TODO: this is currently VERY inneficient
6060
if uint64(offset) != dm.curWrOff {
61-
err := dm.Flush()
61+
size, err := dm.Size()
62+
if err != nil {
63+
return 0, err
64+
}
65+
if offset > size {
66+
err := dm.expandSparse(offset - size)
67+
if err != nil {
68+
return 0, err
69+
}
70+
}
71+
72+
err = dm.Flush()
6273
if err != nil {
6374
return 0, err
6475
}
@@ -68,6 +79,31 @@ func (dm *DagModifier) WriteAt(b []byte, offset int64) (int, error) {
6879
return dm.Write(b)
6980
}
7081

82+
type zeroReader struct{}
83+
84+
func (zr zeroReader) Read(b []byte) (int, error) {
85+
for i, _ := range b {
86+
b[i] = 0
87+
}
88+
return len(b), nil
89+
}
90+
91+
func (dm *DagModifier) expandSparse(size int64) error {
92+
spl := chunk.SizeSplitter{4096}
93+
r := io.LimitReader(zeroReader{}, size)
94+
blks := spl.Split(r)
95+
nnode, err := dm.appendData(dm.curNode, blks)
96+
if err != nil {
97+
return err
98+
}
99+
_, err = dm.dagserv.Add(nnode)
100+
if err != nil {
101+
return err
102+
}
103+
dm.curNode = nnode
104+
return nil
105+
}
106+
71107
func (dm *DagModifier) Write(b []byte) (int, error) {
72108
if dm.read != nil {
73109
dm.read = nil
@@ -90,6 +126,7 @@ func (dm *DagModifier) Write(b []byte) (int, error) {
90126
}
91127

92128
func (dm *DagModifier) Size() (int64, error) {
129+
// TODO: compute size without flushing, should be easy
93130
err := dm.Flush()
94131
if err != nil {
95132
return 0, err
@@ -149,8 +186,6 @@ func (dm *DagModifier) modifyDag(node *mdag.Node, offset uint64, data io.Reader)
149186
}
150187

151188
if len(node.Links) == 0 && (f.GetType() == ftpb.Data_Raw || f.GetType() == ftpb.Data_File) {
152-
// A data block! lets write to it
153-
154189
n, err := data.Read(f.Data[offset:])
155190
if err != nil && err != io.EOF {
156191
return "", 0, false, err

0 commit comments

Comments
 (0)