diff --git a/importer/helpers/dagbuilder.go b/importer/helpers/dagbuilder.go index f391495871a..ede6759fb66 100644 --- a/importer/helpers/dagbuilder.go +++ b/importer/helpers/dagbuilder.go @@ -78,6 +78,11 @@ func (db *DagBuilderHelper) Next() []byte { return d } +// GetDagServ returns the dagservice object this Helper is using +func (db *DagBuilderHelper) GetDagServ() dag.DAGService { + return db.dserv +} + // FillNodeLayer will add datanodes as children to the give node until // at most db.indirSize ndoes are added // @@ -86,7 +91,7 @@ func (db *DagBuilderHelper) FillNodeLayer(node *UnixfsNode) error { // while we have room AND we're not done for node.NumChildren() < db.maxlinks && !db.Done() { - child := NewUnixfsNode() + child := NewUnixfsBlock() if err := db.FillNodeWithData(child); err != nil { return err @@ -110,7 +115,7 @@ func (db *DagBuilderHelper) FillNodeWithData(node *UnixfsNode) error { return ErrSizeLimitExceeded } - node.setData(data) + node.SetData(data) return nil } diff --git a/importer/helpers/helpers.go b/importer/helpers/helpers.go index 7e4a55f2792..be0415db038 100644 --- a/importer/helpers/helpers.go +++ b/importer/helpers/helpers.go @@ -38,20 +38,51 @@ var ErrSizeLimitExceeded = fmt.Errorf("object size limit exceeded") // of unixfs DAG trees type UnixfsNode struct { node *dag.Node - ufmt *ft.MultiBlock + ufmt *ft.FSNode } +// NewUnixfsNode creates a new Unixfs node to represent a file func NewUnixfsNode() *UnixfsNode { return &UnixfsNode{ node: new(dag.Node), - ufmt: new(ft.MultiBlock), + ufmt: &ft.FSNode{Type: ft.TFile}, } } +// NewUnixfsBlock creates a new Unixfs node to represent a raw data block +func NewUnixfsBlock() *UnixfsNode { + return &UnixfsNode{ + node: new(dag.Node), + ufmt: &ft.FSNode{Type: ft.TRaw}, + } +} + +// NewUnixfsNodeFromDag reconstructs a Unixfs node from a given dag node +func NewUnixfsNodeFromDag(nd *dag.Node) (*UnixfsNode, error) { + mb, err := ft.FSNodeFromBytes(nd.Data) + if err != nil { + return nil, err + } + + return &UnixfsNode{ + node: nd, + ufmt: mb, + }, nil +} + func (n *UnixfsNode) NumChildren() int { return n.ufmt.NumChildren() } +func (n *UnixfsNode) GetChild(i int, ds dag.DAGService) (*UnixfsNode, error) { + nd, err := n.node.Links[i].GetNode(ds) + if err != nil { + return nil, err + } + + return NewUnixfsNodeFromDag(nd) +} + // addChild will add the given UnixfsNode as a child of the receiver. // the passed in DagBuilderHelper is used to store the child node an // pin it locally so it doesnt get lost @@ -83,7 +114,13 @@ func (n *UnixfsNode) AddChild(child *UnixfsNode, db *DagBuilderHelper) error { return nil } -func (n *UnixfsNode) setData(data []byte) { +// Removes the child node at the given index +func (n *UnixfsNode) RemoveChild(index int) { + n.ufmt.RemoveBlockSize(index) + n.node.Links = append(n.node.Links[:index], n.node.Links[index+1:]...) +} + +func (n *UnixfsNode) SetData(data []byte) { n.ufmt.Data = data } diff --git a/unixfs/format.go b/unixfs/format.go index 5e12d52acaf..61bb2ec9e84 100644 --- a/unixfs/format.go +++ b/unixfs/format.go @@ -9,6 +9,13 @@ import ( pb "github.com/jbenet/go-ipfs/unixfs/pb" ) +const ( + TRaw = pb.Data_Raw + TFile = pb.Data_File + TDirectory = pb.Data_Directory + TMetadata = pb.Data_Metadata +) + var ErrMalformedFileFormat = errors.New("malformed data in file format") var ErrInvalidDirLocation = errors.New("found directory node in unexpected place") var ErrUnrecognizedType = errors.New("unrecognized node type") @@ -98,33 +105,60 @@ func DataSize(data []byte) (uint64, error) { } } -type MultiBlock struct { - Data []byte +type FSNode struct { + Data []byte + + // total data size for each child blocksizes []uint64 - subtotal uint64 + + // running sum of blocksizes + subtotal uint64 + + // node type of this node + Type pb.Data_DataType +} + +func FSNodeFromBytes(b []byte) (*FSNode, error) { + pbn := new(pb.Data) + err := proto.Unmarshal(b, pbn) + if err != nil { + return nil, err + } + + n := new(FSNode) + n.Data = pbn.Data + n.blocksizes = pbn.Blocksizes + n.subtotal = pbn.GetFilesize() - uint64(len(n.Data)) + n.Type = pbn.GetType() + return n, nil +} + +// AddBlockSize adds the size of the next child block of this node +func (n *FSNode) AddBlockSize(s uint64) { + n.subtotal += s + n.blocksizes = append(n.blocksizes, s) } -func (mb *MultiBlock) AddBlockSize(s uint64) { - mb.subtotal += s - mb.blocksizes = append(mb.blocksizes, s) +func (n *FSNode) RemoveBlockSize(i int) { + n.subtotal -= n.blocksizes[i] + n.blocksizes = append(n.blocksizes[:i], n.blocksizes[i+1:]...) } -func (mb *MultiBlock) GetBytes() ([]byte, error) { +func (n *FSNode) GetBytes() ([]byte, error) { pbn := new(pb.Data) - t := pb.Data_File - pbn.Type = &t - pbn.Filesize = proto.Uint64(uint64(len(mb.Data)) + mb.subtotal) - pbn.Blocksizes = mb.blocksizes - pbn.Data = mb.Data + pbn.Type = &n.Type + pbn.Filesize = proto.Uint64(uint64(len(n.Data)) + n.subtotal) + pbn.Blocksizes = n.blocksizes + pbn.Data = n.Data return proto.Marshal(pbn) } -func (mb *MultiBlock) FileSize() uint64 { - return uint64(len(mb.Data)) + mb.subtotal +func (n *FSNode) FileSize() uint64 { + return uint64(len(n.Data)) + n.subtotal } -func (mb *MultiBlock) NumChildren() int { - return len(mb.blocksizes) +func (n *FSNode) NumChildren() int { + return len(n.blocksizes) } type Metadata struct { diff --git a/unixfs/format_test.go b/unixfs/format_test.go index 5065c7bc5ee..b15ed07891e 100644 --- a/unixfs/format_test.go +++ b/unixfs/format_test.go @@ -7,15 +7,16 @@ import ( pb "github.com/jbenet/go-ipfs/unixfs/pb" ) -func TestMultiBlock(t *testing.T) { - mbf := new(MultiBlock) +func TestFSNode(t *testing.T) { + fsn := new(FSNode) + fsn.Type = TFile for i := 0; i < 15; i++ { - mbf.AddBlockSize(100) + fsn.AddBlockSize(100) } - mbf.Data = make([]byte, 128) + fsn.Data = make([]byte, 128) - b, err := mbf.GetBytes() + b, err := fsn.GetBytes() if err != nil { t.Fatal(err) }