Replace coreunix.Add with shell/fsnode's AddFromReader

cmd/ipfs/init.go

@@ -1,21 +1,22 @@
 package main
 
 import (
-	"bytes"
 	"errors"
 	"fmt"
 	"io"
+	"strings"
 
 	context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
 	assets "github.com/ipfs/go-ipfs/assets"
 	cmds "github.com/ipfs/go-ipfs/commands"
 	core "github.com/ipfs/go-ipfs/core"
-	coreunix "github.com/ipfs/go-ipfs/core/coreunix"
+	importer "github.com/ipfs/go-ipfs/importer"
+	chunk "github.com/ipfs/go-ipfs/importer/chunk"
+	merkledag "github.com/ipfs/go-ipfs/merkledag"
 	namesys "github.com/ipfs/go-ipfs/namesys"
 	config "github.com/ipfs/go-ipfs/repo/config"
 	fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo"
-	uio "github.com/ipfs/go-ipfs/unixfs/io"
-	u "github.com/ipfs/go-ipfs/util"
+	unixfs "github.com/ipfs/go-ipfs/unixfs"
 )
 
 const nBitsForKeypairDefault = 2048
@@ -132,23 +133,24 @@ func addDefaultAssets(out io.Writer, repoRoot string) error {
 	}
 	defer nd.Close()
 
-	dirb := uio.NewDirectory(nd.DAG)
+	dir := &merkledag.Node{Data: unixfs.FolderPBData()}
 
 	// add every file in the assets pkg
 	for fname, file := range assets.Init_dir {
-		buf := bytes.NewBufferString(file)
-		s, err := coreunix.Add(nd, buf)
+		reader := strings.NewReader(file)
+		dagNode, err := importer.BuildDagFromReader(
+			reader,
+			nd.DAG,
+			nd.Pinning.GetManual(),
+			chunk.DefaultSplitter)
 		if err != nil {
 			return err
 		}
-
-		k := u.B58KeyDecode(s)
-		if err := dirb.AddChild(fname, k); err != nil {
+		if err := dir.AddNodeLink(fname, dagNode); err != nil {
 			return err
 		}
 	}
 
-	dir := dirb.GetNode()
 	dkey, err := nd.DAG.Add(dir)
 	if err != nil {
 		return err

cmd/ipfswatch/main.go

@@ -121,12 +121,7 @@ func run(ipfsPath, watchPath string) error {
 					}
 				}
 				proc.Go(func(p process.Process) {
-					file, err := os.Open(e.Name)
-					if err != nil {
-						log.Println(err)
-					}
-					defer file.Close()
-					k, err := coreunix.Add(node, file)
+					k, err := coreunix.AddR(node, e.Name)
 					if err != nil {
 						log.Println(err)
 					}

core/corehttp/corehttp.go

@@ -23,9 +23,9 @@ var log = eventlog.Logger("core/server")
 // initially passed in if not.
 type ServeOption func(*core.IpfsNode, *http.ServeMux) (*http.ServeMux, error)
 
-// makeHandler turns a list of ServeOptions into a http.Handler that implements
+// MakeHandler turns a list of ServeOptions into a http.Handler that implements
 // all of the given options, in order.
-func makeHandler(n *core.IpfsNode, options ...ServeOption) (http.Handler, error) {
+func MakeHandler(n *core.IpfsNode, options ...ServeOption) (http.Handler, error) {
 	topMux := http.NewServeMux()
 	mux := topMux
 	for _, option := range options {
@@ -49,7 +49,7 @@ func ListenAndServe(n *core.IpfsNode, listeningMultiAddr string, options ...Serv
 	if err != nil {
 		return err
 	}
-	handler, err := makeHandler(n, options...)
+	handler, err := MakeHandler(n, options...)
 	if err != nil {
 		return err
 	}

core/corehttp/gateway_test.go

@@ -1,4 +1,4 @@
-package corehttp
+package corehttp_test
 
 import (
 	"errors"
@@ -10,12 +10,13 @@ import (
 
 	context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
 	core "github.com/ipfs/go-ipfs/core"
-	coreunix "github.com/ipfs/go-ipfs/core/coreunix"
+	corehttp "github.com/ipfs/go-ipfs/core/corehttp"
 	namesys "github.com/ipfs/go-ipfs/namesys"
 	ci "github.com/ipfs/go-ipfs/p2p/crypto"
 	path "github.com/ipfs/go-ipfs/path"
 	repo "github.com/ipfs/go-ipfs/repo"
 	config "github.com/ipfs/go-ipfs/repo/config"
+	unixfs "github.com/ipfs/go-ipfs/shell/unixfs"
 	testutil "github.com/ipfs/go-ipfs/util/testutil"
 )
 
@@ -60,15 +61,20 @@ func TestGatewayGet(t *testing.T) {
 	t.Skip("not sure whats going on here")
 	ns := mockNamesys{}
 	n := newNodeWithMockNamesys(t, ns)
-	k, err := coreunix.Add(n, strings.NewReader("fnord"))
+	dagNode, err := unixfs.AddFromReader(n, strings.NewReader("fnord"))
 	if err != nil {
 		t.Fatal(err)
 	}
+	key, err := dagNode.Key()
+	if err != nil {
+		t.Fatal(err)
+	}
+	k := key.String()
 	ns["example.com"] = path.FromString("/ipfs/" + k)
 
-	h, err := makeHandler(n,
-		IPNSHostnameOption(),
-		GatewayOption(false),
+	h, err := corehttp.MakeHandler(n,
+		corehttp.IPNSHostnameOption(),
+		corehttp.GatewayOption(false),
 	)
 	if err != nil {
 		t.Fatal(err)

core/coreunix/add.go

@@ -22,8 +22,12 @@ import (
 
 var log = eventlog.Logger("coreunix")
 
-// Add builds a merkledag from the a reader, pinning all objects to the local
-// datastore. Returns a key representing the root node.
+// DEPRECATED: Add builds a merkledag from the a reader, pinning all
+// objects to the local datastore. Returns a key representing the root
+// node.
+//
+// This function is deprecated and will be removed in 0.4.0.  You
+// should use shell/unixfs's AddFromReader instead.
 func Add(n *core.IpfsNode, r io.Reader) (string, error) {
 	// TODO more attractive function signature importer.BuildDagFromReader
 	dagNode, err := importer.BuildDagFromReader(

docs/release-notes/unreleased.md

@@ -0,0 +1,26 @@
+# `core/coreunix`'s `Add` → `shell/unixfs`'s `AddFromReader`
+
+We've added an `AddFromReader` function to `shell/unixfs`.  The old
+`Add` from `core/coreunix` is deprecated and will be removed in
+version 0.4.0. To update your existing code, change usage like:
+
+    keyString, err := coreunix.Add(ipfsNode, reader)
+    if err != nil {
+        return err
+    }
+
+to
+
+    fileNode, err := unixfs.AddFromReader(ipfsNode, reader)
+    if err != nil {
+        return err
+    }
+    key, err := fileNode.Key()
+    if err != nil {
+        return err
+    }
+    keyString := key.String()
+
+That's a bit more verbose if all you want is the stringified key, but
+returning a `dag.Node` makes it easier to perform other operations
+like adding the file node to a directory node.

shell/shell.go

@@ -0,0 +1,8 @@
+/*
+Package shell implements a high-level interface for common IPFS activity.
+
+These wrappers around the low-level core interface make it easy to
+accomplish common tasks with default settings.  For steps where the
+defaults aren't appropriate, you can use the core package directly.
+*/
+package shell

shell/unixfs/add.go

@@ -0,0 +1,28 @@
+package unixfs
+
+import (
+	"io"
+
+	core "github.com/ipfs/go-ipfs/core"
+	importer "github.com/ipfs/go-ipfs/importer"
+	chunk "github.com/ipfs/go-ipfs/importer/chunk"
+	dag "github.com/ipfs/go-ipfs/merkledag"
+)
+
+// Add builds a merkledag from a reader, pinning all objects to the
+// local datastore.  Returns the root node.
+func AddFromReader(node *core.IpfsNode, reader io.Reader) (*dag.Node, error) {
+	fileNode, err := importer.BuildDagFromReader(
+		reader,
+		node.DAG,
+		node.Pinning.GetManual(),
+		chunk.DefaultSplitter,
+	)
+	if err != nil {
+		return nil, err
+	}
+	if err := node.Pinning.Flush(); err != nil {
+		return nil, err
+	}
+	return fileNode, nil
+}

shell/unixfs/unixfs.go

@@ -0,0 +1,10 @@
+/*
+Package unixfs is a high-level interface for filesystem nodes.
+
+Simple wrappers to:
+
+* create file and directory nodes from paths and readers,
+* extract file and directory nodes to your local filesytem, and
+* print file contents to writers.
+*/
+package unixfs

test/integration/addcat_test.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"io"
 	"math"
 	"os"
 	"testing"
@@ -14,11 +13,11 @@ import (
 	context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
 
 	"github.com/ipfs/go-ipfs/core"
-	coreunix "github.com/ipfs/go-ipfs/core/coreunix"
 	mocknet "github.com/ipfs/go-ipfs/p2p/net/mock"
 	"github.com/ipfs/go-ipfs/p2p/peer"
 	"github.com/ipfs/go-ipfs/thirdparty/unit"
 	testutil "github.com/ipfs/go-ipfs/util/testutil"
+	addcat "github.com/ipfs/go-ipfs/util/testutil/addcat"
 )
 
 const kSeed = 1
@@ -126,22 +125,10 @@ func DirectAddCat(data []byte, conf testutil.LatencyConfig) error {
 		return err
 	}
 
-	added, err := coreunix.Add(adder, bytes.NewReader(data))
+	err = addcat.AddCat(adder, catter, data)
 	if err != nil {
 		return err
 	}
-
-	readerCatted, err := coreunix.Cat(catter, added)
-	if err != nil {
-		return err
-	}
-
-	// verify
-	var bufout bytes.Buffer
-	io.Copy(&bufout, readerCatted)
-	if 0 != bytes.Compare(bufout.Bytes(), data) {
-		return errors.New("catted data does not match added data")
-	}
 	return nil
 }
 

test/integration/bench_cat_test.go

@@ -1,19 +1,17 @@
 package integrationtest
 
 import (
-	"bytes"
 	"errors"
-	"io"
 	"math"
 	"testing"
 
 	context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
 	"github.com/ipfs/go-ipfs/core"
-	coreunix "github.com/ipfs/go-ipfs/core/coreunix"
 	mocknet "github.com/ipfs/go-ipfs/p2p/net/mock"
 	"github.com/ipfs/go-ipfs/p2p/peer"
 	"github.com/ipfs/go-ipfs/thirdparty/unit"
 	testutil "github.com/ipfs/go-ipfs/util/testutil"
+	addcat "github.com/ipfs/go-ipfs/util/testutil/addcat"
 )
 
 func BenchmarkCat1MB(b *testing.B) { benchmarkVarCat(b, unit.MB*1) }
@@ -74,22 +72,9 @@ func benchCat(b *testing.B, data []byte, conf testutil.LatencyConfig) error {
 		return err
 	}
 
-	added, err := coreunix.Add(adder, bytes.NewReader(data))
+	err = addcat.AddCat(adder, catter, data)
 	if err != nil {
 		return err
 	}
-
-	b.StartTimer()
-	readerCatted, err := coreunix.Cat(catter, added)
-	if err != nil {
-		return err
-	}
-
-	// verify
-	var bufout bytes.Buffer
-	io.Copy(&bufout, readerCatted)
-	if 0 != bytes.Compare(bufout.Bytes(), data) {
-		return errors.New("catted data does not match added data")
-	}
 	return nil
 }

test/integration/grandcentral_test.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"io"
 	"math"
 	"testing"
 
@@ -14,14 +13,14 @@ import (
 
 	core "github.com/ipfs/go-ipfs/core"
 	"github.com/ipfs/go-ipfs/core/corerouting"
-	"github.com/ipfs/go-ipfs/core/coreunix"
 	mocknet "github.com/ipfs/go-ipfs/p2p/net/mock"
 	"github.com/ipfs/go-ipfs/p2p/peer"
 	"github.com/ipfs/go-ipfs/thirdparty/iter"
 	"github.com/ipfs/go-ipfs/thirdparty/unit"
 	"github.com/ipfs/go-ipfs/util"
 	ds2 "github.com/ipfs/go-ipfs/util/datastore2"
 	testutil "github.com/ipfs/go-ipfs/util/testutil"
+	addcat "github.com/ipfs/go-ipfs/util/testutil/addcat"
 )
 
 func TestSupernodeBootstrappedAddCat(t *testing.T) {
@@ -54,25 +53,10 @@ func RunSupernodeBootstrappedAddCat(data []byte, conf testutil.LatencyConfig) er
 	adder := clients[0]
 	catter := clients[1]
 
-	log.Info("adder is", adder.Identity)
-	log.Info("catter is", catter.Identity)
-
-	keyAdded, err := coreunix.Add(adder, bytes.NewReader(data))
+	err = addcat.AddCat(adder, catter, data)
 	if err != nil {
 		return err
 	}
-
-	readerCatted, err := coreunix.Cat(catter, keyAdded)
-	if err != nil {
-		return err
-	}
-
-	// verify
-	var bufout bytes.Buffer
-	io.Copy(&bufout, readerCatted)
-	if 0 != bytes.Compare(bufout.Bytes(), data) {
-		return errors.New("catted data does not match added data")
-	}
 	return nil
 }