feat: add daemon command

Author: kylehuntsman

.gitignore

@@ -0,0 +1,2 @@
+lassie
+*.car

cli/flags.go

@@ -0,0 +1,31 @@
+package cli
+
+import "github.com/urfave/cli/v2"
+
+// IsVerbose is a global var signaling if the CLI is running in
+// verbose mode or not (default: false).
+var IsVerbose bool
+
+// FlagVerbose enables verbose mode, which shows verbose information about
+// operations invoked in the CLI. It should be included as a flag on the
+// top-level command (e.g. lassie -v).
+var FlagVerbose = &cli.BoolFlag{
+	Name:        "verbose",
+	Aliases:     []string{"v"},
+	Usage:       "enable verbose mode for logging",
+	Destination: &IsVerbose,
+}
+
+// IsVeryVerbose is a global var signaling if the CLI is running in
+// very verbose mode or not (default: false).
+var IsVeryVerbose bool
+
+// FlagVerbose enables verbose mode, which shows verbose information about
+// operations invoked in the CLI. It should be included as a flag on the
+// top-level command (e.g. lassie -v).
+var FlagVeryVerbose = &cli.BoolFlag{
+	Name:        "very-verbose",
+	Aliases:     []string{"vv"},
+	Usage:       "enable very verbose mode for debugging",
+	Destination: &IsVeryVerbose,
+}

cmd/lassie/daemon.go

@@ -0,0 +1,73 @@
+package main
+
+import (
+	"fmt"
+
+	lcli "github.com/filecoin-project/lassie/cli"
+	httpserver "github.com/filecoin-project/lassie/server/http"
+	"github.com/urfave/cli/v2"
+)
+
+var daemonFlags = []cli.Flag{
+	&cli.StringFlag{
+		Name:        "address",
+		Aliases:     []string{"a"},
+		Usage:       "the address the http server listens on",
+		Value:       "127.0.0.1",
+		DefaultText: "127.0.0.1",
+		EnvVars:     []string{"LASSIE_ADDRESS"},
+	},
+	&cli.UintFlag{
+		Name:        "port",
+		Aliases:     []string{"p"},
+		Usage:       "the port the http server listens on",
+		Value:       0,
+		DefaultText: "random",
+		EnvVars:     []string{"LASSIE_PORT"},
+	},
+	lcli.FlagVerbose,
+	lcli.FlagVeryVerbose,
+}
+
+var daemonCmd = &cli.Command{
+	Name:   "daemon",
+	Usage:  "Starts a lassie daemon, accepting http requests",
+	Before: before,
+	Flags:  daemonFlags,
+	Action: daemonCommand,
+}
+
+func daemonCommand(cctx *cli.Context) error {
+	address := cctx.String("address")
+	port := cctx.Uint("port")
+	httpServer, err := httpserver.NewHttpServer(cctx.Context, address, port)
+	if err != nil {
+		log.Errorw("failed to create http server", "err", err)
+		return err
+	}
+
+	serverErrChan := make(chan error, 1)
+	go func() {
+		fmt.Printf("Lassie daemon listening on address %s\n", httpServer.Addr())
+		fmt.Println("Hit CTRL-C to stop the daemon")
+		serverErrChan <- httpServer.Start()
+	}()
+
+	select {
+	case <-cctx.Done(): // command was cancelled
+	case err = <-serverErrChan: // error from server
+		log.Errorw("failed to start http server", "err", err)
+	}
+
+	fmt.Println("Shutting down Lassie daemon")
+	if err = httpServer.Close(); err != nil {
+		log.Errorw("failed to close http server", "err", err)
+	}
+
+	fmt.Println("Lassie daemon stopped")
+	if err != nil {
+		return cli.Exit(err, 1)
+	}
+
+	return nil
+}

cmd/lassie/fetch.go

@@ -3,52 +3,46 @@ package main
 import (
 	"context"
 	"fmt"
-	"log"
 	"strings"
 	"time"
 
 	"github.com/dustin/go-humanize"
-	"github.com/filecoin-project/lassie/cmd/lassie/internal"
-	"github.com/filecoin-project/lassie/pkg/client"
+	lcli "github.com/filecoin-project/lassie/cli"
+	"github.com/filecoin-project/lassie/internal"
 	"github.com/filecoin-project/lassie/pkg/events"
-	"github.com/filecoin-project/lassie/pkg/indexerlookup"
 	"github.com/filecoin-project/lassie/pkg/retriever"
 	"github.com/filecoin-project/lassie/pkg/types"
-	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
-	"github.com/ipfs/go-datastore"
-	dss "github.com/ipfs/go-datastore/sync"
 	"github.com/ipfs/go-graphsync/storeutil"
-	blockstore "github.com/ipfs/go-ipfs-blockstore"
 	carblockstore "github.com/ipld/go-car/v2/blockstore"
 	"github.com/libp2p/go-libp2p/core/peer"
-	"github.com/multiformats/go-multiaddr"
 	"github.com/urfave/cli/v2"
 )
 
 var fetchProviderAddrInfo *peer.AddrInfo
 
 var fetchCmd = &cli.Command{
 	Name:   "fetch",
-	Usage:  "fetch content from Filecoin",
+	Usage:  "Fetches content from Filecoin",
+	Before: before,
 	Action: Fetch,
 	Flags: []cli.Flag{
 		&cli.StringFlag{
 			Name:      "output",
 			Aliases:   []string{"o"},
-			Usage:     "The CAR file to write to, may be an existing or a new CAR",
+			Usage:     "the CAR file to write to, may be an existing or a new CAR",
 			TakesFile: true,
 		},
 		&cli.DurationFlag{
 			Name:    "timeout",
 			Aliases: []string{"t"},
-			Usage:   "Consider it an error after not receiving a response from a storage provider for this long",
+			Usage:   "consider it an error after not receiving a response from a storage provider for this long",
 			Value:   20 * time.Second,
 		},
 		&cli.BoolFlag{
 			Name:    "progress",
 			Aliases: []string{"p"},
-			Usage:   "Print progress output",
+			Usage:   "print progress output",
 		},
 		&cli.StringFlag{
 			Name:        "provider",
@@ -60,6 +54,8 @@ var fetchCmd = &cli.Command{
 				return err
 			},
 		},
+		lcli.FlagVerbose,
+		lcli.FlagVeryVerbose,
 	},
 }
 
@@ -95,15 +91,15 @@ func Fetch(c *cli.Context) error {
 		}
 	}
 	timeout := c.Duration("timeout")
-	bstore := &putCbBlockstore{parentOpener: parentOpener, cb: putCb}
+	bstore := internal.NewPutCbBlockstore(parentOpener, putCb)
 
 	linkSystem := storeutil.LinkSystemForBlockstore(bstore)
 
 	var ret *retriever.Retriever
 	if fetchProviderAddrInfo == nil {
-		ret, err = setupRetriever(c, timeout)
+		ret, err = internal.SetupRetriever(c.Context, timeout)
 	} else {
-		ret, err = setupRetrieverWithFinder(c, timeout, explicitCandidateFinder{provider: *fetchProviderAddrInfo})
+		ret, err = internal.SetupRetrieverWithFinder(c.Context, timeout, explicitCandidateFinder{provider: *fetchProviderAddrInfo})
 	}
 	if err != nil {
 		return err
@@ -142,132 +138,6 @@ func Fetch(c *cli.Context) error {
 	return bstore.Finalize()
 }
 
-func setupRetriever(c *cli.Context, timeout time.Duration) (*retriever.Retriever, error) {
-	return setupRetrieverWithFinder(c, timeout, indexerlookup.NewCandidateFinder("https://cid.contact"))
-}
-
-func setupRetrieverWithFinder(c *cli.Context, timeout time.Duration, finder retriever.CandidateFinder) (*retriever.Retriever, error) {
-	datastore := dss.MutexWrap(datastore.NewMapDatastore())
-
-	host, err := internal.InitHost(c.Context, multiaddr.StringCast("/ip4/0.0.0.0/tcp/6746"))
-	if err != nil {
-		return nil, err
-	}
-
-	retrievalClient, err := client.NewClient(datastore, host, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	indexer := indexerlookup.NewCandidateFinder("https://cid.contact")
-
-	retrieverCfg := retriever.RetrieverConfig{
-		DefaultMinerConfig: retriever.MinerConfig{
-			RetrievalTimeout: timeout,
-		},
-	}
-
-	ret, err := retriever.NewRetriever(c.Context, retrieverCfg, retrievalClient, indexer)
-	if err != nil {
-		return nil, err
-	}
-	ret.Start()
-	return ret, nil
-}
-
-// putCbBlockstore simply calls a callback on each put(), with the number of blocks put
-var _ blockstore.Blockstore = (*putCbBlockstore)(nil)
-
-type putCbBlockstore struct {
-	// parentOpener lazily opens the parent blockstore upon first call to this blockstore.
-	// This avoids blockstore instantiation until there is some interaction from the retriever.
-	// In the case of CARv2 blockstores, this will avoid creation of empty .car files should
-	// the retriever fail to find any candidates.
-	parentOpener func() (*carblockstore.ReadWrite, error)
-	// parent is lazily instantiated and should not be directly used; use parentBlockstore instead.
-	parent *carblockstore.ReadWrite
-	cb     func(putCount int, putBytes int)
-}
-
-func (pcb *putCbBlockstore) DeleteBlock(ctx context.Context, cid cid.Cid) error {
-	pbs, err := pcb.parentBlockstore()
-	if err != nil {
-		return err
-	}
-	return pbs.DeleteBlock(ctx, cid)
-}
-func (pcb *putCbBlockstore) Has(ctx context.Context, cid cid.Cid) (bool, error) {
-	pbs, err := pcb.parentBlockstore()
-	if err != nil {
-		return false, err
-	}
-	return pbs.Has(ctx, cid)
-}
-func (pcb *putCbBlockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) {
-	pbs, err := pcb.parentBlockstore()
-	if err != nil {
-		return nil, err
-	}
-	return pbs.Get(ctx, cid)
-}
-func (pcb *putCbBlockstore) GetSize(ctx context.Context, cid cid.Cid) (int, error) {
-	pbs, err := pcb.parentBlockstore()
-	if err != nil {
-		return 0, err
-	}
-	return pbs.GetSize(ctx, cid)
-}
-func (pcb *putCbBlockstore) Put(ctx context.Context, block blocks.Block) error {
-	pbs, err := pcb.parentBlockstore()
-	if err != nil {
-		return err
-	}
-	pcb.cb(1, len(block.RawData()))
-	return pbs.Put(ctx, block)
-}
-func (pcb *putCbBlockstore) PutMany(ctx context.Context, blocks []blocks.Block) error {
-	pbs, err := pcb.parentBlockstore()
-	if err != nil {
-		return err
-	}
-	var byts int
-	for _, b := range blocks {
-		byts += len(b.RawData())
-	}
-	pcb.cb(len(blocks), byts)
-	return pbs.PutMany(ctx, blocks)
-}
-func (pcb *putCbBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
-	pbs, err := pcb.parentBlockstore()
-	if err != nil {
-		return nil, err
-	}
-	return pbs.AllKeysChan(ctx)
-}
-func (pcb *putCbBlockstore) HashOnRead(enabled bool) {
-	if pbs, err := pcb.parentBlockstore(); err != nil {
-		log.Printf("Failed to instantiate blockstore while setting HashOnRead: %v\n", err)
-	} else {
-		pbs.HashOnRead(enabled)
-	}
-}
-func (pcb *putCbBlockstore) Finalize() error {
-	if pbs, err := pcb.parentBlockstore(); err != nil {
-		return err
-	} else {
-		return pbs.Finalize()
-	}
-}
-func (pcb *putCbBlockstore) parentBlockstore() (*carblockstore.ReadWrite, error) {
-	if pcb.parent == nil {
-		var err error
-		if pcb.parent, err = pcb.parentOpener(); err != nil {
-			return nil, err
-		}
-	}
-	return pcb.parent, nil
-}
-
 type progressPrinter struct {
 	candidatesFound int
 }