fix(cmds): CIDv1 and correct multicodecs in 'block put' and 'cid codecs' (#8568)

BREAKING CHANGES: 
- see #8568 (comment)

Co-authored-by: Marcin Rataj <[email protected]>

Author: schomatis

CHANGELOG.md

@@ -1,5 +1,22 @@
 # go-ipfs changelog
 
+## v0.13 (DRAFT)
+
+### BREAKING CHANGES
+
+- `ipfs block put` command produces CIDv1 with `raw` codec by default now
+    - `ipfs block put --cid-codec` makes `block put` return CID with alternative codec
+       - this impacts only the returned CID, it does  not trigger any validation or data transformation
+       - codec names are validated against tables from https://github.com/multiformats/go-multicodec
+    - `ipfs block put --format` is deprecated. It used incorrect codec names and should be avoided for new deployments. Use it only if you need the old, invalid behavior, namely:
+      - `ipfs block put --format=v0` will produce CIDv0 (implicit dag-pb)
+      - `ipfs block put --format=cbor` will produce CIDv1 with dag-cbor (!)
+      - `ipfs block put --format=protobuf` will produce CIDv1 with dag-pb (!)
+- `ipfs cid codecs` command
+  - it now lists codecs from https://github.com/multiformats/go-multicodec
+  - `ipfs cid codecs --supported` can be passed to only show codecs supported in various go-ipfs commands
+
+
 ## v0.12.2 and v0.11.1 2022-04-08
 
 This patch release fixes a security issue wherein traversing some malformed DAGs can cause the node to panic.

core/commands/block.go

@@ -31,8 +31,8 @@ var BlockCmd = &cmds.Command{
 		Tagline: "Interact with raw IPFS blocks.",
 		ShortDescription: `
 'ipfs block' is a plumbing command used to manipulate raw IPFS blocks.
-Reads from stdin or writes to stdout, and <key> is a base58 encoded
-multihash.
+Reads from stdin or writes to stdout. A block is identified by a Multihash
+passed with a valid CID.
 `,
 	},
 
@@ -51,14 +51,14 @@ var blockStatCmd = &cmds.Command{
 'ipfs block stat' is a plumbing command for retrieving information
 on raw IPFS blocks. It outputs the following to stdout:
 
-	Key  - the base58 encoded multihash
+	Key  - the CID of the block
 	Size - the size of the block in bytes
 
 `,
 	},
 
 	Arguments: []cmds.Argument{
-		cmds.StringArg("key", true, false, "The base58 multihash of an existing block to stat.").EnableStdin(),
+		cmds.StringArg("cid", true, false, "The CID of an existing block to stat.").EnableStdin(),
 	},
 	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
 		api, err := cmdenv.GetApi(env, req)
@@ -90,12 +90,12 @@ var blockGetCmd = &cmds.Command{
 		Tagline: "Get a raw IPFS block.",
 		ShortDescription: `
 'ipfs block get' is a plumbing command for retrieving raw IPFS blocks.
-It outputs to stdout, and <key> is a base58 encoded multihash.
+It takes a <cid>, and outputs the block to stdout.
 `,
 	},
 
 	Arguments: []cmds.Argument{
-		cmds.StringArg("key", true, false, "The base58 multihash of an existing block to get.").EnableStdin(),
+		cmds.StringArg("cid", true, false, "The CID of an existing block to get.").EnableStdin(),
 	},
 	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
 		api, err := cmdenv.GetApi(env, req)
@@ -113,32 +113,41 @@ It outputs to stdout, and <key> is a base58 encoded multihash.
 }
 
 const (
-	blockFormatOptionName = "format"
-	mhtypeOptionName      = "mhtype"
-	mhlenOptionName       = "mhlen"
+	blockFormatOptionName   = "format"
+	blockCidCodecOptionName = "cid-codec"
+	mhtypeOptionName        = "mhtype"
+	mhlenOptionName         = "mhlen"
 )
 
 var blockPutCmd = &cmds.Command{
 	Helptext: cmds.HelpText{
 		Tagline: "Store input as an IPFS block.",
 		ShortDescription: `
 'ipfs block put' is a plumbing command for storing raw IPFS blocks.
-It reads from stdin, and outputs the block's CID to stdout.
+It reads data from stdin, and outputs the block's CID to stdout.
 
-Unless specified, this command returns dag-pb CIDv0 CIDs. Setting 'mhtype' to anything
-other than 'sha2-256' or format to anything other than 'v0' will result in CIDv1.
+Unless cid-codec is specified, this command returns raw (0x55) CIDv1 CIDs.
+
+Passing alternative --cid-codec does not modify imported data, nor run any
+validation. It is provided solely for convenience for users who create blocks
+in userland.
+
+NOTE:
+Do not use --format for any new code. It got superseded by --cid-codec and left
+only for backward compatibility when a legacy CIDv0 is required (--format=v0).
 `,
 	},
 
 	Arguments: []cmds.Argument{
 		cmds.FileArg("data", true, true, "The data to be stored as an IPFS block.").EnableStdin(),
 	},
 	Options: []cmds.Option{
-		cmds.StringOption(blockFormatOptionName, "f", "cid format for blocks to be created with."),
-		cmds.StringOption(mhtypeOptionName, "multihash hash function").WithDefault("sha2-256"),
-		cmds.IntOption(mhlenOptionName, "multihash hash length").WithDefault(-1),
-		cmds.BoolOption(pinOptionName, "pin added blocks recursively").WithDefault(false),
+		cmds.StringOption(blockCidCodecOptionName, "Multicodec to use in returned CID").WithDefault("raw"),
+		cmds.StringOption(mhtypeOptionName, "Multihash hash function").WithDefault("sha2-256"),
+		cmds.IntOption(mhlenOptionName, "Multihash hash length").WithDefault(-1),
+		cmds.BoolOption(pinOptionName, "Pin added blocks recursively").WithDefault(false),
 		cmdutils.AllowBigBlockOption,
+		cmds.StringOption(blockFormatOptionName, "f", "Use legacy format for returned CID (DEPRECATED)"),
 	},
 	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
 		api, err := cmdenv.GetApi(env, req)
@@ -157,13 +166,15 @@ other than 'sha2-256' or format to anything other than 'v0' will result in CIDv1
 			return errors.New("missing option \"mhlen\"")
 		}
 
-		format, formatSet := req.Options[blockFormatOptionName].(string)
-		if !formatSet {
-			if mhtval != mh.SHA2_256 || (mhlen != -1 && mhlen != 32) {
-				format = "protobuf"
-			} else {
-				format = "v0"
+		cidCodec, _ := req.Options[blockCidCodecOptionName].(string)
+		format, _ := req.Options[blockFormatOptionName].(string) // deprecated
+
+		// use of legacy 'format' needs to supress 'cid-codec'
+		if format != "" {
+			if cidCodec != "" && cidCodec != "raw" {
+				return fmt.Errorf("unable to use %q (deprecated) and a custom %q at the same time", blockFormatOptionName, blockCidCodecOptionName)
 			}
+			cidCodec = "" // makes it no-op
 		}
 
 		pin, _ := req.Options[pinOptionName].(bool)
@@ -177,6 +188,7 @@ other than 'sha2-256' or format to anything other than 'v0' will result in CIDv1
 
 			p, err := api.Block().Put(req.Context, file,
 				options.Block.Hash(mhtval, mhlen),
+				options.Block.CidCodec(cidCodec),
 				options.Block.Format(format),
 				options.Block.Pin(pin))
 			if err != nil {
@@ -219,14 +231,14 @@ type removedBlock struct {
 
 var blockRmCmd = &cmds.Command{
 	Helptext: cmds.HelpText{
-		Tagline: "Remove IPFS block(s).",
+		Tagline: "Remove IPFS block(s) from the local datastore.",
 		ShortDescription: `
 'ipfs block rm' is a plumbing command for removing raw ipfs blocks.
-It takes a list of base58 encoded multihashes to remove.
+It takes a list of CIDs to remove from the local datastore..
 `,
 	},
 	Arguments: []cmds.Argument{
-		cmds.StringArg("hash", true, true, "Bash58 encoded multihash of block(s) to remove."),
+		cmds.StringArg("cid", true, true, "CIDs of block(s) to remove."),
 	},
 	Options: []cmds.Option{
 		cmds.BoolOption(forceOptionName, "f", "Ignore nonexistent blocks."),

core/commands/cid.go

@@ -11,7 +11,9 @@ import (
 	cidutil "github.com/ipfs/go-cidutil"
 	cmds "github.com/ipfs/go-ipfs-cmds"
 	verifcid "github.com/ipfs/go-verifcid"
+	"github.com/ipld/go-ipld-prime/multicodec"
 	mbase "github.com/multiformats/go-multibase"
+	mc "github.com/multiformats/go-multicodec"
 	mhash "github.com/multiformats/go-multihash"
 )
 
@@ -46,7 +48,7 @@ The optional format string is a printf style format string:
 ` + cidutil.FormatRef,
 	},
 	Arguments: []cmds.Argument{
-		cmds.StringArg("cid", true, true, "Cids to format.").EnableStdin(),
+		cmds.StringArg("cid", true, true, "CIDs to format.").EnableStdin(),
 	},
 	Options: []cmds.Option{
 		cmds.StringOption(cidFormatOptionName, "Printf style format string.").WithDefault("%s"),
@@ -63,14 +65,14 @@ The optional format string is a printf style format string:
 		opts := cidFormatOpts{}
 
 		if strings.IndexByte(fmtStr, '%') == -1 {
-			return fmt.Errorf("invalid format string: %s", fmtStr)
+			return fmt.Errorf("invalid format string: %q", fmtStr)
 		}
 		opts.fmtStr = fmtStr
 
 		if codecStr != "" {
 			codec, ok := cid.Codecs[codecStr]
 			if !ok {
-				return fmt.Errorf("unknown IPLD codec: %s", codecStr)
+				return fmt.Errorf("unknown IPLD codec: %q", codecStr)
 			}
 			opts.newCodec = codec
 		} // otherwise, leave it as 0 (not a valid IPLD codec)
@@ -80,13 +82,13 @@ The optional format string is a printf style format string:
 			// noop
 		case "0":
 			if opts.newCodec != 0 && opts.newCodec != cid.DagProtobuf {
-				return fmt.Errorf("cannot convert to CIDv0 with any codec other than DagPB")
+				return fmt.Errorf("cannot convert to CIDv0 with any codec other than dag-pb")
 			}
 			opts.verConv = toCidV0
 		case "1":
 			opts.verConv = toCidV1
 		default:
-			return fmt.Errorf("invalid cid version: %s", verStr)
+			return fmt.Errorf("invalid cid version: %q", verStr)
 		}
 
 		if baseStr != "" {
@@ -123,9 +125,13 @@ type CidFormatRes struct {
 var base32Cmd = &cmds.Command{
 	Helptext: cmds.HelpText{
 		Tagline: "Convert CIDs to Base32 CID version 1.",
+		ShortDescription: `
+'ipfs cid base32' normalizes passes CIDs to their canonical case-insensitive encoding.
+Useful when processing third-party CIDs which could come with arbitrary formats.
+`,
 	},
 	Arguments: []cmds.Argument{
-		cmds.StringArg("cid", true, true, "Cids to convert.").EnableStdin(),
+		cmds.StringArg("cid", true, true, "CIDs to convert.").EnableStdin(),
 	},
 	Run: func(req *cmds.Request, resp cmds.ResponseEmitter, env cmds.Environment) error {
 		opts := cidFormatOpts{
@@ -232,7 +238,7 @@ func emitCids(req *cmds.Request, resp cmds.ResponseEmitter, opts cidFormatOpts)
 
 func toCidV0(c cid.Cid) (cid.Cid, error) {
 	if c.Type() != cid.DagProtobuf {
-		return cid.Cid{}, fmt.Errorf("can't convert non-protobuf nodes to cidv0")
+		return cid.Cid{}, fmt.Errorf("can't convert non-dag-pb nodes to cidv0")
 	}
 	return cid.NewCidV0(c.Hash()), nil
 }
@@ -254,6 +260,9 @@ const (
 var basesCmd = &cmds.Command{
 	Helptext: cmds.HelpText{
 		Tagline: "List available multibase encodings.",
+		ShortDescription: `
+'ipfs cid bases' relies on https://github.com/multiformats/go-multibase
+`,
 	},
 	Options: []cmds.Option{
 		cmds.BoolOption(prefixOptionName, "also include the single letter prefixes in addition to the code"),
@@ -296,21 +305,45 @@ var basesCmd = &cmds.Command{
 }
 
 const (
-	codecsNumericOptionName = "numeric"
+	codecsNumericOptionName   = "numeric"
+	codecsSupportedOptionName = "supported"
 )
 
 var codecsCmd = &cmds.Command{
 	Helptext: cmds.HelpText{
 		Tagline: "List available CID codecs.",
+		ShortDescription: `
+'ipfs cid codecs' relies on https://github.com/multiformats/go-multicodec
+`,
 	},
 	Options: []cmds.Option{
-		cmds.BoolOption(codecsNumericOptionName, "also include numeric codes"),
+		cmds.BoolOption(codecsNumericOptionName, "n", "also include numeric codes"),
+		cmds.BoolOption(codecsSupportedOptionName, "s", "list only codecs supported by go-ipfs commands"),
 	},
 	Run: func(req *cmds.Request, resp cmds.ResponseEmitter, env cmds.Environment) error {
+		listSupported, _ := req.Options[codecsSupportedOptionName].(bool)
+		supportedCodecs := make(map[uint64]struct{})
+		if listSupported {
+			for _, code := range multicodec.ListEncoders() {
+				supportedCodecs[code] = struct{}{}
+			}
+			for _, code := range multicodec.ListDecoders() {
+				supportedCodecs[code] = struct{}{}
+			}
+			// add libp2p-key
+			supportedCodecs[uint64(mc.Libp2pKey)] = struct{}{}
+		}
+
 		var res []CodeAndName
-		// use CodecToStr as there are multiple names for a given code
-		for code, name := range cid.CodecToStr {
-			res = append(res, CodeAndName{int(code), name})
+		for _, code := range mc.KnownCodes() {
+			if code.Tag() == "ipld" {
+				if listSupported {
+					if _, ok := supportedCodecs[uint64(code)]; !ok {
+						continue
+					}
+				}
+				res = append(res, CodeAndName{int(code), mc.Code(code).String()})
+			}
 		}
 		return cmds.EmitOnce(resp, res)
 	},
@@ -334,6 +367,9 @@ var codecsCmd = &cmds.Command{
 var hashesCmd = &cmds.Command{
 	Helptext: cmds.HelpText{
 		Tagline: "List available multihashes.",
+		ShortDescription: `
+'ipfs cid hashes' relies on https://github.com/multiformats/go-multihash
+`,
 	},
 	Options: codecsCmd.Options,
 	Run: func(req *cmds.Request, resp cmds.ResponseEmitter, env cmds.Environment) error {

core/coreapi/block.go

@@ -31,7 +31,7 @@ func (api *BlockAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.Bloc
 	ctx, span := tracing.Span(ctx, "CoreAPI.BlockAPI", "Put")
 	defer span.End()
 
-	settings, pref, err := caopts.BlockPutOptions(opts...)
+	settings, err := caopts.BlockPutOptions(opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -41,7 +41,7 @@ func (api *BlockAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.Bloc
 		return nil, err
 	}
 
-	bcid, err := pref.Sum(data)
+	bcid, err := settings.CidPrefix.Sum(data)
 	if err != nil {
 		return nil, err
 	}

go.mod

@@ -57,7 +57,7 @@ require (
 	github.com/ipfs/go-unixfs v0.3.1
 	github.com/ipfs/go-unixfsnode v1.1.3
 	github.com/ipfs/go-verifcid v0.0.1
-	github.com/ipfs/interface-go-ipfs-core v0.6.2
+	github.com/ipfs/interface-go-ipfs-core v0.7.0
 	github.com/ipfs/tar-utils v0.0.2
 	github.com/ipld/go-car v0.3.2
 	github.com/ipld/go-car/v2 v2.1.1
@@ -96,7 +96,7 @@ require (
 	github.com/multiformats/go-multiaddr v0.5.0
 	github.com/multiformats/go-multiaddr-dns v0.3.1
 	github.com/multiformats/go-multibase v0.0.3
-	github.com/multiformats/go-multicodec v0.4.0
+	github.com/multiformats/go-multicodec v0.4.1
 	github.com/multiformats/go-multihash v0.1.0
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/pkg/errors v0.9.1

go.sum

@@ -605,8 +605,8 @@ github.com/ipfs/go-unixfsnode v1.1.3/go.mod h1:ZZxUM5wXBC+G0Co9FjrYTOm+UlhZTjxLf
 github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E=
 github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0=
 github.com/ipfs/interface-go-ipfs-core v0.4.0/go.mod h1:UJBcU6iNennuI05amq3FQ7g0JHUkibHFAfhfUIy927o=
-github.com/ipfs/interface-go-ipfs-core v0.6.2 h1:nnkq9zhb5O8lPzkZeynEymc83RqkTRqfYH4x5JNUkT4=
-github.com/ipfs/interface-go-ipfs-core v0.6.2/go.mod h1:h3NuO3wzv2KuKazt0zDF2/i8AFRqiKHusyh5DUQQdPA=
+github.com/ipfs/interface-go-ipfs-core v0.7.0 h1:7tb+2upz8oCcjIyjo1atdMk+P+u7wPmI+GksBlLE8js=
+github.com/ipfs/interface-go-ipfs-core v0.7.0/go.mod h1:lF27E/nnSPbylPqKVXGZghal2hzifs3MmjyiEjnc9FY=
 github.com/ipfs/tar-utils v0.0.2 h1:UNgHB4x/PPzbMkmJi+7EqC9LNMPDztOVSnx1HAqSNg4=
 github.com/ipfs/tar-utils v0.0.2/go.mod h1:4qlnRWgTVljIMhSG2SqRYn66NT+3wrv/kZt9V+eqxDM=
 github.com/ipld/go-car v0.3.2 h1:V9wt/80FNfbMRWSD98W5br6fyjUAyVgI2lDOTZX16Lg=
@@ -1188,8 +1188,8 @@ github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPw
 github.com/multiformats/go-multicodec v0.2.0/go.mod h1:/y4YVwkfMyry5kFbMTbLJKErhycTIftytRV+llXdyS4=
 github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ=
 github.com/multiformats/go-multicodec v0.3.1-0.20210902112759-1539a079fd61/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ=
-github.com/multiformats/go-multicodec v0.4.0 h1:fbqb6ky7erjdD+/zaEBJgZWu1i8D6i/wmPywGK7sdow=
-github.com/multiformats/go-multicodec v0.4.0/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ=
+github.com/multiformats/go-multicodec v0.4.1 h1:BSJbf+zpghcZMZrwTYBGwy0CPcVZGWiC72Cp8bBd4R4=
+github.com/multiformats/go-multicodec v0.4.1/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ=
 github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U=
 github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po=
 github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=