config: introduce Import section (#10421)

Co-authored-by: Marcin Rataj <[email protected]>

config/config.go

@@ -36,6 +36,7 @@ type Config struct {
 	Experimental Experiments
 	Plugins      Plugins
 	Pinning      Pinning
+	Import       Import
 
 	Internal Internal // experimental/unstable options
 }

config/import.go

@@ -0,0 +1,17 @@
+package config
+
+const (
+	DefaultCidVersion      = 0
+	DefaultUnixFSRawLeaves = false
+	DefaultUnixFSChunker   = "size-262144"
+	DefaultHashFunction    = "sha2-256"
+)
+
+// Import configures the default options for ingesting data. This affects commands
+// that ingest data, such as 'ipfs add', 'ipfs dag put, 'ipfs block put', 'ipfs files write'.
+type Import struct {
+	CidVersion      OptionalInteger
+	UnixFSRawLeaves Flag
+	UnixFSChunker   OptionalString
+	HashFunction    OptionalString
+}

config/profile.go

@@ -204,6 +204,28 @@ fetching may be degraded.
 			return nil
 		},
 	},
+	"legacy-cid-v0": {
+		Description: `Makes UnixFS import produce legacy CIDv0 with no raw leaves, sha2-256 and 256 KiB chunks.`,
+
+		Transform: func(c *Config) error {
+			c.Import.CidVersion = *NewOptionalInteger(0)
+			c.Import.UnixFSRawLeaves = False
+			c.Import.UnixFSChunker = *NewOptionalString("size-262144")
+			c.Import.HashFunction = *NewOptionalString("sha2-256")
+			return nil
+		},
+	},
+	"test-cid-v1": {
+		Description: `Makes UnixFS import produce modern CIDv1 with raw leaves, sha2-256 and 1 MiB chunks.`,
+
+		Transform: func(c *Config) error {
+			c.Import.CidVersion = *NewOptionalInteger(1)
+			c.Import.UnixFSRawLeaves = True
+			c.Import.UnixFSChunker = *NewOptionalString("size-1048576")
+			c.Import.HashFunction = *NewOptionalString("sha2-256")
+			return nil
+		},
+	},
 }
 
 func getAvailablePort() (port int, err error) {

core/commands/add.go

@@ -8,6 +8,7 @@ import (
 	gopath "path"
 	"strings"
 
+	"github.com/ipfs/kubo/config"
 	"github.com/ipfs/kubo/core/commands/cmdenv"
 
 	"github.com/cheggaaa/pb"
@@ -155,12 +156,12 @@ See 'dag export' and 'dag import' for more information.
 		cmds.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation."),
 		cmds.BoolOption(onlyHashOptionName, "n", "Only chunk and hash - do not write to disk."),
 		cmds.BoolOption(wrapOptionName, "w", "Wrap files with a directory object."),
-		cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm, size-[bytes], rabin-[min]-[avg]-[max] or buzhash").WithDefault("size-262144"),
+		cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm, size-[bytes], rabin-[min]-[avg]-[max] or buzhash"),
 		cmds.BoolOption(rawLeavesOptionName, "Use raw blocks for leaf nodes."),
 		cmds.BoolOption(noCopyOptionName, "Add the file using filestore. Implies raw-leaves. (experimental)"),
 		cmds.BoolOption(fstoreCacheOptionName, "Check the filestore for pre-existing blocks. (experimental)"),
 		cmds.IntOption(cidVersionOptionName, "CID version. Defaults to 0 unless an option that depends on CIDv1 is passed. Passing version 1 will cause the raw-leaves option to default to true."),
-		cmds.StringOption(hashOptionName, "Hash function to use. Implies CIDv1 if not sha2-256. (experimental)").WithDefault("sha2-256"),
+		cmds.StringOption(hashOptionName, "Hash function to use. Implies CIDv1 if not sha2-256. (experimental)"),
 		cmds.BoolOption(inlineOptionName, "Inline small blocks into CIDs. (experimental)"),
 		cmds.IntOption(inlineLimitOptionName, "Maximum block size to inline. (experimental)").WithDefault(32),
 		cmds.BoolOption(pinOptionName, "Pin locally to protect added files from garbage collection.").WithDefault(true),
@@ -191,6 +192,16 @@ See 'dag export' and 'dag import' for more information.
 			return err
 		}
 
+		nd, err := cmdenv.GetNode(env)
+		if err != nil {
+			return err
+		}
+
+		cfg, err := nd.Repo.Config()
+		if err != nil {
+			return err
+		}
+
 		progress, _ := req.Options[progressOptionName].(bool)
 		trickle, _ := req.Options[trickleOptionName].(bool)
 		wrap, _ := req.Options[wrapOptionName].(bool)
@@ -207,6 +218,24 @@ See 'dag export' and 'dag import' for more information.
 		inlineLimit, _ := req.Options[inlineLimitOptionName].(int)
 		toFilesStr, toFilesSet := req.Options[toFilesOptionName].(string)
 
+		if chunker == "" {
+			chunker = cfg.Import.UnixFSChunker.WithDefault(config.DefaultUnixFSChunker)
+		}
+
+		if hashFunStr == "" {
+			hashFunStr = cfg.Import.HashFunction.WithDefault(config.DefaultHashFunction)
+		}
+
+		if !cidVerSet && !cfg.Import.CidVersion.IsDefault() {
+			cidVerSet = true
+			cidVer = int(cfg.Import.CidVersion.WithDefault(config.DefaultCidVersion))
+		}
+
+		if !rbset && cfg.Import.UnixFSRawLeaves != config.Default {
+			rbset = true
+			rawblks = cfg.Import.UnixFSRawLeaves.WithDefault(config.DefaultUnixFSRawLeaves)
+		}
+
 		if onlyHash && toFilesSet {
 			return fmt.Errorf("%s and %s options are not compatible", onlyHashOptionName, toFilesOptionName)
 		}

core/commands/block.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/ipfs/boxo/files"
 
+	"github.com/ipfs/kubo/config"
 	cmdenv "github.com/ipfs/kubo/core/commands/cmdenv"
 	"github.com/ipfs/kubo/core/commands/cmdutils"
 
@@ -153,7 +154,7 @@ only for backward compatibility when a legacy CIDv0 is required (--format=v0).
 	},
 	Options: []cmds.Option{
 		cmds.StringOption(blockCidCodecOptionName, "Multicodec to use in returned CID").WithDefault("raw"),
-		cmds.StringOption(mhtypeOptionName, "Multihash hash function").WithDefault("sha2-256"),
+		cmds.StringOption(mhtypeOptionName, "Multihash hash function"),
 		cmds.IntOption(mhlenOptionName, "Multihash hash length").WithDefault(-1),
 		cmds.BoolOption(pinOptionName, "Pin added blocks recursively").WithDefault(false),
 		cmdutils.AllowBigBlockOption,
@@ -165,7 +166,21 @@ only for backward compatibility when a legacy CIDv0 is required (--format=v0).
 			return err
 		}
 
+		nd, err := cmdenv.GetNode(env)
+		if err != nil {
+			return err
+		}
+
+		cfg, err := nd.Repo.Config()
+		if err != nil {
+			return err
+		}
+
 		mhtype, _ := req.Options[mhtypeOptionName].(string)
+		if mhtype == "" {
+			mhtype = cfg.Import.HashFunction.WithDefault(config.DefaultHashFunction)
+		}
+
 		mhtval, ok := mh.Names[mhtype]
 		if !ok {
 			return fmt.Errorf("unrecognized multihash function: %s", mhtype)

core/commands/dag/dag.go

@@ -87,7 +87,7 @@ into an object of the specified format.
 		cmds.StringOption("store-codec", "Codec that the stored object will be encoded with").WithDefault("dag-cbor"),
 		cmds.StringOption("input-codec", "Codec that the input object is encoded in").WithDefault("dag-json"),
 		cmds.BoolOption("pin", "Pin this object when adding."),
-		cmds.StringOption("hash", "Hash function to use").WithDefault("sha2-256"),
+		cmds.StringOption("hash", "Hash function to use"),
 		cmdutils.AllowBigBlockOption,
 	},
 	Run:  dagPut,

core/commands/dag/put.go

@@ -7,6 +7,7 @@ import (
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
 	ipldlegacy "github.com/ipfs/go-ipld-legacy"
+	"github.com/ipfs/kubo/config"
 	"github.com/ipfs/kubo/core/commands/cmdenv"
 	"github.com/ipfs/kubo/core/commands/cmdutils"
 	"github.com/ipld/go-ipld-prime/multicodec"
@@ -32,11 +33,25 @@ func dagPut(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) e
 		return err
 	}
 
+	nd, err := cmdenv.GetNode(env)
+	if err != nil {
+		return err
+	}
+
+	cfg, err := nd.Repo.Config()
+	if err != nil {
+		return err
+	}
+
 	inputCodec, _ := req.Options["input-codec"].(string)
 	storeCodec, _ := req.Options["store-codec"].(string)
 	hash, _ := req.Options["hash"].(string)
 	dopin, _ := req.Options["pin"].(bool)
 
+	if hash == "" {
+		hash = cfg.Import.HashFunction.WithDefault(config.DefaultHashFunction)
+	}
+
 	var icodec mc.Code
 	if err := icodec.Set(inputCodec); err != nil {
 		return err

core/commands/files.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 
 	humanize "github.com/dustin/go-humanize"
+	"github.com/ipfs/kubo/config"
 	"github.com/ipfs/kubo/core"
 	"github.com/ipfs/kubo/core/commands/cmdenv"
 
@@ -802,18 +803,28 @@ See '--to-files' in 'ipfs add --help' for more information.
 			return err
 		}
 
+		nd, err := cmdenv.GetNode(env)
+		if err != nil {
+			return err
+		}
+
+		cfg, err := nd.Repo.Config()
+		if err != nil {
+			return err
+		}
+
 		create, _ := req.Options[filesCreateOptionName].(bool)
 		mkParents, _ := req.Options[filesParentsOptionName].(bool)
 		trunc, _ := req.Options[filesTruncateOptionName].(bool)
 		flush, _ := req.Options[filesFlushOptionName].(bool)
 		rawLeaves, rawLeavesDef := req.Options[filesRawLeavesOptionName].(bool)
 
-		prefix, err := getPrefixNew(req)
-		if err != nil {
-			return err
+		if !rawLeavesDef && cfg.Import.UnixFSRawLeaves != config.Default {
+			rawLeavesDef = true
+			rawLeaves = cfg.Import.UnixFSRawLeaves.WithDefault(config.DefaultUnixFSRawLeaves)
 		}
 
-		nd, err := cmdenv.GetNode(env)
+		prefix, err := getPrefixNew(req)
 		if err != nil {
 			return err
 		}

docs/changelogs/v0.29.md

@@ -7,6 +7,7 @@
 - [Overview](#overview)
 - [🔦 Highlights](#-highlights)
   - [Add search functionality for pin names](#add-search-functionality-for-pin-names)
+  - [Customizing `ipfs add` defaults](#customizing-ipfs-add-defaults)
 - [📝 Changelog](#-changelog)
 - [👨‍👩‍👧‍👦 Contributors](#-contributors)
 
@@ -18,6 +19,16 @@
 
 It is now possible to search for pins by name. To do so, use `ipfs pin ls --name "SomeName"`. The search is case-sensitive and will return all pins having a name which contains the exact word provided.
 
+#### Customizing `ipfs add` defaults
+
+This release supports overriding global data ingestion defaults used by commands like `ipfs add` via user-defined [`Import.*` configuration options](../config.md#import).
+The hash function, CID version, or UnixFS raw leaves and chunker behaviors can be set once, and used as the new implicit default for `ipfs add`.
+
+> [!TIP]
+> As a convenience, two CID [profiles](../config.md#profile) are provided: `legacy-cid-v0` and `test-cid-v1`.
+> A test profile that defaults to modern CIDv1 can be applied via `ipfs config profile apply test-cid-v1`.
+> We encourage users to try it and report any issues.
+
 ### 📝 Changelog
 
 ### 👨‍👩‍👧‍👦 Contributors

docs/config.md

@@ -175,6 +175,11 @@ config file at runtime.
   - [`DNS`](#dns)
     - [`DNS.Resolvers`](#dnsresolvers)
     - [`DNS.MaxCacheTTL`](#dnsmaxcachettl)
+  - [`Import`](#import)
+    - [`Import.CidVersion`](#importcidversion)
+    - [`Import.UnixFSRawLeaves`](#importunixfsrawleaves)
+    - [`Import.UnixFSChunker`](#importunixfschunker)
+    - [`Import.HashFunction`](#importhashfunction)
 
 ## Profiles
 
@@ -265,6 +270,21 @@ documented in `ipfs config profile --help`.
 
   Use this profile with caution.
 
+- `legacy-cid-v0`
+
+  Makes UnixFS import (`ipfs add`) produce legacy CIDv0 with no raw leaves, sha2-256 and 256 KiB chunks.
+
+  > [!WARNING]
+  > This profile is provided for legacy users and should not be used for new projects.
+
+- `test-cid-v1`
+
+  Makes UnixFS import (`ipfs add`) produce modern CIDv1 with raw leaves, sha2-256 and 1 MiB chunks.
+
+  > [!NOTE]
+  > This profile will become the new implicit default, provided for testing purposes.
+  > Follow [kubo#4143](https://github.com/ipfs/kubo/issues/4143) for more details.
+
 ## Types
 
 This document refers to the standard JSON types (e.g., `null`, `string`,
@@ -2377,3 +2397,41 @@ Note: this does NOT work with Go's default DNS resolver. To make this a global s
 Default: Respect DNS Response TTL
 
 Type: `optionalDuration`
+
+## `Import`
+
+Options to configure the default options used for ingesting data, in commands such as `ipfs add` or `ipfs block put`. All affected commands are detailed per option.
+
+Note that using flags will override the options defined here.
+
+### `Import.CidVersion`
+
+The default CID version. Commands affected: `ipfs add`.
+
+Default: `0`
+
+Type: `optionalInteger`
+
+### `Import.UnixFSRawLeaves`
+
+The default UnixFS raw leaves option. Commands affected: `ipfs add`, `ipfs files write`.
+
+Default: `false` if `CidVersion=0`; `true` if `CidVersion=1`
+
+Type: `flag`
+
+### `Import.UnixFSChunker`
+
+The default UnixFS chunker. Commands affected: `ipfs add`.
+
+Default: `size-262144`
+
+Type: `optionalString`
+
+### `Import.HashFunction`
+
+The default hash function. Commands affected: `ipfs add`, `ipfs block put`, `ipfs dag put`.
+
+Default: `sha2-256`
+
+Type: `optionalString`