Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

config: introduce Import section #10421

Merged
merged 3 commits into from
May 14, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ type Config struct {
Experimental Experiments
Plugins Plugins
Pinning Pinning
Import Import

Internal Internal // experimental/unstable options
}
Expand Down
17 changes: 17 additions & 0 deletions config/import.go
hacdias marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
package config

const (
DefaultCidVersion = 0
DefaultUnixFSRawLeaves = false
DefaultUnixFSChunker = "size-262144"
DefaultHashFunction = "sha2-256"
)

// Import configures the default options for ingesting data. This affects commands
// that ingest data, such as 'ipfs add', 'ipfs dag put, 'ipfs block put', 'ipfs files write'.
type Import struct {
CidVersion OptionalInteger
UnixFSRawLeaves Flag
UnixFSChunker OptionalString
HashFunction OptionalString
}
22 changes: 22 additions & 0 deletions config/profile.go
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,28 @@ fetching may be degraded.
return nil
},
},
"legacy-cid-v0": {
Description: `Makes UnixFS import produce legacy CIDv0 with no raw leaves, sha2-256 and 256 KiB chunks.`,

Transform: func(c *Config) error {
c.Import.CidVersion = *NewOptionalInteger(0)
c.Import.UnixFSRawLeaves = False
c.Import.UnixFSChunker = *NewOptionalString("size-262144")
c.Import.HashFunction = *NewOptionalString("sha2-256")
return nil
},
},
"test-cid-v1": {
Description: `Makes UnixFS import produce modern CIDv1 with raw leaves, sha2-256 and 1 MiB chunks.`,

Transform: func(c *Config) error {
c.Import.CidVersion = *NewOptionalInteger(1)
c.Import.UnixFSRawLeaves = True
c.Import.UnixFSChunker = *NewOptionalString("size-1048576")
c.Import.HashFunction = *NewOptionalString("sha2-256")
return nil
},
},
}

func getAvailablePort() (port int, err error) {
Expand Down
33 changes: 31 additions & 2 deletions core/commands/add.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
gopath "path"
"strings"

"github.com/ipfs/kubo/config"
"github.com/ipfs/kubo/core/commands/cmdenv"

"github.com/cheggaaa/pb"
Expand Down Expand Up @@ -155,12 +156,12 @@ See 'dag export' and 'dag import' for more information.
cmds.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation."),
cmds.BoolOption(onlyHashOptionName, "n", "Only chunk and hash - do not write to disk."),
cmds.BoolOption(wrapOptionName, "w", "Wrap files with a directory object."),
cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm, size-[bytes], rabin-[min]-[avg]-[max] or buzhash").WithDefault("size-262144"),
cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm, size-[bytes], rabin-[min]-[avg]-[max] or buzhash"),
cmds.BoolOption(rawLeavesOptionName, "Use raw blocks for leaf nodes."),
cmds.BoolOption(noCopyOptionName, "Add the file using filestore. Implies raw-leaves. (experimental)"),
cmds.BoolOption(fstoreCacheOptionName, "Check the filestore for pre-existing blocks. (experimental)"),
cmds.IntOption(cidVersionOptionName, "CID version. Defaults to 0 unless an option that depends on CIDv1 is passed. Passing version 1 will cause the raw-leaves option to default to true."),
cmds.StringOption(hashOptionName, "Hash function to use. Implies CIDv1 if not sha2-256. (experimental)").WithDefault("sha2-256"),
cmds.StringOption(hashOptionName, "Hash function to use. Implies CIDv1 if not sha2-256. (experimental)"),
cmds.BoolOption(inlineOptionName, "Inline small blocks into CIDs. (experimental)"),
cmds.IntOption(inlineLimitOptionName, "Maximum block size to inline. (experimental)").WithDefault(32),
cmds.BoolOption(pinOptionName, "Pin locally to protect added files from garbage collection.").WithDefault(true),
Expand Down Expand Up @@ -191,6 +192,16 @@ See 'dag export' and 'dag import' for more information.
return err
}

nd, err := cmdenv.GetNode(env)
if err != nil {
return err
}

cfg, err := nd.Repo.Config()
if err != nil {
return err
}

progress, _ := req.Options[progressOptionName].(bool)
trickle, _ := req.Options[trickleOptionName].(bool)
wrap, _ := req.Options[wrapOptionName].(bool)
Expand All @@ -207,6 +218,24 @@ See 'dag export' and 'dag import' for more information.
inlineLimit, _ := req.Options[inlineLimitOptionName].(int)
toFilesStr, toFilesSet := req.Options[toFilesOptionName].(string)

if chunker == "" {
chunker = cfg.Import.UnixFSChunker.WithDefault(config.DefaultUnixFSChunker)
}

if hashFunStr == "" {
hashFunStr = cfg.Import.HashFunction.WithDefault(config.DefaultHashFunction)
}

if !cidVerSet && !cfg.Import.CidVersion.IsDefault() {
cidVerSet = true
cidVer = int(cfg.Import.CidVersion.WithDefault(config.DefaultCidVersion))
}

if !rbset && cfg.Import.UnixFSRawLeaves != config.Default {
rbset = true
rawblks = cfg.Import.UnixFSRawLeaves.WithDefault(config.DefaultUnixFSRawLeaves)
}
hacdias marked this conversation as resolved.
Show resolved Hide resolved

if onlyHash && toFilesSet {
return fmt.Errorf("%s and %s options are not compatible", onlyHashOptionName, toFilesOptionName)
}
Expand Down
17 changes: 16 additions & 1 deletion core/commands/block.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (

"github.com/ipfs/boxo/files"

"github.com/ipfs/kubo/config"
cmdenv "github.com/ipfs/kubo/core/commands/cmdenv"
"github.com/ipfs/kubo/core/commands/cmdutils"

Expand Down Expand Up @@ -153,7 +154,7 @@ only for backward compatibility when a legacy CIDv0 is required (--format=v0).
},
Options: []cmds.Option{
cmds.StringOption(blockCidCodecOptionName, "Multicodec to use in returned CID").WithDefault("raw"),
cmds.StringOption(mhtypeOptionName, "Multihash hash function").WithDefault("sha2-256"),
cmds.StringOption(mhtypeOptionName, "Multihash hash function"),
cmds.IntOption(mhlenOptionName, "Multihash hash length").WithDefault(-1),
cmds.BoolOption(pinOptionName, "Pin added blocks recursively").WithDefault(false),
cmdutils.AllowBigBlockOption,
Expand All @@ -165,7 +166,21 @@ only for backward compatibility when a legacy CIDv0 is required (--format=v0).
return err
}

nd, err := cmdenv.GetNode(env)
if err != nil {
return err
}

cfg, err := nd.Repo.Config()
if err != nil {
return err
}

mhtype, _ := req.Options[mhtypeOptionName].(string)
if mhtype == "" {
mhtype = cfg.Import.HashFunction.WithDefault(config.DefaultHashFunction)
}

mhtval, ok := mh.Names[mhtype]
if !ok {
return fmt.Errorf("unrecognized multihash function: %s", mhtype)
Expand Down
2 changes: 1 addition & 1 deletion core/commands/dag/dag.go
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ into an object of the specified format.
cmds.StringOption("store-codec", "Codec that the stored object will be encoded with").WithDefault("dag-cbor"),
cmds.StringOption("input-codec", "Codec that the input object is encoded in").WithDefault("dag-json"),
cmds.BoolOption("pin", "Pin this object when adding."),
cmds.StringOption("hash", "Hash function to use").WithDefault("sha2-256"),
cmds.StringOption("hash", "Hash function to use"),
cmdutils.AllowBigBlockOption,
},
Run: dagPut,
Expand Down
15 changes: 15 additions & 0 deletions core/commands/dag/put.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
ipldlegacy "github.com/ipfs/go-ipld-legacy"
"github.com/ipfs/kubo/config"
"github.com/ipfs/kubo/core/commands/cmdenv"
"github.com/ipfs/kubo/core/commands/cmdutils"
"github.com/ipld/go-ipld-prime/multicodec"
Expand All @@ -32,11 +33,25 @@ func dagPut(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) e
return err
}

nd, err := cmdenv.GetNode(env)
if err != nil {
return err
}

cfg, err := nd.Repo.Config()
if err != nil {
return err
}

inputCodec, _ := req.Options["input-codec"].(string)
storeCodec, _ := req.Options["store-codec"].(string)
hash, _ := req.Options["hash"].(string)
dopin, _ := req.Options["pin"].(bool)

if hash == "" {
hash = cfg.Import.HashFunction.WithDefault(config.DefaultHashFunction)
}

var icodec mc.Code
if err := icodec.Set(inputCodec); err != nil {
return err
Expand Down
19 changes: 15 additions & 4 deletions core/commands/files.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"strings"

humanize "github.com/dustin/go-humanize"
"github.com/ipfs/kubo/config"
"github.com/ipfs/kubo/core"
"github.com/ipfs/kubo/core/commands/cmdenv"

Expand Down Expand Up @@ -802,18 +803,28 @@ See '--to-files' in 'ipfs add --help' for more information.
return err
}

nd, err := cmdenv.GetNode(env)
if err != nil {
return err
}

cfg, err := nd.Repo.Config()
if err != nil {
return err
}

create, _ := req.Options[filesCreateOptionName].(bool)
mkParents, _ := req.Options[filesParentsOptionName].(bool)
trunc, _ := req.Options[filesTruncateOptionName].(bool)
flush, _ := req.Options[filesFlushOptionName].(bool)
rawLeaves, rawLeavesDef := req.Options[filesRawLeavesOptionName].(bool)

prefix, err := getPrefixNew(req)
if err != nil {
return err
if !rawLeavesDef && cfg.Import.UnixFSRawLeaves != config.Default {
rawLeavesDef = true
rawLeaves = cfg.Import.UnixFSRawLeaves.WithDefault(config.DefaultUnixFSRawLeaves)
}

nd, err := cmdenv.GetNode(env)
prefix, err := getPrefixNew(req)
if err != nil {
return err
}
Expand Down
11 changes: 11 additions & 0 deletions docs/changelogs/v0.29.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
- [Overview](#overview)
- [🔦 Highlights](#-highlights)
- [Add search functionality for pin names](#add-search-functionality-for-pin-names)
- [Customizing `ipfs add` defaults](#customizing-ipfs-add-defaults)
- [📝 Changelog](#-changelog)
- [👨‍👩‍👧‍👦 Contributors](#-contributors)

Expand All @@ -18,6 +19,16 @@

It is now possible to search for pins by name. To do so, use `ipfs pin ls --name "SomeName"`. The search is case-sensitive and will return all pins having a name which contains the exact word provided.

#### Customizing `ipfs add` defaults

This release supports overriding global data ingestion defaults used by commands like `ipfs add` via user-defined [`Import.*` configuration options](../config.md#import).
The hash function, CID version, or UnixFS raw leaves and chunker behaviors can be set once, and used as the new implicit default for `ipfs add`.

> [!TIP]
> As a convenience, two CID [profiles](../config.md#profile) are provided: `legacy-cid-v0` and `test-cid-v1`.
> A test profile that defaults to modern CIDv1 can be applied via `ipfs config profile apply test-cid-v1`.
> We encourage users to try it and report any issues.
### 📝 Changelog

### 👨‍👩‍👧‍👦 Contributors
58 changes: 58 additions & 0 deletions docs/config.md
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,11 @@ config file at runtime.
- [`DNS`](#dns)
- [`DNS.Resolvers`](#dnsresolvers)
- [`DNS.MaxCacheTTL`](#dnsmaxcachettl)
- [`Import`](#import)
- [`Import.CidVersion`](#importcidversion)
- [`Import.UnixFSRawLeaves`](#importunixfsrawleaves)
- [`Import.UnixFSChunker`](#importunixfschunker)
- [`Import.HashFunction`](#importhashfunction)

## Profiles

Expand Down Expand Up @@ -265,6 +270,21 @@ documented in `ipfs config profile --help`.

Use this profile with caution.

- `legacy-cid-v0`

Makes UnixFS import (`ipfs add`) produce legacy CIDv0 with no raw leaves, sha2-256 and 256 KiB chunks.

> [!WARNING]
> This profile is provided for legacy users and should not be used for new projects.
- `test-cid-v1`

Makes UnixFS import (`ipfs add`) produce modern CIDv1 with raw leaves, sha2-256 and 1 MiB chunks.

> [!NOTE]
> This profile will become the new implicit default, provided for testing purposes.
> Follow [kubo#4143](https://github.com/ipfs/kubo/issues/4143) for more details.
## Types

This document refers to the standard JSON types (e.g., `null`, `string`,
Expand Down Expand Up @@ -2377,3 +2397,41 @@ Note: this does NOT work with Go's default DNS resolver. To make this a global s
Default: Respect DNS Response TTL

Type: `optionalDuration`

## `Import`

Options to configure the default options used for ingesting data, in commands such as `ipfs add` or `ipfs block put`. All affected commands are detailed per option.

Note that using flags will override the options defined here.

### `Import.CidVersion`

The default CID version. Commands affected: `ipfs add`.

Default: `0`

Type: `optionalInteger`

### `Import.UnixFSRawLeaves`

The default UnixFS raw leaves option. Commands affected: `ipfs add`, `ipfs files write`.

Default: `false` if `CidVersion=0`; `true` if `CidVersion=1`

Type: `flag`

### `Import.UnixFSChunker`

The default UnixFS chunker. Commands affected: `ipfs add`.

Default: `size-262144`

Type: `optionalString`

### `Import.HashFunction`

The default hash function. Commands affected: `ipfs add`, `ipfs block put`, `ipfs dag put`.

Default: `sha2-256`

Type: `optionalString`
Loading
Loading