Skip to content

Commit

Permalink
btrfs-progs: mkfs: add zstd to --compress option
Browse files Browse the repository at this point in the history 
Allows --compress to work with zstd, when compiled in.

Signed-off-by: Mark Harmstone <[email protected]>
  • Loading branch information
@maharmstone
maharmstone committed Oct 15, 2024
1 parent 82cec85 commit c614436
Show file tree
Hide file tree
Showing 3 changed files with 221 additions and 4 deletions.
4 changes: 2 additions & 2 deletions Documentation/mkfs.btrfs.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -214,8 +214,8 @@ OPTIONS
--compress <algo>:<level>
Try to compress files when using *--rootdir*. Supported values for *algo* are
*no* (the default) and *zlib*. The optional value *level* is a
compression level, from 1 to 9 for ZLIB.
*no* (the default), *zlib*, and *zstd*. The optional value *level* is a
compression level, from 1 to 9 for ZLIB and from 1 to 15 for ZSTD.

As with the kernel, :command:`mkfs.btrfs` won't write compressed extents when
they would be larger than the uncompressed versions, and will mark a file as
Expand Down
2 changes: 1 addition & 1 deletion mkfs/main.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -443,7 +443,7 @@ static const char * const mkfs_usage[] = {
OPTLINE("-u|--subvol TYPE:SUBDIR", "create SUBDIR as subvolume rather than normal directory, can be specified multiple times"),
OPTLINE("--shrink", "(with --rootdir) shrink the filled filesystem to minimal size"),
OPTLINE("-K|--nodiscard", "do not perform whole device TRIM"),
OPTLINE("--compress ALGO:LEVEL", "compression algorithm and level to use; ALGO can be no (default), zlib"),
OPTLINE("--compress ALGO:LEVEL", "compression algorithm and level to use; ALGO can be no (default), zlib, zstd"),
OPTLINE("-f|--force", "force overwrite of existing filesystem"),
"General:",
OPTLINE("-q|--quiet", "no messages except errors"),
Expand Down
219 changes: 218 additions & 1 deletion mkfs/rootdir.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,10 @@
#include <limits.h>
#include <stdlib.h>
#include <string.h>
#if COMPRESSION_ZSTD
#include <zstd.h>
#include <zstd_errors.h>
#endif
#include <zlib.h>
#include "kernel-lib/sizes.h"
#include "kernel-shared/accessors.h"
Expand All @@ -49,6 +53,9 @@
#define ZLIB_BTRFS_DEFAULT_LEVEL 3
#define ZLIB_BTRFS_MAX_LEVEL 9

#define ZSTD_BTRFS_DEFAULT_LEVEL 3
#define ZSTD_BTRFS_MAX_LEVEL 15

static u32 fs_block_size;

/*
Expand Down Expand Up @@ -438,6 +445,107 @@ static ssize_t zlib_compress_extent(struct btrfs_inode_item *btrfs_inode,
return 0;
}

#if COMPRESSION_ZSTD
static ssize_t zstd_compress_extent(struct btrfs_inode_item *btrfs_inode,
u32 sectorsize, const void *in_buf,
size_t in_size, void *out_buf)
{
ZSTD_CCtx *zstd_ctx;
ZSTD_inBuffer input;
ZSTD_outBuffer output;
size_t zstd_ret;
ssize_t ret;

zstd_ctx = ZSTD_createCCtx();
if (!zstd_ctx) {
error_msg(ERROR_MSG_MEMORY, NULL);
return -ENOMEM;
}

zstd_ret = ZSTD_CCtx_setParameter(zstd_ctx, ZSTD_c_compressionLevel,
g_compression_level);
if (ZSTD_isError(zstd_ret)) {
error("ZSTD_CCtx_setParameter failed: %s",
ZSTD_getErrorName(zstd_ret));
ret = -EINVAL;
goto out;
}

zstd_ret = ZSTD_CCtx_setPledgedSrcSize(zstd_ctx, in_size);
if (ZSTD_isError(zstd_ret)) {
error("ZSTD_CCtx_setPledgedSrcSize failed: %s",
ZSTD_getErrorName(zstd_ret));
ret = -EINVAL;
goto out;
}

output.dst = out_buf;
output.size = BTRFS_MAX_COMPRESSED;
output.pos = 0;

input.src = in_buf;
input.pos = 0;

/* Try to compress the first sector - if it would be larger, mark the
* inode as nocompress and return. */
if (!(btrfs_stack_inode_flags(btrfs_inode) & BTRFS_INODE_COMPRESS)) {
input.size = sectorsize;

zstd_ret = ZSTD_compressStream2(zstd_ctx, &output, &input,
ZSTD_e_flush);

if (ZSTD_isError(zstd_ret)) {
error("ZSTD_compressStream2 failed: %s",
ZSTD_getErrorName(zstd_ret));
ret = -EINVAL;
goto out;
}

if (zstd_ret != 0 || output.pos > sectorsize) {
u64 flags;

flags = btrfs_stack_inode_flags(btrfs_inode);
flags |= BTRFS_INODE_NOCOMPRESS;
btrfs_set_stack_inode_flags(btrfs_inode, flags);

ret = 0;
goto out;
}
}

input.size = in_size;

zstd_ret = ZSTD_compressStream2(zstd_ctx, &output, &input, ZSTD_e_end);

if (ZSTD_isError(zstd_ret)) {
error("ZSTD_compressStream2 failed: %s",
ZSTD_getErrorName(zstd_ret));
ret = -EINVAL;
goto out;
}

if (zstd_ret == 0 && output.pos <= in_size - sectorsize) {
ret = output.pos;
goto out;
}

if (!(btrfs_stack_inode_flags(btrfs_inode) & BTRFS_INODE_COMPRESS)) {
u64 flags;

flags = btrfs_stack_inode_flags(btrfs_inode);
flags |= BTRFS_INODE_NOCOMPRESS;
btrfs_set_stack_inode_flags(btrfs_inode, flags);
}

ret = 0;

out:
ZSTD_freeCCtx(zstd_ctx);

return ret;
}
#endif

/*
* keep our extent size at 1MB max, this makes it easier to work
* inside the tiny block groups created during mkfs
Expand Down Expand Up @@ -493,6 +601,13 @@ static int read_and_write_extent(struct btrfs_trans_handle *trans,
buf, bytes_read,
comp_buf);
break;
#if COMPRESSION_ZSTD
case BTRFS_COMPRESS_ZSTD:
comp_ret = zstd_compress_extent(btrfs_inode, sectorsize,
buf, bytes_read,
comp_buf);
break;
#endif
default:
comp_ret = -EINVAL;
break;
Expand Down Expand Up @@ -528,7 +643,7 @@ static int read_and_write_extent(struct btrfs_trans_handle *trans,
}

if (do_comp) {
u64 flags;
u64 flags, features;

to_write = round_up(comp_ret, sectorsize);
write_buf = comp_buf;
Expand All @@ -537,6 +652,13 @@ static int read_and_write_extent(struct btrfs_trans_handle *trans,
flags = btrfs_stack_inode_flags(btrfs_inode);
flags |= BTRFS_INODE_COMPRESS;
btrfs_set_stack_inode_flags(btrfs_inode, flags);

if (g_compression == BTRFS_COMPRESS_ZSTD) {
features = btrfs_super_incompat_flags(trans->fs_info->super_copy);
features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD;
btrfs_set_super_incompat_flags(trans->fs_info->super_copy,
features);
}
} else {
to_write = round_up(to_read, sectorsize);
write_buf = buf;
Expand Down Expand Up @@ -632,6 +754,82 @@ static int zlib_compress_inline_extent(char *buf, u64 size, char **comp_buf,
return ret;
}

#if COMPRESSION_ZSTD
static int zstd_compress_inline_extent(char *buf, u64 size, char **comp_buf,
u64 *comp_size)
{
ZSTD_CCtx *zstd_ctx;
ZSTD_inBuffer input;
ZSTD_outBuffer output;
size_t zstd_ret;
ssize_t ret;
char *out = NULL;

zstd_ctx = ZSTD_createCCtx();
if (!zstd_ctx) {
error_msg(ERROR_MSG_MEMORY, NULL);
return -ENOMEM;
}

zstd_ret = ZSTD_CCtx_setParameter(zstd_ctx, ZSTD_c_compressionLevel,
g_compression_level);
if (ZSTD_isError(zstd_ret)) {
error("ZSTD_CCtx_setParameter failed: %s",
ZSTD_getErrorName(zstd_ret));
ret = -EINVAL;
goto out;
}

zstd_ret = ZSTD_CCtx_setPledgedSrcSize(zstd_ctx, size);
if (ZSTD_isError(zstd_ret)) {
error("ZSTD_CCtx_setPledgedSrcSize failed: %s",
ZSTD_getErrorName(zstd_ret));
ret = -EINVAL;
goto out;
}

out = malloc(size);
if (!out) {
error_msg(ERROR_MSG_MEMORY, NULL);
ret = -ENOMEM;
goto out;
}

output.dst = out;
output.size = size;
output.pos = 0;

input.src = buf;
input.pos = 0;
input.size = size;

zstd_ret = ZSTD_compressStream2(zstd_ctx, &output, &input, ZSTD_e_end);

if (ZSTD_isError(zstd_ret)) {
error("ZSTD_compressStream2 failed: %s",
ZSTD_getErrorName(zstd_ret));
ret = -EINVAL;
goto out;
}

if (zstd_ret == 0 && output.pos < size) {
ret = 1;
*comp_buf = out;
*comp_size = output.pos;
} else {
ret = 0;
}

out:
if (ret != 1)
free(out);

ZSTD_freeCCtx(zstd_ctx);

return ret;
}
#endif

static int add_file_items(struct btrfs_trans_handle *trans,
struct btrfs_root *root,
struct btrfs_inode_item *btrfs_inode, u64 objectid,
Expand Down Expand Up @@ -679,6 +877,14 @@ static int add_file_items(struct btrfs_trans_handle *trans,
if (ret < 0)
goto end;
break;
#if COMPRESSION_ZSTD
case BTRFS_COMPRESS_ZSTD:
ret = zstd_compress_inline_extent(buffer, st->st_size,
&comp_buf, &comp_size);
if (ret < 0)
goto end;
break;
#endif
default:
ret = 0;
break;
Expand Down Expand Up @@ -1153,6 +1359,17 @@ int btrfs_mkfs_fill_dir(struct btrfs_trans_handle *trans, const char *source_dir
else if (compression_level == 0)
compression_level = ZLIB_BTRFS_DEFAULT_LEVEL;
break;
case BTRFS_COMPRESS_ZSTD:
#if !COMPRESSION_ZSTD
error("zstd support not compiled in");
return -EINVAL;
#else
if (compression_level > ZSTD_BTRFS_MAX_LEVEL)
compression_level = ZSTD_BTRFS_MAX_LEVEL;
else if (compression_level == 0)
compression_level = ZSTD_BTRFS_DEFAULT_LEVEL;
break;
#endif
default:
error("unsupported compression type");
return -EINVAL;
Expand Down

0 comments on commit c614436

Please sign in to comment.