Skip to content

Commit

Permalink
Make extraction buffer bigger
Browse files Browse the repository at this point in the history
Tiny buffer for extracting led to worse performance, so increase the
size of it to where it should be able to accomodate SquashFS's max 1MiB
block size and now allocate libdeflate_decompressor on the stack. This
should make it thread-safe, but hasn't appeared to make much of a
performance impact and finally updated README.md, removing the reference
to `squashfuse_tool` and changing the time to the new improved
performance
  • Loading branch information
mgord9518 committed Dec 6, 2023
1 parent 82269bd commit 08062a1
Show file tree
Hide file tree
Showing 4 changed files with 56 additions and 24 deletions.
8 changes: 4 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,11 @@ My main goals for this project are as follows:
but the vast majority is still just bindings

With some very basic benchmarking, extracting a zlib-compressed AppImage
(FreeCAD, the largest AppImage I've been able to find so far), takes 3.7
seconds using squashfuse-zig's `squashfuse_tool`. Currently, `squashfs_tool`
is single-thread only.
(FreeCAD, the largest AppImage I've been able to find so far), takes 3.1
seconds using the `--extract` flag with squashfuse-zig's CLI tool, which
is currently single-thread only.

For reference, `unsquashfs` with multi-threaded decompression takes 1.57 seconds
For reference, `unsquashfs` with multi-threaded decompression takes 1.5 seconds
and single-threaded takes 6.5 seconds.

Surely almost all of the single-threaded performace gain can be chalked up to
Expand Down
37 changes: 32 additions & 5 deletions lib/SquashFs.zig
Original file line number Diff line number Diff line change
Expand Up @@ -650,15 +650,42 @@ usingnamespace if (build_options.enable_zlib)
*usize,
) c_int;

var ldef_decompressor: ?*anyopaque = null;
// Deflate constants
const litlen_syms = 288;
const offset_syms = 32;
const max_lens_overrun = 137;
const max_num_syms = 288;
const precode_syms = 19;

// LibDeflate constants
const precode_enough = 128;
const litlen_enough = 2342;
const offset_enough = 402;

const Decompressor = extern struct {
_: extern union {
precode_lens: [precode_syms]u8,

_: extern struct {
lens: [litlen_syms + offset_syms + max_lens_overrun]u8,
precode_table: [precode_enough]u32,
},

litlen_decode_table: [litlen_enough]u32,
} = undefined,

offset_decode_table: [offset_enough]u32 = undefined,
sorted_syms: [max_num_syms]u16 = undefined,
static_codes_loaded: bool = false,
litlen_tablebits: u32 = undefined,
free_func: ?*anyopaque = undefined,
};

export fn zig_zlib_decode(in: [*]u8, in_size: usize, out: [*]u8, out_size: *usize) callconv(.C) c.sqfs_err {
if (ldef_decompressor == null) {
ldef_decompressor = libdeflate_alloc_decompressor();
}
var decompressor = Decompressor{};

const err = libdeflate_zlib_decompress(
ldef_decompressor.?,
&decompressor,
in,
in_size,
out,
Expand Down
1 change: 1 addition & 0 deletions lib/test.zig
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ test "open SquashFS image (zlib)" {
}

// TODO: more tests
// TODO: test executable mounting, extracting, etc
fn testWalk(allocator: std.mem.Allocator, sqfs: *SquashFs) !void {
var root_inode = sqfs.getRootInode();

Expand Down
34 changes: 19 additions & 15 deletions src/squashfuse.zig
Original file line number Diff line number Diff line change
Expand Up @@ -29,18 +29,19 @@ pub fn main() !void {
var stdout = std.io.getStdOut().writer();

const params = comptime clap.parseParamsComptime(
\\-h, --help display this help and exit
\\-f, --foreground run in foreground
\\-d, --debug enable debug output (runs in foreground)
\\-x, --extract extract the SquashFS image
\\-l, --list list file tree of SquashFS image
\\-o, --option <str>... use a libFUSE mount option
\\-h, --help display this help and exit
\\-f, --foreground run in foreground
\\-d, --debug enable debug output (runs in foreground)
\\-x, --extract extract the SquashFS image
\\-l, --list list file tree of SquashFS image
\\-o, --option <str>... use a libFUSE mount option
\\
\\--offset <usize> mount at an offset
\\--extract-src <str> must be used with `--extract`; specify the source inode
\\--extract-dest <str> must be used with `--extract`; specify the destination name
\\--version print the current version
\\--verbose enable verbose printing
\\
\\ --offset <usize> mount at an offset
\\ --extract-src <str> must be used with `--extract`; specify the source inode
\\ --extract-dest <str> must be used with `--extract`; specify the destination name
\\ --version print the current version
\\ --verbose enable verbose printing
\\<str>...
);

Expand Down Expand Up @@ -467,6 +468,11 @@ fn extractArchive(

var file_found = false;

// TODO: flag to change buf size
const buf_size = 1024 * 1024;
const buf = try allocator.alloc(u8, buf_size);
defer allocator.free(buf);

// Iterate over the SquashFS image and extract each item
while (try walker.next()) |entry| {
// Skip if the path doesn't match our source
Expand Down Expand Up @@ -503,13 +509,11 @@ fn extractArchive(
try stdout.print("{s}\n", .{prefixed_dest});
}

// TODO: flag to change buf size
var buf: [4096]u8 = undefined;

var inode = entry.inode();
try inode.extract(&buf, prefixed_dest);
try inode.extract(buf, prefixed_dest);
}

// TODO: better error message
if (!file_found) {
std.debug.print("file ({s}) not found!\n", .{real_src});
}
Expand Down

0 comments on commit 08062a1

Please sign in to comment.