From 08062a1c1c37bbdfbae4619a8866c7d4f1f3f0ac Mon Sep 17 00:00:00 2001 From: Mathew Gordon Date: Wed, 6 Dec 2023 04:40:28 -0700 Subject: [PATCH] Make extraction buffer bigger Tiny buffer for extracting led to worse performance, so increase the size of it to where it should be able to accomodate SquashFS's max 1MiB block size and now allocate libdeflate_decompressor on the stack. This should make it thread-safe, but hasn't appeared to make much of a performance impact and finally updated README.md, removing the reference to `squashfuse_tool` and changing the time to the new improved performance --- README.md | 8 ++++---- lib/SquashFs.zig | 37 ++++++++++++++++++++++++++++++++----- lib/test.zig | 1 + src/squashfuse.zig | 34 +++++++++++++++++++--------------- 4 files changed, 56 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index c3bf385..7d703a8 100644 --- a/README.md +++ b/README.md @@ -12,11 +12,11 @@ My main goals for this project are as follows: but the vast majority is still just bindings With some very basic benchmarking, extracting a zlib-compressed AppImage -(FreeCAD, the largest AppImage I've been able to find so far), takes 3.7 -seconds using squashfuse-zig's `squashfuse_tool`. Currently, `squashfs_tool` -is single-thread only. +(FreeCAD, the largest AppImage I've been able to find so far), takes 3.1 +seconds using the `--extract` flag with squashfuse-zig's CLI tool, which +is currently single-thread only. -For reference, `unsquashfs` with multi-threaded decompression takes 1.57 seconds +For reference, `unsquashfs` with multi-threaded decompression takes 1.5 seconds and single-threaded takes 6.5 seconds. Surely almost all of the single-threaded performace gain can be chalked up to diff --git a/lib/SquashFs.zig b/lib/SquashFs.zig index c09603e..f2fea86 100644 --- a/lib/SquashFs.zig +++ b/lib/SquashFs.zig @@ -650,15 +650,42 @@ usingnamespace if (build_options.enable_zlib) *usize, ) c_int; - var ldef_decompressor: ?*anyopaque = null; + // Deflate constants + const litlen_syms = 288; + const offset_syms = 32; + const max_lens_overrun = 137; + const max_num_syms = 288; + const precode_syms = 19; + + // LibDeflate constants + const precode_enough = 128; + const litlen_enough = 2342; + const offset_enough = 402; + + const Decompressor = extern struct { + _: extern union { + precode_lens: [precode_syms]u8, + + _: extern struct { + lens: [litlen_syms + offset_syms + max_lens_overrun]u8, + precode_table: [precode_enough]u32, + }, + + litlen_decode_table: [litlen_enough]u32, + } = undefined, + + offset_decode_table: [offset_enough]u32 = undefined, + sorted_syms: [max_num_syms]u16 = undefined, + static_codes_loaded: bool = false, + litlen_tablebits: u32 = undefined, + free_func: ?*anyopaque = undefined, + }; export fn zig_zlib_decode(in: [*]u8, in_size: usize, out: [*]u8, out_size: *usize) callconv(.C) c.sqfs_err { - if (ldef_decompressor == null) { - ldef_decompressor = libdeflate_alloc_decompressor(); - } + var decompressor = Decompressor{}; const err = libdeflate_zlib_decompress( - ldef_decompressor.?, + &decompressor, in, in_size, out, diff --git a/lib/test.zig b/lib/test.zig index c5d4194..8e1def7 100644 --- a/lib/test.zig +++ b/lib/test.zig @@ -17,6 +17,7 @@ test "open SquashFS image (zlib)" { } // TODO: more tests +// TODO: test executable mounting, extracting, etc fn testWalk(allocator: std.mem.Allocator, sqfs: *SquashFs) !void { var root_inode = sqfs.getRootInode(); diff --git a/src/squashfuse.zig b/src/squashfuse.zig index c5b4191..146d902 100644 --- a/src/squashfuse.zig +++ b/src/squashfuse.zig @@ -29,18 +29,19 @@ pub fn main() !void { var stdout = std.io.getStdOut().writer(); const params = comptime clap.parseParamsComptime( - \\-h, --help display this help and exit - \\-f, --foreground run in foreground - \\-d, --debug enable debug output (runs in foreground) - \\-x, --extract extract the SquashFS image - \\-l, --list list file tree of SquashFS image - \\-o, --option ... use a libFUSE mount option + \\-h, --help display this help and exit + \\-f, --foreground run in foreground + \\-d, --debug enable debug output (runs in foreground) + \\-x, --extract extract the SquashFS image + \\-l, --list list file tree of SquashFS image + \\-o, --option ... use a libFUSE mount option + \\ + \\--offset mount at an offset + \\--extract-src must be used with `--extract`; specify the source inode + \\--extract-dest must be used with `--extract`; specify the destination name + \\--version print the current version + \\--verbose enable verbose printing \\ - \\ --offset mount at an offset - \\ --extract-src must be used with `--extract`; specify the source inode - \\ --extract-dest must be used with `--extract`; specify the destination name - \\ --version print the current version - \\ --verbose enable verbose printing \\... ); @@ -467,6 +468,11 @@ fn extractArchive( var file_found = false; + // TODO: flag to change buf size + const buf_size = 1024 * 1024; + const buf = try allocator.alloc(u8, buf_size); + defer allocator.free(buf); + // Iterate over the SquashFS image and extract each item while (try walker.next()) |entry| { // Skip if the path doesn't match our source @@ -503,13 +509,11 @@ fn extractArchive( try stdout.print("{s}\n", .{prefixed_dest}); } - // TODO: flag to change buf size - var buf: [4096]u8 = undefined; - var inode = entry.inode(); - try inode.extract(&buf, prefixed_dest); + try inode.extract(buf, prefixed_dest); } + // TODO: better error message if (!file_found) { std.debug.print("file ({s}) not found!\n", .{real_src}); }