From 89544870d22beb9d60e1198b997430cc03e84ad2 Mon Sep 17 00:00:00 2001 From: Mathew Gordon Date: Tue, 27 Aug 2024 07:19:07 -0600 Subject: [PATCH] Further simplify compression code, add libz support --- build.zig | 1 + build.zig.zon | 2 +- lib/compression.zig | 87 +++++++++++++++++++---------- lib/compression/lz4_liblz4.zig | 21 +++---- lib/compression/xz_liblzma.zig | 33 +++++------ lib/compression/zlib_libdeflate.zig | 81 +++++++++++++-------------- lib/compression/zlib_libz.zig | 51 +++++++++++++++++ lib/compression/zstd_libzstd.zig | 29 +++++----- 8 files changed, 187 insertions(+), 118 deletions(-) create mode 100644 lib/compression/zlib_libz.zig diff --git a/build.zig b/build.zig index 1c1db56..8543771 100644 --- a/build.zig +++ b/build.zig @@ -487,6 +487,7 @@ pub fn buildLibzstd( const ZlibDecompressor = enum { zig_stdlib, libdeflate, + libz, }; const XzDecompressor = enum { diff --git a/build.zig.zon b/build.zig.zon index 6c62b15..672e679 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -1,6 +1,6 @@ .{ .name = "squashfuse-zig", - .version = "0.3.1", + .version = "0.3.2", .paths = [][]const u8{ "build.zig", diff --git a/lib/compression.zig b/lib/compression.zig index c5d23a2..8e9d03e 100644 --- a/lib/compression.zig +++ b/lib/compression.zig @@ -98,24 +98,25 @@ pub const Decompressor = *const fn ( out: []u8, ) DecompressError!usize; -pub fn builtWithDecompression(comptime compression: Compression) bool { - if (compression == .none) return true; - - const decl_name = "static_" ++ @tagName(compression); - return @hasDecl(build_options, decl_name) and @field(build_options, decl_name); -} - pub fn getDecompressor(kind: Compression) SquashFsError!Decompressor { switch (kind) { .zlib => { - if (build_options.zlib_decompressor == .zig_stdlib) { - return @import("compression/zlib_zig.zig").decode; - } - const libdeflate = @import("compression/zlib_libdeflate.zig"); - initDecompressionSymbol(libdeflate, .zlib, "deflate") catch return error.Error; + const libz = @import("compression/zlib_libz.zig"); + + return switch (build_options.zlib_decompressor) { + .zig_stdlib => @import("compression/zlib_zig.zig").decode, + .libdeflate => blk: { + initDecompressionSymbol(libdeflate, .zlib, "libdeflate") catch return error.Error; - return libdeflate.decode; + break :blk libdeflate.decode; + }, + .libz => blk: { + initDecompressionSymbol(libz, .zlib, "libz") catch return error.Error; + + break :blk libz.decode; + }, + }; }, .lzma, .lzo => return error.InvalidCompression, .xz => { @@ -124,13 +125,13 @@ pub fn getDecompressor(kind: Compression) SquashFsError!Decompressor { } const libxz = @import("compression/xz_liblzma.zig"); - initDecompressionSymbol(libxz, .xz, "lzma") catch return error.Error; + initDecompressionSymbol(libxz, .xz, "liblzma") catch return error.Error; return libxz.decode; }, .lz4 => { const liblz4 = @import("compression/lz4_liblz4.zig"); - initDecompressionSymbol(liblz4, .lz4, "lz4") catch return error.Error; + initDecompressionSymbol(liblz4, .lz4, "liblz4") catch return error.Error; return liblz4.decode; }, @@ -140,7 +141,7 @@ pub fn getDecompressor(kind: Compression) SquashFsError!Decompressor { } const libzstd = @import("compression/zstd_libzstd.zig"); - initDecompressionSymbol(libzstd, .zstd, "zstd") catch return error.Error; + initDecompressionSymbol(libzstd, .zstd, "libzstd") catch return error.Error; return libzstd.decode; }, @@ -151,29 +152,48 @@ pub fn getDecompressor(kind: Compression) SquashFsError!Decompressor { return error.InvalidCompression; } +pub fn builtWithDecompression(comptime compression: Compression) bool { + if (compression == .none) return true; + + const decl_name = "static_" ++ @tagName(compression); + + return @hasDecl(build_options, decl_name) and @field(build_options, decl_name); +} + +// Initializes decompression function pointers for C-ABI compression libs +// If `static_[DECOMPRESSOR]` is not given, an attempt will be made to +// dlload the library from the system fn initDecompressionSymbol( comptime T: type, comptime compression: Compression, comptime library_name: []const u8, ) !void { - if (builtWithDecompression(compression)) { - T.lib_decode = @extern( - *const T.LibDecodeFn, - .{ .name = T.lib_decode_name }, - ); + if (comptime builtWithDecompression(compression)) { + inline for (@typeInfo(T.required_symbols).Struct.decls) |decl| { + @field(T.required_symbols, decl.name) = @extern( + @TypeOf(@field(T.required_symbols, decl.name)), + .{ .name = decl.name }, + ); + } return; } + var found = false; + // Looks like it wasn't statically linked, attempt to find the library on // the system inline for (.{ - "/lib/lib{s}.so.1", - "/lib64/lib{s}.so.1", - "/usr/lib/lib{s}.so.1", - "/usr/lib64/lib{s}.so.1", + "/lib/{s}.so.1", + "/lib64/{s}.so.1", + "/usr/lib/{s}.so.1", + "/usr/lib64/{s}.so.1", }) |fmt| { - const path = std.fmt.comptimePrint(fmt, .{library_name}); + const path = std.fmt.comptimePrint( + fmt, + .{library_name}, + ); + // TODO: close libs var lib = DynLib.open(path) catch |err| { switch (err) { @@ -182,11 +202,18 @@ fn initDecompressionSymbol( } }; - T.lib_decode = lib.lookup( - *const T.LibDecodeFn, - T.lib_decode_name, - ) orelse return error.SymbolNotFound; + inline for (@typeInfo(T.required_symbols).Struct.decls) |decl| { + @field(T.required_symbols, decl.name) = lib.lookup( + @TypeOf(@field(T.required_symbols, decl.name)), + decl.name, + ) orelse return error.SymbolNotFound; + } + + found = true; + break; } + + if (!found) return error.DynLibNotFound; } // Since the decompressor will never be called on uncompressed blocks, diff --git a/lib/compression/lz4_liblz4.zig b/lib/compression/lz4_liblz4.zig index b1e8e62..1f87eb2 100644 --- a/lib/compression/lz4_liblz4.zig +++ b/lib/compression/lz4_liblz4.zig @@ -2,17 +2,14 @@ const std = @import("std"); const compression = @import("../compression.zig"); const DecompressError = compression.DecompressError; -pub const LibDecodeFn = fn ( - [*]const u8, - [*]u8, - c_int, - c_int, -) callconv(.C) c_int; - -pub const lib_decode_name = "LZ4_decompress_safe"; - -// Initialized in `compression.zig` -pub var lib_decode: *const LibDecodeFn = undefined; +pub const required_symbols = struct { + pub var LZ4_decompress_safe: *const fn ( + [*]const u8, + [*]u8, + c_int, + c_int, + ) callconv(.C) c_int = undefined; +}; pub fn decode( allocator: std.mem.Allocator, @@ -21,7 +18,7 @@ pub fn decode( ) DecompressError!usize { _ = allocator; - const ret = lib_decode( + const ret = required_symbols.LZ4_decompress_safe( in.ptr, out.ptr, @intCast(in.len), diff --git a/lib/compression/xz_liblzma.zig b/lib/compression/xz_liblzma.zig index 275abda..c8c35a2 100644 --- a/lib/compression/xz_liblzma.zig +++ b/lib/compression/xz_liblzma.zig @@ -3,23 +3,20 @@ const io = std.io; const compression = @import("../compression.zig"); const DecompressError = compression.DecompressError; -pub const lib_decode_name = "lzma_stream_buffer_decode"; - -// Initialized in `compression.zig` -pub var lib_decode: *const LibDecodeFn = undefined; - -pub const LibDecodeFn = fn ( - memlemit: *u64, - flags: u32, - // TODO: set allocator - allocator: ?*anyopaque, - in: [*]const u8, - in_pos: *usize, - in_size: usize, - out: [*]u8, - out_pos: *usize, - out_size: usize, -) callconv(.C) c_int; +pub const required_symbols = struct { + pub var lzma_stream_buffer_decode: *const fn ( + memlemit: *u64, + flags: u32, + // TODO: set allocator + allocator: ?*anyopaque, + in: [*]const u8, + in_pos: *usize, + in_size: usize, + out: [*]u8, + out_pos: *usize, + out_size: usize, + ) callconv(.C) c_int = undefined; +}; pub fn decode( allocator: std.mem.Allocator, @@ -33,7 +30,7 @@ pub fn decode( var inpos: usize = 0; var outpos: usize = 0; - const err = lib_decode( + const err = required_symbols.lzma_stream_buffer_decode( &memlimit, 0, null, diff --git a/lib/compression/zlib_libdeflate.zig b/lib/compression/zlib_libdeflate.zig index 8f2b4ae..2e07421 100644 --- a/lib/compression/zlib_libdeflate.zig +++ b/lib/compression/zlib_libdeflate.zig @@ -2,19 +2,47 @@ const std = @import("std"); const compression = @import("../compression.zig"); const DecompressError = compression.DecompressError; -pub const LibDecodeFn = fn ( - *anyopaque, - [*]const u8, - usize, - [*]u8, - usize, - *usize, -) callconv(.C) c_int; +pub const required_symbols = struct { + pub var libdeflate_zlib_decompress: *const fn ( + *anyopaque, + [*]const u8, + usize, + [*]u8, + usize, + *usize, + ) callconv(.C) c_int = undefined; +}; + +pub fn decode( + allocator: std.mem.Allocator, + in: []const u8, + out: []u8, +) DecompressError!usize { + _ = allocator; + + var decompressor = Decompressor{}; + + var written: usize = undefined; + + const err = required_symbols.libdeflate_zlib_decompress( + &decompressor, + in.ptr, + in.len, + out.ptr, + out.len, + &written, + ); -pub const lib_decode_name = "libdeflate_zlib_decompress"; + return switch (err) { + // Defined in + 0 => written, -// Initialized in `compression.zig` -pub var lib_decode: *const LibDecodeFn = undefined; + 1 => DecompressError.CorruptInput, + 2 => DecompressError.ShortOutput, + 3 => DecompressError.NoSpaceLeft, + else => DecompressError.Error, + }; +} // Deflate constants const litlen_syms = 288; @@ -46,34 +74,3 @@ const Decompressor = extern struct { litlen_tablebits: u32 = undefined, free_func: ?*anyopaque = undefined, }; - -pub fn decode( - allocator: std.mem.Allocator, - in: []const u8, - out: []u8, -) DecompressError!usize { - _ = allocator; - - var decompressor = Decompressor{}; - - var written: usize = undefined; - - const err = lib_decode( - &decompressor, - in.ptr, - in.len, - out.ptr, - out.len, - &written, - ); - - return switch (err) { - // Defined in - 0 => written, - - 1 => DecompressError.CorruptInput, - 2 => DecompressError.ShortOutput, - 3 => DecompressError.NoSpaceLeft, - else => DecompressError.Error, - }; -} diff --git a/lib/compression/zlib_libz.zig b/lib/compression/zlib_libz.zig new file mode 100644 index 0000000..d6ad1ad --- /dev/null +++ b/lib/compression/zlib_libz.zig @@ -0,0 +1,51 @@ +const std = @import("std"); +const compression = @import("../compression.zig"); +const DecompressError = compression.DecompressError; + +pub const required_symbols = struct { + pub var uncompress: *const fn ( + [*]u8, + *c_ulong, + [*]const u8, + c_ulong, + ) callconv(.C) c_int = undefined; +}; + +pub fn decode( + allocator: std.mem.Allocator, + in: []const u8, + out: []u8, +) DecompressError!usize { + _ = allocator; + + var written: c_ulong = @intCast(out.len); + + const err = required_symbols.uncompress( + out.ptr, + &written, + in.ptr, + in.len, + ); + + // TODO: audit these and ensure they're correct + if (err < 0) return switch (err) { + // Z_STREAM_ERROR + -2 => DecompressError.ShortOutput, + + // Z_DATA_ERROR + -3 => DecompressError.CorruptInput, + + // Z_MEM_ERROR + -4 => DecompressError.OutOfMemory, + + // Z_BUF_ERROR + -5 => DecompressError.NoSpaceLeft, + + // Z_ERRNO, Z_VERSION_ERROR + -1, -6 => DecompressError.Error, + + else => DecompressError.Error, + }; + + return written; +} diff --git a/lib/compression/zstd_libzstd.zig b/lib/compression/zstd_libzstd.zig index 13468fb..6d470af 100644 --- a/lib/compression/zstd_libzstd.zig +++ b/lib/compression/zstd_libzstd.zig @@ -2,19 +2,18 @@ const std = @import("std"); const compression = @import("../compression.zig"); const DecompressError = compression.DecompressError; -pub const LibDecodeFn = fn ( - [*]u8, - usize, - [*]const u8, - usize, -) callconv(.C) usize; - -pub const lib_decode_name = "ZSTD_decompress"; - -// Initialized in `compression.zig` -pub var lib_decode: *const LibDecodeFn = undefined; - -extern fn ZSTD_getErrorCode(usize) usize; +pub const required_symbols = struct { + pub var ZSTD_decompress: *const fn ( + [*]u8, + usize, + [*]const u8, + usize, + ) callconv(.C) usize = undefined; + + pub var ZSTD_getErrorCode: *const fn ( + usize, + ) callconv(.C) usize = undefined; +}; pub fn decode( allocator: std.mem.Allocator, @@ -23,14 +22,14 @@ pub fn decode( ) DecompressError!usize { _ = allocator; - const ret = lib_decode( + const ret = required_symbols.ZSTD_decompress( out.ptr, out.len, in.ptr, in.len, ); - return switch (ZSTD_getErrorCode(ret)) { + return switch (required_symbols.ZSTD_getErrorCode(ret)) { 0 => ret, 20 => error.CorruptInput,