From 904a043b8d0228bcf6010c466da201833e801da5 Mon Sep 17 00:00:00 2001 From: ddukbg Date: Sat, 2 Nov 2024 20:33:52 +0900 Subject: [PATCH] fix: Add proper compression level handling to ZstdCompressor Signed-off-by: ddukbg --- lib/fluent/plugin/out_s3.rb | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/lib/fluent/plugin/out_s3.rb b/lib/fluent/plugin/out_s3.rb index 9149a0b..f0e0701 100644 --- a/lib/fluent/plugin/out_s3.rb +++ b/lib/fluent/plugin/out_s3.rb @@ -632,20 +632,32 @@ def compress(chunk, tmp) end class ZstdCompressor < Compressor + require 'zstd-ruby' + + DEFAULT_LEVEL = 3 + + def initialize(options = {}) + @level = (options.is_a?(Hash) && options[:level]) || DEFAULT_LEVEL + end + def ext 'zst'.freeze end - + def content_type 'application/x-zst'.freeze end - + def compress(chunk, tmp) - compressed_data = Zstd.compress(chunk.read, level: @level) - tmp.write(compressed_data) - rescue => e - log.warn "zstd compression failed: #{e.message}" - raise e + begin + data = chunk.read.gsub(/\r\n/, "\n").force_encoding('UTF-8') + compressed = Zstd.compress(data, level: @level) + tmp.binmode + tmp.write(compressed) + rescue => e + log.warn "zstd compression failed: #{e.message}" + raise e + end end end