Skip to content

Commit

Permalink
Update test_out_s3.rb
Browse files Browse the repository at this point in the history
  • Loading branch information
ddukbg authored Nov 6, 2024
1 parent 42028c7 commit aaab4a5
Showing 1 changed file with 12 additions and 35 deletions.
47 changes: 12 additions & 35 deletions test/test_out_s3.rb
Original file line number Diff line number Diff line change
Expand Up @@ -471,49 +471,26 @@ def test_write_with_custom_s3_object_key_format_containing_hex_random_placeholde
def test_write_with_zstd
setup_mocks(true)
s3_local_file_path = "/tmp/s3-test.zst"

expected_s3path = "log/events/ts=20110102-13/events_0-#{Socket.gethostname}.zst"

setup_s3_object_mocks(
s3_local_file_path: s3_local_file_path,
s3path: expected_s3path
)


setup_s3_object_mocks(s3_local_file_path: s3_local_file_path, s3path: expected_s3path)

config = CONFIG_TIME_SLICE + "\nstore_as zstd\n"
d = create_time_sliced_driver(config)

# Get compressor instance for debugging
compressor = d.instance.instance_variable_get(:@compressor)
puts "Compressor class: #{compressor.class}"
puts "Compressor config: #{compressor.instance_variable_get(:@compress_config).inspect}"


time = event_time("2011-01-02 13:14:15 UTC")
d.run(default_tag: "test") do
d.feed(time, { "a" => 1 })
d.feed(time, { "a" => 2 })
end

puts "Checking file: #{s3_local_file_path}"
puts "File exists?: #{File.exist?(s3_local_file_path)}"
if File.exist?(s3_local_file_path)
puts "File size: #{File.size(s3_local_file_path)}"
File.open(s3_local_file_path, 'rb') do |file|
compressed_data = file.read
puts "Read data size: #{compressed_data.bytesize}"
puts "First few bytes: #{compressed_data[0..10].bytes.map { |b| sprintf('%02x', b) }.join(' ')}"

begin
uncompressed_data = Zstd.decompress(compressed_data)
puts "Successfully decompressed data: #{uncompressed_data[0..50]}"
expected_data = %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
%[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
assert_equal expected_data, uncompressed_data
rescue => e
puts "Decompression error: #{e.class} - #{e.message}"
puts "Backtrace:\n#{e.backtrace.join("\n")}"
raise
end
end

File.open(s3_local_file_path, 'rb') do |file|
compressed_data = file.read
uncompressed_data = Zstd.decompress(compressed_data)
expected_data = %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
%[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
assert_equal expected_data, uncompressed_data
end
FileUtils.rm_f(s3_local_file_path)
end
Expand Down

0 comments on commit aaab4a5

Please sign in to comment.