Merge branch 'master' into fix-fuzz-2024-07-17

Signed-off-by: Chris Hennick <[email protected]>

fuzz/fuzz_targets/fuzz_write.rs

@@ -4,9 +4,8 @@ use arbitrary::Arbitrary;
 use core::fmt::{Debug};
 use libfuzzer_sys::fuzz_target;
 use replace_with::replace_with_or_abort;
-use std::borrow::Cow;
 use std::fmt::{Arguments, Formatter, Write};
-use std::io::Cursor;
+use std::io::{Cursor, Seek, SeekFrom};
 use std::io::Write as IoWrite;
 use std::path::PathBuf;
 use tikv_jemallocator::Jemalloc;
@@ -31,6 +30,7 @@ pub enum BasicFileOperation<'k> {
     ShallowCopy(Box<FileOperation<'k>>),
     DeepCopy(Box<FileOperation<'k>>),
     MergeWithOtherFile {
+        initial_junk: Box<[u8]>,
         operations: Box<[(FileOperation<'k>, bool)]>,
     },
     SetArchiveComment(Box<[u8]>),
@@ -52,26 +52,27 @@ pub struct FileOperation<'k> {
 }
 
 impl<'k> FileOperation<'k> {
-    fn get_path(&self) -> Option<Cow<PathBuf>> {
+    fn get_path(&self) -> Option<PathBuf> {
         match &self.basic {
             BasicFileOperation::SetArchiveComment(_) => None,
-            BasicFileOperation::WriteDirectory(_) => Some(Cow::Owned(self.path.join("/"))),
-            BasicFileOperation::MergeWithOtherFile { operations } => operations
+            BasicFileOperation::WriteDirectory(_) => Some(self.path.join("/")),
+            BasicFileOperation::MergeWithOtherFile { operations, .. } => operations
                 .iter()
                 .flat_map(|(op, abort)| if !abort { op.get_path() } else { None })
                 .next(),
-            _ => Some(Cow::Borrowed(&self.path)),
+            _ => Some(self.path.to_owned()),
         }
     }
 }
 
 #[derive(Arbitrary, Clone)]
 pub struct FuzzTestCase<'k> {
+    initial_junk: Box<[u8]>,
     operations: Box<[(FileOperation<'k>, bool)]>,
     flush_on_finish_file: bool,
 }
 
-fn deduplicate_paths(copy: &mut Cow<PathBuf>, original: &PathBuf) {
+fn deduplicate_paths(copy: &mut PathBuf, original: &PathBuf) {
     if path_to_string(&**copy) == path_to_string(original) {
         let new_path = match original.file_name() {
             Some(name) => {
@@ -81,27 +82,26 @@ fn deduplicate_paths(copy: &mut Cow<PathBuf>, original: &PathBuf) {
             }
             None => copy.with_file_name("copy"),
         };
-        *copy = Cow::Owned(new_path);
+        *copy = new_path;
     }
 }
 
 fn do_operation<'k>(
     writer: &mut zip::ZipWriter<Cursor<Vec<u8>>>,
-    operation: &FileOperation<'k>,
+    operation: FileOperation<'k>,
     abort: bool,
     flush_on_finish_file: bool,
     files_added: &mut usize,
     stringifier: &mut impl Write,
     panic_on_error: bool
 ) -> Result<(), Box<dyn std::error::Error>> {
     writer.set_flush_on_finish_file(flush_on_finish_file);
-    let mut path = Cow::Borrowed(&operation.path);
-    match &operation.basic {
+    let FileOperation { basic, mut path, reopen} = operation;
+    match basic {
         BasicFileOperation::WriteNormalFile {
-            contents, options, ..
+            contents, mut options, ..
         } => {
             let uncompressed_size = contents.iter().map(|chunk| chunk.len()).sum::<usize>();
-            let mut options = (*options).to_owned();
             if uncompressed_size >= u32::MAX as usize {
                 options = options.large_file(true);
             }
@@ -132,7 +132,7 @@ fn do_operation<'k>(
                 return Ok(());
             };
             deduplicate_paths(&mut path, &base_path);
-            do_operation(writer, &base, false, flush_on_finish_file, files_added, stringifier, panic_on_error)?;
+            do_operation(writer, *base, false, flush_on_finish_file, files_added, stringifier, panic_on_error)?;
             writeln!(stringifier, "writer.shallow_copy_file_from_path({:?}, {:?});", base_path, path)?;
             writer.shallow_copy_file_from_path(&*base_path, &*path)?;
             *files_added += 1;
@@ -142,21 +142,31 @@ fn do_operation<'k>(
                 return Ok(());
             };
             deduplicate_paths(&mut path, &base_path);
-            do_operation(writer, &base, false, flush_on_finish_file, files_added, stringifier, panic_on_error)?;
+            do_operation(writer, *base, false, flush_on_finish_file, files_added, stringifier, panic_on_error)?;
             writeln!(stringifier, "writer.deep_copy_file_from_path({:?}, {:?});", base_path, path)?;
-            writer.deep_copy_file_from_path(&*base_path, &*path)?;
+            writer.deep_copy_file_from_path(&*base_path, path)?;
             *files_added += 1;
         }
-        BasicFileOperation::MergeWithOtherFile { operations } => {
-            let mut other_writer = zip::ZipWriter::new(Cursor::new(Vec::new()));
+        BasicFileOperation::MergeWithOtherFile { operations, initial_junk } => {
+            if initial_junk.is_empty() {
+                writeln!(stringifier, "let sub_writer = {{\n\
+                     let mut writer = ZipWriter::new(Cursor::new(Vec::new()));")?;
+            } else {
+                writeln!(stringifier,
+                         "let sub_writer = {{\n\
+                          let mut initial_junk = Cursor::new(vec!{:?});\n\
+                          initial_junk.seek(SeekFrom::End(0))?;
+                          let mut writer = ZipWriter::new(initial_junk);", initial_junk)?;
+            }
+            let mut initial_junk = Cursor::new(initial_junk.into_vec());
+            initial_junk.seek(SeekFrom::End(0))?;
+            let mut other_writer = zip::ZipWriter::new(initial_junk);
             let mut inner_files_added = 0;
-            writeln!(stringifier,
-                "let sub_writer = {{\nlet mut writer = ZipWriter::new(Cursor::new(Vec::new()));")?;
-            operations.iter().for_each(|(operation, abort)| {
+            operations.into_vec().into_iter().for_each(|(operation, abort)| {
                 let _ = do_operation(
                     &mut other_writer,
-                    &operation,
-                    *abort,
+                    operation,
+                    abort,
                     false,
                     &mut inner_files_added,
                     stringifier,
@@ -180,7 +190,7 @@ fn do_operation<'k>(
     // If a comment is set, we finish the archive, reopen it for append and then set a shorter
     // comment, then there will be junk after the new comment that we can't get rid of. Thus, we
     // can only check that the expected is a prefix of the actual
-    match operation.reopen {
+    match reopen {
         ReopenOption::DoNotReopen => {
             writeln!(stringifier, "writer")?;
             return Ok(())
@@ -222,9 +232,11 @@ fn do_operation<'k>(
 }
 
 impl <'k> FuzzTestCase<'k> {
-    fn execute(&self, stringifier: &mut impl Write, panic_on_error: bool) -> ZipResult<()> {
+    fn execute(self, stringifier: &mut impl Write, panic_on_error: bool) -> ZipResult<()> {
+        let mut initial_junk = Cursor::new(self.initial_junk.into_vec());
+        initial_junk.seek(SeekFrom::End(0))?;
+        let mut writer = zip::ZipWriter::new(initial_junk);
         let mut files_added = 0;
-        let mut writer = zip::ZipWriter::new(Cursor::new(Vec::new()));
         let mut final_reopen = false;
         if let Some((last_op, _)) = self.operations.last() {
             if last_op.reopen != ReopenOption::ViaFinishIntoReadable {
@@ -233,11 +245,11 @@ impl <'k> FuzzTestCase<'k> {
         }
         #[allow(unknown_lints)]
         #[allow(boxed_slice_into_iter)]
-        for (operation, abort) in self.operations.iter() {
+        for (operation, abort) in self.operations.into_vec().into_iter() {
             let _ = do_operation(
                 &mut writer,
-                &operation,
-                *abort,
+                operation,
+                abort,
                 self.flush_on_finish_file,
                 &mut files_added,
                 stringifier,
@@ -255,8 +267,14 @@ impl <'k> FuzzTestCase<'k> {
 
 impl <'k> Debug for FuzzTestCase<'k> {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        writeln!(f, "let mut writer = ZipWriter::new(Cursor::new(Vec::new()));")?;
-        let _ = self.execute(f, false);
+        if self.initial_junk.is_empty() {
+            writeln!(f, "let mut writer = ZipWriter::new(Cursor::new(Vec::new()));")?;
+        } else {
+            writeln!(f, "let mut initial_junk = Cursor::new(vec!{:?});\n\
+                         initial_junk.seek(SeekFrom::End(0))?;\n\
+                         let mut writer = ZipWriter::new(initial_junk);", &self.initial_junk)?;
+        }
+        let _ = self.clone().execute(f, false);
         Ok(())
     }
 }

src/write.rs

@@ -992,6 +992,8 @@ impl<W: Write + Seek> ZipWriter<W> {
                     buffer: vec![],
                     keys,
                 };
+                self.stats.start = zipwriter.writer.stream_position()?;
+                // crypto_header is counted as part of the data
                 let crypto_header = [0u8; 12];
                 let result = zipwriter.write_all(&crypto_header);
                 self.ok_or_abort_file(result)?;