download_sysext: fix issues of reading into buffer

Fix bugs when reading from File to buffer. We need to first create a
BufReader for reading from the buffer, pass that into parsing functions.
That would make the code much easier to maintain, instead of passing
File itself. Then we can read data without having to first open the file
and track read positions.

We need to introduce get_header_data() to read header and data, setting
the begining of the stream including the whole data including delta
update header as well as manifest. Doing that, signature verification
works well.

Also introduce get_data_blob() to read only data without header,
manifest.

Cargo.toml

@@ -17,6 +17,7 @@ log = "0.4"
 argh = "0.1"
 globset = "0.4"
 protobuf = "3.2.0"
+bzip2 = "0.4.4"
 
 [dependencies.hard-xml]
 path = "vendor/hard-xml"

src/bin/download_sysext.rs

@@ -4,6 +4,7 @@ use std::path::Path;
 use std::fs::File;
 use std::fs;
 use std::io;
+use std::io::BufReader;
 
 #[macro_use]
 extern crate log;
@@ -49,6 +50,20 @@ impl<'a> Package<'a> {
         ))
     }
 
+    #[rustfmt::skip]
+    fn hash_on_mem(&mut self, buf: &[u8]) -> Result<omaha::Hash<omaha::Sha256>, Box<dyn Error>> {
+        use sha2::{Sha256, Digest};
+
+        let mut hasher = Sha256::new();
+        let mut bufreader = BufReader::new(buf);
+
+        io::copy(&mut bufreader, &mut hasher)?;
+
+        Ok(omaha::Hash::from_bytes(
+            hasher.finalize().into()
+        ))
+    }
+
     #[rustfmt::skip]
     fn check_download(&mut self, in_dir: &Path) -> Result<(), Box<dyn Error>> {
         let path = in_dir.join(&*self.name);
@@ -113,20 +128,36 @@ impl<'a> Package<'a> {
         }
     }
 
-    fn verify_signature_on_disk(&mut self, from_path: &Path, pubkey_path: &str) -> Result<(), Box<dyn Error>> {
+    fn verify_signature_on_disk(&mut self, from_path: &Path, pubkey_path: &str) -> Result<Vec<u8>, Box<dyn Error>> {
         let upfile = File::open(from_path)?;
 
+        // create a BufReader to pass down to parsing functions.
+        let upfreader = &mut BufReader::new(upfile);
+
         // Read update payload from file, read delta update header from the payload.
-        let res_data = fs::read_to_string(from_path);
+        let header = delta_update::read_delta_update_header(upfreader)?;
 
-        let header = delta_update::read_delta_update_header(&upfile)?;
+        let mut delta_archive_manifest = delta_update::get_manifest_bytes(upfreader, &header)?;
 
         // Extract signature from header.
-        let sigbytes = delta_update::get_signatures_bytes(&upfile, &header)?;
+        let sigbytes = delta_update::get_signatures_bytes(upfreader, &header, &mut delta_archive_manifest)?;
+
+        // Extract header and data blobs together.
+        let header_data = delta_update::get_header_data(upfreader, &header, &delta_archive_manifest)?;
 
-        // Parse signature data from the signature containing data, version, special fields.
-        let _sigdata = match delta_update::parse_signature_data(res_data.unwrap().as_bytes(), &sigbytes, pubkey_path) {
-            Some(data) => data,
+        // Extract data blobs
+        let datablobs = delta_update::get_data_blobs(upfreader, &header, &delta_archive_manifest)?;
+
+        // Check for hash of data blobs with new_partition_info hash.
+        let pinfo_hash = delta_archive_manifest.new_partition_info.hash.clone().unwrap_or_default();
+        let datahash = self.hash_on_mem(datablobs.as_slice())?;
+        if datahash != omaha::Hash::from_bytes(pinfo_hash.as_slice()[..].try_into().unwrap_or_default()) {
+            return Err("data hash mismatch with new_partition_info hash".into());
+        }
+
+        // Parse signature data from sig blobs, data blobs, public key.
+        match delta_update::parse_signature_data(&sigbytes, header_data.as_slice(), pubkey_path) {
+            Some(_) => (),
             _ => {
                 self.status = PackageStatus::BadSignature;
                 return Err("unable to parse signature data".into());
@@ -136,7 +167,7 @@ impl<'a> Package<'a> {
         println!("Parsed and verified signature data from file {:?}", from_path);
 
         self.status = PackageStatus::Verified;
-        Ok(())
+        Ok(datablobs)
     }
 }
 
@@ -265,12 +296,13 @@ async fn main() -> Result<(), Box<dyn Error>> {
         pkg.download(&unverified_dir, &client).await?;
 
         let pkg_unverified = unverified_dir.join(&*pkg.name);
-        let pkg_verified = output_dir.join(&*pkg.name);
+        let pkg_verified = output_dir.join(pkg_unverified.with_extension("raw").file_name().unwrap_or_default());
 
         match pkg.verify_signature_on_disk(&pkg_unverified, &args.pubkey_file) {
-            Ok(_) => {
-                // move the verified file back from unverified_dir to output_dir
-                fs::rename(&pkg_unverified, &pkg_verified)?;
+            Ok(datablobs) => {
+                // write extracted data into the final data.
+                fs::write(pkg_verified.clone(), &datablobs)?;
+                debug!("data blobs written into file {:?}", pkg_verified);
             }
             _ => return Err(format!("unable to verify signature \"{}\"", pkg.name).into()),
         };

test/crau_verify.rs

@@ -1,8 +1,8 @@
-use std::io::Write;
+use std::io::{BufReader, Write};
 use std::error::Error;
 use std::fs;
 
-use update_format_crau::delta_update;
+use update_format_crau::{delta_update, proto};
 
 use argh::FromArgs;
 
@@ -28,15 +28,20 @@ fn main() -> Result<(), Box<dyn Error>> {
 
     // Read update payload from srcpath, read delta update header from the payload.
     let upfile = fs::File::open(srcpath.clone())?;
-    let header = delta_update::read_delta_update_header(&upfile)?;
+
+    let freader = &mut BufReader::new(upfile);
+    let header = delta_update::read_delta_update_header(freader)?;
+
+    let mut delta_archive_manifest: proto::DeltaArchiveManifest = Default::default();
 
     // Extract signature from header.
-    let sigbytes = delta_update::get_signatures_bytes(&upfile, &header)?;
+    let sigbytes = delta_update::get_signatures_bytes(freader, &header, &mut delta_archive_manifest)?;
 
-    const TESTDATA: &str = "test data for verifying signature";
+    // Parse signature data from the signature containing data, version, special fields.
+    let header_data = delta_update::get_header_data(freader, &header, &delta_archive_manifest)?;
 
     // Parse signature data from the signature containing data, version, special fields.
-    let sigdata = match delta_update::parse_signature_data(TESTDATA.as_bytes(), &sigbytes, PUBKEY_FILE) {
+    let sigdata = match delta_update::parse_signature_data(&sigbytes, header_data.as_slice(), PUBKEY_FILE) {
         Some(data) => Box::leak(data),
         _ => return Err("unable to parse signature data".into()),
     };

update-format-crau/Cargo.toml

@@ -6,6 +6,7 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+bzip2 = "0.4.4"
 log = "0.4.19"
 protobuf = "3"
 rsa = { version = "0.9.2", features = ["sha2"] }

update-format-crau/src/delta_update.rs

@@ -1,7 +1,8 @@
-use std::io::{Read, Seek, SeekFrom};
+use std::io::{BufReader, Read, Seek, SeekFrom};
 use std::error::Error;
 use std::fs::File;
 use log::{error, debug};
+use bzip2::read::BzDecoder;
 
 use protobuf::Message;
 
@@ -29,7 +30,7 @@ impl DeltaUpdateFileHeader {
 }
 
 // Read delta update header from the given file, return DeltaUpdateFileHeader.
-pub fn read_delta_update_header(mut f: &File) -> Result<DeltaUpdateFileHeader, Box<dyn Error>> {
+pub fn read_delta_update_header(f: &mut BufReader<File>) -> Result<DeltaUpdateFileHeader, Box<dyn Error>> {
     let mut header = DeltaUpdateFileHeader {
         magic: [0; 4],
         file_format_version: 0,
@@ -54,17 +55,23 @@ pub fn read_delta_update_header(mut f: &File) -> Result<DeltaUpdateFileHeader, B
     Ok(header)
 }
 
-// Take a file stream and DeltaUpdateFileHeader,
-// return a bytes slice of the actual signature data as well as its length.
-pub fn get_signatures_bytes<'a>(mut f: &'a File, header: &'a DeltaUpdateFileHeader) -> Result<Box<[u8]>, Box<dyn Error>> {
+// Take a buffer stream and DeltaUpdateFileHeader,
+// return DeltaArchiveManifest that contains manifest.
+pub fn get_manifest_bytes(f: &mut BufReader<File>, header: &DeltaUpdateFileHeader) -> Result<proto::DeltaArchiveManifest, Box<dyn Error>> {
     let manifest_bytes = {
         let mut buf = vec![0u8; header.manifest_size as usize];
         f.read_exact(&mut buf)?;
         buf.into_boxed_slice()
     };
 
-    let manifest = proto::DeltaArchiveManifest::parse_from_bytes(&manifest_bytes)?;
+    let delta_archive_manifest = proto::DeltaArchiveManifest::parse_from_bytes(&manifest_bytes)?;
+
+    Ok(delta_archive_manifest)
+}
 
+// Take a buffer stream and DeltaUpdateFileHeader,
+// return a bytes slice of the actual signature data as well as its length.
+pub fn get_signatures_bytes<'a>(f: &'a mut BufReader<File>, header: &'a DeltaUpdateFileHeader, manifest: &mut proto::DeltaArchiveManifest) -> Result<Box<[u8]>, Box<dyn Error>> {
     // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
     // !!! signature offsets are from the END of the manifest !!!
     // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@@ -85,10 +92,64 @@ pub fn get_signatures_bytes<'a>(mut f: &'a File, header: &'a DeltaUpdateFileHead
     Ok(signatures_bytes.unwrap())
 }
 
+// Take a buffer reader, delta file header, manifest as input.
+// Return the whole data blobs, including header, manifest.
+pub fn get_header_data<'a>(f: &'a mut BufReader<File>, header: &'a DeltaUpdateFileHeader, manifest: &proto::DeltaArchiveManifest) -> Result<Vec<u8>, Box<dyn Error>> {
+    // Reset to the beginning of the stream, which means the whole buffer including
+    // delta update header as well as manifest. That is because data that must be verified
+    // with signatures start from the beginning. No header.translate_offset(0).
+    //
+    // Payload data structure:
+    //  | header | manifest | data blobs | signatures |
+
+    f.seek(SeekFrom::Start(0))?;
+
+    let data_length = header.translate_offset(manifest.signatures_offset.unwrap());
+
+    let mut header_data = vec![0u8; data_length as usize];
+    f.read_exact(&mut header_data)?;
+
+    Ok(header_data)
+}
+
+// Take a buffer reader, delta file header, manifest as input.
+// Return the data blobs, without header, manifest, or signatures.
+pub fn get_data_blobs<'a>(f: &'a mut BufReader<File>, header: &'a DeltaUpdateFileHeader, manifest: &proto::DeltaArchiveManifest) -> Result<Vec<u8>, Box<dyn Error>> {
+    // Read from the beginning of header, which means buffer including only data blobs.
+    // It means it is necessary to call header.translate_offset(), in contrast to
+    // get_header_data.
+    // Iterate each partition_operations to get data offset and data length.
+
+    let mut data_blobs = Vec::new();
+
+    for pop in &manifest.partition_operations {
+        let data_offset = pop.data_offset.unwrap();
+        let data_length = pop.data_length.unwrap();
+
+        let mut partdata = vec![0u8; data_length as usize];
+
+        f.seek(SeekFrom::Start(header.translate_offset(data_offset.into())))?;
+        f.read_exact(&mut partdata)?;
+
+        // In case of bzip2-compressed chunks, extract.
+        if pop.type_.unwrap() == proto::install_operation::Type::REPLACE_BZ.into() {
+            let mut bzdecoder = BzDecoder::new(&partdata[..]);
+            let mut partdata_unpacked = Vec::new();
+            bzdecoder.read_to_end(&mut partdata_unpacked)?;
+
+            data_blobs.append(&mut partdata_unpacked);
+        } else {
+            data_blobs.append(&mut partdata);
+        }
+    }
+
+    Ok(data_blobs)
+}
+
 #[rustfmt::skip]
-// parse_signature_data takes a bytes slice for signature and public key file path.
+// parse_signature_data takes bytes slices for signature and data blobs and public key file path.
 // Return only actual data, without version and special fields.
-pub fn parse_signature_data(testdata: &[u8], sigbytes: &[u8], pubkeyfile: &str) -> Option<Box<[u8]>> {
+pub fn parse_signature_data(sigbytes: &[u8], databytes: &[u8], pubkeyfile: &str) -> Option<Box<[u8]>> {
     // Signatures has a container of the fields, i.e. version, data, and
     // special fields.
     let sigmessage = match proto::Signatures::parse_from_bytes(sigbytes) {
@@ -102,7 +163,7 @@ pub fn parse_signature_data(testdata: &[u8], sigbytes: &[u8], pubkeyfile: &str)
     // Return the first valid signature, iterate into the next slot if invalid.
     sigmessage.signatures.iter()
         .find_map(|sig|
-            verify_sig_pubkey(testdata, sig, pubkeyfile)
+            verify_sig_pubkey(databytes, sig, pubkeyfile)
             .map(Vec::into_boxed_slice))
 }