Cleanup of npk packing and tests (#998)

* Fix mount point removal when repacking a npk during test

Repacking a npk during a test must remove the mountpoints before
actually calling mksquashfs. The path calculation must start from root
instead of the tmpdir.

* Add none compression option

mksquashfs allow to store data uncompressed.

* Serialize manifest directly into zip

The manfest can be serialized directly in to the npk/zip container
instead of dumping into memory first.

* Remove overkill builder pattern for npk creation

* Cleanup of npk packing API

* Move npk tests into northstar-runtime

Makefile.toml

@@ -36,11 +36,11 @@ args = [ "nextest", "run" ]
 
 [tasks.console]
 command = "cargo"
-args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "console" ]
+args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--compression", "none", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "console" ]
 
 [tasks.cpueater]
 command = "cargo"
-args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "cpueater" ]
+args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--compression", "none", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "cpueater" ]
 
 [tasks.crashing]
 command = "cargo"
@@ -51,73 +51,73 @@ script = [
     "cargo build --bin ferris",
     "ROOT=`mktemp -d`",
     "cp target/debug/ferris $ROOT",
-    "cargo run --bin northstar-sextant pack --out target/northstar/repository --key examples/northstar.key --manifest-path examples/ferris/manifest.yaml --root $ROOT"
+    "cargo run --bin northstar-sextant pack --compression none --out target/northstar/repository --key examples/northstar.key --manifest-path examples/ferris/manifest.yaml --root $ROOT"
 ]
 
 [tasks.hello-ferris]
 command = "cargo"
-args = [ "run", "--bin", "northstar-sextant", "pack", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "--manifest-path", "examples/hello-ferris/manifest.yaml" ]
+args = [ "run", "--bin", "northstar-sextant", "pack", "--compression", "none", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "--manifest-path", "examples/hello-ferris/manifest.yaml" ]
 
 [tasks.hello-resource]
 command = "cargo"
-args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "hello-resource" ]
+args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--compression", "none", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "hello-resource" ]
 
 [tasks.hello-world]
 command = "cargo"
-args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "hello-world" ]
+args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--compression", "none", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "hello-world" ]
 
 [tasks.inspect]
 command = "cargo"
-args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "inspect" ]
+args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--compression", "none", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "inspect" ]
 
 [tasks.memeater]
 command = "cargo"
-args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "memeater" ]
+args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--compression", "none", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "memeater" ]
 
 [tasks.message001]
 command = "cargo"
-args = [ "run", "--bin", "northstar-sextant", "pack", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "--manifest-path", "examples/message-0.0.1/manifest.yaml", "--root", "examples/message-0.0.1/root" ]
+args = [ "run", "--bin", "northstar-sextant", "pack", "--compression", "none", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "--manifest-path", "examples/message-0.0.1/manifest.yaml", "--root", "examples/message-0.0.1/root" ]
 
 [tasks.message002]
 command = "cargo"
-args = [ "run", "--bin", "northstar-sextant", "pack", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "--manifest-path", "examples/message-0.0.2/manifest.yaml", "--root", "examples/message-0.0.2/root" ]
+args = [ "run", "--bin", "northstar-sextant", "pack", "--compression", "none", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "--manifest-path", "examples/message-0.0.2/manifest.yaml", "--root", "examples/message-0.0.2/root" ]
 
 [tasks.netns]
 command = "cargo"
-args = [ "run", "--bin", "northstar-sextant", "pack", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "--manifest-path", "examples/netns/manifest.yaml" ]
+args = [ "run", "--bin", "northstar-sextant", "pack", "--compression", "none", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "--manifest-path", "examples/netns/manifest.yaml" ]
 
 [tasks.persistence]
 command = "cargo"
-args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "persistence" ]
+args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--compression", "none", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "persistence" ]
 
 [tasks.redis-client]
 command = "cargo"
-args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "redis-client" ]
+args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--compression", "none", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "redis-client" ]
 
 [tasks.redis-server]
 command = "cargo"
-args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "redis-server" ]
+args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--compression", "none", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "redis-server" ]
 
 [tasks.seccomp]
 command = "cargo"
-args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "seccomp" ]
+args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--compression", "none", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "seccomp" ]
 
 [tasks.sockets]
 command = "cargo"
-args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "sockets" ]
+args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--compression", "none", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "sockets" ]
 
 [tasks.test-container]
 command = "cargo"
-args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "test-container" ]
+args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--compression", "none", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "test-container" ]
 
 [tasks.test-resource]
 command = "cargo"
-args = [ "run", "--bin", "northstar-sextant", "pack", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "--manifest-path", "examples/test-resource/manifest.yaml", "--root", "examples/test-resource/root" ]
+args = [ "run", "--bin", "northstar-sextant", "pack", "--compression", "none", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "--manifest-path", "examples/test-resource/manifest.yaml", "--root", "examples/test-resource/root" ]
 
 [tasks.token-client]
 command = "cargo"
-args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "token-client" ]
+args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--compression", "none", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "token-client" ]
 
 [tasks.token-server]
 command = "cargo"
-args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "token-server" ]
+args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--compression", "none", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "token-server" ]

cargo-npk/src/cli.rs

@@ -35,6 +35,7 @@ pub enum ColorChoice {
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum)]
 pub enum Compression {
+    None,
     Gzip,
     Lzma,
     Lzo,
@@ -45,6 +46,7 @@ pub enum Compression {
 impl From<Compression> for NpkCompressionAlgorithm {
     fn from(c: Compression) -> Self {
         match c {
+            Compression::None => NpkCompressionAlgorithm::None,
             Compression::Gzip => NpkCompressionAlgorithm::Gzip,
             Compression::Lzma => NpkCompressionAlgorithm::Lzma,
             Compression::Lzo => NpkCompressionAlgorithm::Lzo,

cargo-npk/src/lib.rs

@@ -19,7 +19,7 @@ use std::path::Path;
 use anyhow::Result;
 use cargo_subcommand::{Profile, Subcommand};
 use clap::Parser;
-use northstar_runtime::npk::npk::{pack_with_manifest, SquashfsOptions};
+use northstar_runtime::npk::npk::{NpkBuilder, SquashfsOptions};
 
 use crate::metadata::Metadata;
 
@@ -156,6 +156,13 @@ fn pack(
         cmd.build_dir(target)
     };
 
+    let builder = NpkBuilder::default().root(&root, Some(&squashfs_opts));
+    let builder = if let Some(key) = key {
+        builder.key(key)
+    } else {
+        builder
+    };
+
     if let Some(clones) = clones {
         let name = northstar_manifest.name.clone();
         let num = clones.to_string().chars().count();
@@ -164,16 +171,16 @@ fn pack(
             manifest.name = format!("{name}-{n:0num$}")
                 .try_into()
                 .context("failed to parse name")?;
-            let npk = pack_with_manifest(&manifest, &root, &out, key, &squashfs_opts)?;
-            let npk_size = human_bytes(fs::metadata(&npk)?.len() as f64);
+            let (npk, npk_size) = builder.clone().manifest(&manifest).to_dir(&out)?;
+            let npk_size = human_bytes(npk_size as f64);
             let msg = format!("{} [{}, {}]", npk.display(), npk_size, compression);
             log("Packed", &msg)?;
         }
         let duration = format_duration(time::Duration::from_secs(start.elapsed().as_secs()));
         log("Finished", &format!("{clones} clones in {duration}"))?;
     } else {
-        let npk = pack_with_manifest(northstar_manifest, &root, &out, key, &squashfs_opts)?;
-        let npk_size = human_bytes(fs::metadata(&npk)?.len() as f64);
+        let (npk, npk_size) = builder.manifest(northstar_manifest).to_dir(&out)?;
+        let npk_size = human_bytes(npk_size as f64);
         let duration = format_duration(time::Duration::from_secs(start.elapsed().as_secs()));
         let msg = format!(
             "{} [{}, {}] in {}",

northstar-runtime/Cargo.toml

@@ -76,6 +76,7 @@ seccomp = ["bindgen", "caps", "lazy_static", "memoffset", "nix", "npk"]
 anyhow = { version = "1.0.71", features = ["backtrace"] }
 memfd = "0.6.2"
 proptest = "1.2.0"
+rstest = { version = "0.18.1", default-features = false }
 serde_json = "1.0.95"
 tokio = { version = "1.29.1", features = ["test-util"] }
 tokio-test = "0.4.2"

northstar-runtime/src/npk/dm_verity.rs

@@ -7,7 +7,7 @@ use std::{
     io::{Read, SeekFrom::Start, Write},
 };
 
-use std::{io::Seek, path::Path};
+use std::io::Seek;
 use uuid::Uuid;
 
 pub const SHA256_SIZE: usize = 32;
@@ -122,11 +122,14 @@ impl VerityHeader {
 /// and a dm-verity hash_tree
 /// <https://gitlab.com/cryptsetup/cryptsetup/-/wikis/DMVerity#hash-tree>
 /// to the given file.
-pub fn append_dm_verity_block(fsimg: &Path, fsimg_size: u64) -> Result<Sha256Digest> {
+pub fn append_dm_verity_block<I: Read + Write + Seek>(
+    mut fsimg: I,
+    fsimg_size: u64,
+) -> Result<Sha256Digest> {
     let (level_offsets, tree_size) =
         calculate_hash_tree_level_offsets(fsimg_size as usize, BLOCK_SIZE, SHA256_SIZE);
     let (salt, root_hash, hash_tree) =
-        generate_hash_tree(fsimg, fsimg_size, &level_offsets, tree_size)?;
+        generate_hash_tree(&mut fsimg, fsimg_size, &level_offsets, tree_size)?;
     append_superblock_and_hashtree(fsimg, fsimg_size, &salt, &hash_tree)?;
     Ok(root_hash)
 }
@@ -169,17 +172,15 @@ fn calculate_hash_tree_level_offsets(
     (level_offsets, tree_size)
 }
 
-fn generate_hash_tree(
-    fsimg: &Path,
+fn generate_hash_tree<R: Read + Seek>(
+    mut fsimg: R,
     image_size: u64,
     level_offsets: &[usize],
     tree_size: usize,
 ) -> Result<(Salt, Sha256Digest, Vec<u8>)> {
     // For a description of the overall hash tree generation logic see
     // https://source.android.com/security/verifiedboot/dm-verity#hash-tree
 
-    let mut fsimg = &std::fs::File::open(fsimg)
-        .with_context(|| format!("failed to open {}", &fsimg.display()))?;
     let mut hashes: Vec<[u8; SHA256_SIZE]> = vec![];
     let mut level_num = 0;
     let mut level_size = image_size;
@@ -257,17 +258,12 @@ fn generate_hash_tree(
     Ok((salt, root_hash, hash_tree))
 }
 
-fn append_superblock_and_hashtree(
-    fsimg: &Path,
+fn append_superblock_and_hashtree<W: Write + Seek>(
+    mut fsimg: W,
     fsimg_size: u64,
     salt: &Salt,
     hash_tree: &[u8],
 ) -> Result<()> {
-    let mut fsimg = std::fs::OpenOptions::new()
-        .write(true)
-        .append(true)
-        .open(fsimg)
-        .with_context(|| format!("failed to open {}", &fsimg.display()))?;
     let mut uuid = [0u8; 16];
     uuid.copy_from_slice(
         hex::decode(Uuid::new_v4().to_string().replace('-', ""))

northstar-runtime/src/npk/mod.rs

@@ -17,3 +17,6 @@ pub const VERSION: Version = Version::new(
     pkg_version_minor!(),
     pkg_version_patch!(),
 );
+
+#[cfg(test)]
+mod tests;