diff --git a/Cargo.lock b/Cargo.lock index bf1cc3326..6c7a08b18 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1625,6 +1625,7 @@ dependencies = [ "proptest", "rand_core 0.6.4", "rlimit", + "rstest", "semver", "serde", "serde_json", @@ -1699,6 +1700,7 @@ dependencies = [ "tempfile", "tokio", "url", + "zip", ] [[package]] @@ -2119,6 +2121,12 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c" +[[package]] +name = "relative-path" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bf2521270932c3c7bed1a59151222bd7643c79310f2916f01925e1e16255698" + [[package]] name = "rlimit" version = "0.9.1" @@ -2128,6 +2136,33 @@ dependencies = [ "libc", ] +[[package]] +name = "rstest" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b96577ca10cb3eade7b337eb46520108a67ca2818a24d0b63f41fd62bc9651c" +dependencies = [ + "rstest_macros", + "rustc_version", +] + +[[package]] +name = "rstest_macros" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225e674cf31712b8bb15fdbca3ec0c1b9d825c5a24407ff2b7e005fb6a29ba03" +dependencies = [ + "cfg-if", + "glob", + "proc-macro2", + "quote", + "regex", + "relative-path", + "rustc_version", + "syn 2.0.15", + "unicode-ident", +] + [[package]] name = "rustc-demangle" version = "0.1.23" @@ -2140,6 +2175,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "0.37.13" diff --git a/Makefile.toml b/Makefile.toml index 84f04537b..0a779d598 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -36,11 +36,11 @@ args = [ "nextest", "run" ] [tasks.console] command = "cargo" -args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "console" ] +args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--compression", "none", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "console" ] [tasks.cpueater] command = "cargo" -args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "cpueater" ] +args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--compression", "none", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "cpueater" ] [tasks.crashing] command = "cargo" @@ -51,73 +51,73 @@ script = [ "cargo build --bin ferris", "ROOT=`mktemp -d`", "cp target/debug/ferris $ROOT", - "cargo run --bin northstar-sextant pack --out target/northstar/repository --key examples/northstar.key --manifest-path examples/ferris/manifest.yaml --root $ROOT" + "cargo run --bin northstar-sextant pack --compression none --out target/northstar/repository --key examples/northstar.key --manifest-path examples/ferris/manifest.yaml --root $ROOT" ] [tasks.hello-ferris] command = "cargo" -args = [ "run", "--bin", "northstar-sextant", "pack", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "--manifest-path", "examples/hello-ferris/manifest.yaml" ] +args = [ "run", "--bin", "northstar-sextant", "pack", "--compression", "none", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "--manifest-path", "examples/hello-ferris/manifest.yaml" ] [tasks.hello-resource] command = "cargo" -args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "hello-resource" ] +args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--compression", "none", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "hello-resource" ] [tasks.hello-world] command = "cargo" -args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "hello-world" ] +args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--compression", "none", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "hello-world" ] [tasks.inspect] command = "cargo" -args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "inspect" ] +args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--compression", "none", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "inspect" ] [tasks.memeater] command = "cargo" -args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "memeater" ] +args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--compression", "none", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "memeater" ] [tasks.message001] command = "cargo" -args = [ "run", "--bin", "northstar-sextant", "pack", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "--manifest-path", "examples/message-0.0.1/manifest.yaml", "--root", "examples/message-0.0.1/root" ] +args = [ "run", "--bin", "northstar-sextant", "pack", "--compression", "none", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "--manifest-path", "examples/message-0.0.1/manifest.yaml", "--root", "examples/message-0.0.1/root" ] [tasks.message002] command = "cargo" -args = [ "run", "--bin", "northstar-sextant", "pack", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "--manifest-path", "examples/message-0.0.2/manifest.yaml", "--root", "examples/message-0.0.2/root" ] +args = [ "run", "--bin", "northstar-sextant", "pack", "--compression", "none", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "--manifest-path", "examples/message-0.0.2/manifest.yaml", "--root", "examples/message-0.0.2/root" ] [tasks.netns] command = "cargo" -args = [ "run", "--bin", "northstar-sextant", "pack", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "--manifest-path", "examples/netns/manifest.yaml" ] +args = [ "run", "--bin", "northstar-sextant", "pack", "--compression", "none", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "--manifest-path", "examples/netns/manifest.yaml" ] [tasks.persistence] command = "cargo" -args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "persistence" ] +args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--compression", "none", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "persistence" ] [tasks.redis-client] command = "cargo" -args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "redis-client" ] +args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--compression", "none", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "redis-client" ] [tasks.redis-server] command = "cargo" -args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "redis-server" ] +args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--compression", "none", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "redis-server" ] [tasks.seccomp] command = "cargo" -args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "seccomp" ] +args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--compression", "none", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "seccomp" ] [tasks.sockets] command = "cargo" -args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "sockets" ] +args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--compression", "none", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "sockets" ] [tasks.test-container] command = "cargo" -args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "test-container" ] +args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--compression", "none", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "test-container" ] [tasks.test-resource] command = "cargo" -args = [ "run", "--bin", "northstar-sextant", "pack", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "--manifest-path", "examples/test-resource/manifest.yaml", "--root", "examples/test-resource/root" ] +args = [ "run", "--bin", "northstar-sextant", "pack", "--compression", "none", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "--manifest-path", "examples/test-resource/manifest.yaml", "--root", "examples/test-resource/root" ] [tasks.token-client] command = "cargo" -args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "token-client" ] +args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--compression", "none", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "token-client" ] [tasks.token-server] command = "cargo" -args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "token-server" ] +args = [ "run", "--bin", "cargo-npk", "npk", "pack", "--compression", "none", "--out", "target/northstar/repository", "--key", "examples/northstar.key", "-p", "token-server" ] diff --git a/cargo-npk/src/cli.rs b/cargo-npk/src/cli.rs index 06108c9da..4511c15db 100644 --- a/cargo-npk/src/cli.rs +++ b/cargo-npk/src/cli.rs @@ -35,6 +35,7 @@ pub enum ColorChoice { #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum)] pub enum Compression { + None, Gzip, Lzma, Lzo, @@ -45,6 +46,7 @@ pub enum Compression { impl From for NpkCompressionAlgorithm { fn from(c: Compression) -> Self { match c { + Compression::None => NpkCompressionAlgorithm::None, Compression::Gzip => NpkCompressionAlgorithm::Gzip, Compression::Lzma => NpkCompressionAlgorithm::Lzma, Compression::Lzo => NpkCompressionAlgorithm::Lzo, diff --git a/cargo-npk/src/lib.rs b/cargo-npk/src/lib.rs index d2214df5a..7a00f0601 100644 --- a/cargo-npk/src/lib.rs +++ b/cargo-npk/src/lib.rs @@ -19,7 +19,7 @@ use std::path::Path; use anyhow::Result; use cargo_subcommand::{Profile, Subcommand}; use clap::Parser; -use northstar_runtime::npk::npk::{pack_with_manifest, SquashfsOptions}; +use northstar_runtime::npk::npk::{NpkBuilder, SquashfsOptions}; use crate::metadata::Metadata; @@ -156,6 +156,13 @@ fn pack( cmd.build_dir(target) }; + let builder = NpkBuilder::default().root(&root, Some(&squashfs_opts)); + let builder = if let Some(key) = key { + builder.key(key) + } else { + builder + }; + if let Some(clones) = clones { let name = northstar_manifest.name.clone(); let num = clones.to_string().chars().count(); @@ -164,16 +171,16 @@ fn pack( manifest.name = format!("{name}-{n:0num$}") .try_into() .context("failed to parse name")?; - let npk = pack_with_manifest(&manifest, &root, &out, key, &squashfs_opts)?; - let npk_size = human_bytes(fs::metadata(&npk)?.len() as f64); + let (npk, npk_size) = builder.clone().manifest(&manifest).to_dir(&out)?; + let npk_size = human_bytes(npk_size as f64); let msg = format!("{} [{}, {}]", npk.display(), npk_size, compression); log("Packed", &msg)?; } let duration = format_duration(time::Duration::from_secs(start.elapsed().as_secs())); log("Finished", &format!("{clones} clones in {duration}"))?; } else { - let npk = pack_with_manifest(northstar_manifest, &root, &out, key, &squashfs_opts)?; - let npk_size = human_bytes(fs::metadata(&npk)?.len() as f64); + let (npk, npk_size) = builder.manifest(northstar_manifest).to_dir(&out)?; + let npk_size = human_bytes(npk_size as f64); let duration = format_duration(time::Duration::from_secs(start.elapsed().as_secs())); let msg = format!( "{} [{}, {}] in {}", diff --git a/northstar-runtime/Cargo.toml b/northstar-runtime/Cargo.toml index 7cd2bf124..424fb7dbb 100644 --- a/northstar-runtime/Cargo.toml +++ b/northstar-runtime/Cargo.toml @@ -76,6 +76,7 @@ seccomp = ["bindgen", "caps", "lazy_static", "memoffset", "nix", "npk"] anyhow = { version = "1.0.71", features = ["backtrace"] } memfd = "0.6.2" proptest = "1.2.0" +rstest = { version = "0.18.1", default-features = false } serde_json = "1.0.95" tokio = { version = "1.29.1", features = ["test-util"] } tokio-test = "0.4.2" diff --git a/northstar-runtime/src/npk/dm_verity.rs b/northstar-runtime/src/npk/dm_verity.rs index cc6ddcee0..79cf59b81 100644 --- a/northstar-runtime/src/npk/dm_verity.rs +++ b/northstar-runtime/src/npk/dm_verity.rs @@ -7,7 +7,7 @@ use std::{ io::{Read, SeekFrom::Start, Write}, }; -use std::{io::Seek, path::Path}; +use std::io::Seek; use uuid::Uuid; pub const SHA256_SIZE: usize = 32; @@ -122,11 +122,14 @@ impl VerityHeader { /// and a dm-verity hash_tree /// /// to the given file. -pub fn append_dm_verity_block(fsimg: &Path, fsimg_size: u64) -> Result { +pub fn append_dm_verity_block( + mut fsimg: I, + fsimg_size: u64, +) -> Result { let (level_offsets, tree_size) = calculate_hash_tree_level_offsets(fsimg_size as usize, BLOCK_SIZE, SHA256_SIZE); let (salt, root_hash, hash_tree) = - generate_hash_tree(fsimg, fsimg_size, &level_offsets, tree_size)?; + generate_hash_tree(&mut fsimg, fsimg_size, &level_offsets, tree_size)?; append_superblock_and_hashtree(fsimg, fsimg_size, &salt, &hash_tree)?; Ok(root_hash) } @@ -169,8 +172,8 @@ fn calculate_hash_tree_level_offsets( (level_offsets, tree_size) } -fn generate_hash_tree( - fsimg: &Path, +fn generate_hash_tree( + mut fsimg: R, image_size: u64, level_offsets: &[usize], tree_size: usize, @@ -178,8 +181,6 @@ fn generate_hash_tree( // For a description of the overall hash tree generation logic see // https://source.android.com/security/verifiedboot/dm-verity#hash-tree - let mut fsimg = &std::fs::File::open(fsimg) - .with_context(|| format!("failed to open {}", &fsimg.display()))?; let mut hashes: Vec<[u8; SHA256_SIZE]> = vec![]; let mut level_num = 0; let mut level_size = image_size; @@ -257,17 +258,12 @@ fn generate_hash_tree( Ok((salt, root_hash, hash_tree)) } -fn append_superblock_and_hashtree( - fsimg: &Path, +fn append_superblock_and_hashtree( + mut fsimg: W, fsimg_size: u64, salt: &Salt, hash_tree: &[u8], ) -> Result<()> { - let mut fsimg = std::fs::OpenOptions::new() - .write(true) - .append(true) - .open(fsimg) - .with_context(|| format!("failed to open {}", &fsimg.display()))?; let mut uuid = [0u8; 16]; uuid.copy_from_slice( hex::decode(Uuid::new_v4().to_string().replace('-', "")) diff --git a/northstar-runtime/src/npk/mod.rs b/northstar-runtime/src/npk/mod.rs index 63a4d55c2..e9d53f9d2 100644 --- a/northstar-runtime/src/npk/mod.rs +++ b/northstar-runtime/src/npk/mod.rs @@ -17,3 +17,6 @@ pub const VERSION: Version = Version::new( pkg_version_minor!(), pkg_version_patch!(), ); + +#[cfg(test)] +mod tests; diff --git a/northstar-runtime/src/npk/npk.rs b/northstar-runtime/src/npk/npk.rs index fe3b223a9..9ee492a83 100644 --- a/northstar-runtime/src/npk/npk.rs +++ b/northstar-runtime/src/npk/npk.rs @@ -64,6 +64,27 @@ pub struct Meta { pub version: Version, } +/// Squashfs Options +#[derive(Clone, Debug)] +pub struct SquashfsOptions { + /// Path to mksquashfs executable + pub mksquashfs: PathBuf, + /// The compression algorithm used (default gzip) + pub compression: Compression, + /// Size of the blocks of data compressed separately + pub block_size: Option, +} + +impl Default for SquashfsOptions { + fn default() -> Self { + SquashfsOptions { + compression: Compression::Gzip, + block_size: None, + mksquashfs: PathBuf::from(MKSQUASHFS), + } + } +} + /// NPK Hashes #[derive(Clone, Eq, PartialEq, Debug)] pub struct Hashes { @@ -77,6 +98,17 @@ pub struct Hashes { pub fs_verity_offset: u64, } +impl Hashes { + /// Read hashes from `reader`. + pub fn from_reader(mut reader: R) -> Result { + let mut buf = String::new(); + reader + .read_to_string(&mut buf) + .context("failed to read hashes")?; + Hashes::from_str(&buf) + } +} + impl FromStr for Hashes { type Err = Error; fn from_str(s: &str) -> Result { @@ -332,54 +364,12 @@ fn decode_signature(s: &str) -> Result { .context("failed to parse signature ed25519 format") } -struct Builder<'a> { - root: &'a Path, - manifest: &'a Manifest, - key: Option<&'a Path>, - squashfs_options: SquashfsOptions, -} - -impl<'a> Builder<'a> { - fn new(root: &'a Path, manifest: &'a Manifest) -> Builder<'a> { - Builder { - root, - manifest, - key: None, - squashfs_options: SquashfsOptions::default(), - } - } - - fn key(mut self, key: &'a Path) -> Builder<'a> { - self.key = Some(key); - self - } - - fn squashfs_opts(mut self, opts: &'a SquashfsOptions) -> Builder<'a> { - self.squashfs_options = opts.clone(); - self - } - - fn build(&self, writer: W) -> Result<()> { - // Create squashfs image - let tmp = tempfile::TempDir::new().context("failed to create temporary directory")?; - let meta = &Meta { version: VERSION }; - let fsimg = tmp.path().join(FS_IMG_NAME); - create_squashfs_img(self.manifest, self.root, &fsimg, &self.squashfs_options)?; - - // Sign and write NPK - if let Some(key) = &self.key { - let signature = signature(key, meta, &fsimg, self.manifest)?; - write_npk(writer, meta, self.manifest, &fsimg, Some(&signature)) - } else { - write_npk(writer, meta, self.manifest, &fsimg, None) - } - } -} - /// Squashfs compression algorithm -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Default)] #[allow(missing_docs)] pub enum Compression { + None, + #[default] Gzip, Lzma, Lzo, @@ -390,6 +380,7 @@ pub enum Compression { impl fmt::Display for Compression { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { + Compression::None => write!(f, "none"), Compression::Gzip => write!(f, "gzip"), Compression::Lzma => write!(f, "lzma"), Compression::Lzo => write!(f, "lzo"), @@ -414,104 +405,181 @@ impl FromStr for Compression { } } -/// Squashfs Options -#[derive(Clone, Debug)] -pub struct SquashfsOptions { - /// Path to mksquashfs executable - pub mksquashfs: PathBuf, - /// The compression algorithm used (default gzip) - pub compression: Compression, - /// Size of the blocks of data compressed separately - pub block_size: Option, +#[derive(Clone, Debug, Default)] +enum NpkBuilderManifest<'a> { + #[default] + None, + Manifest(Manifest), + ManifestPath(&'a Path), } -impl Default for SquashfsOptions { - fn default() -> Self { - SquashfsOptions { - compression: Compression::Gzip, - block_size: None, - mksquashfs: PathBuf::from(MKSQUASHFS), +/// Pack npks. +#[derive(Clone, Debug, Default)] +pub struct NpkBuilder<'a> { + manifest: NpkBuilderManifest<'a>, + root: Option<&'a Path>, + fsimage: Option<&'a Path>, + squashfs_options: Option<&'a SquashfsOptions>, + key: Option<&'a Path>, +} + +impl<'a> NpkBuilder<'a> { + /// Set the manifest. + pub fn manifest(mut self, manifest: &Manifest) -> NpkBuilder<'a> { + self.manifest = NpkBuilderManifest::Manifest(manifest.clone()); + self + } + + /// Set the manifest path. + pub fn manifest_path(mut self, manifest: &'a Path) -> NpkBuilder<'a> { + self.manifest = NpkBuilderManifest::ManifestPath(manifest); + self + } + + /// Set root directory. + pub fn root( + mut self, + root: &'a Path, + squashfs_options: Option<&'a SquashfsOptions>, + ) -> NpkBuilder<'a> { + self.root = Some(root); + self.squashfs_options = squashfs_options; + self + } + + /// Use existing plain `fsimage` as file system image. + pub fn fsimage(mut self, fsimage: &'a Path) -> NpkBuilder<'a> { + self.fsimage = Some(fsimage); + self + } + + /// Use key for signing. + pub fn key(mut self, key: &'a Path) -> NpkBuilder<'a> { + self.key = Some(key); + self + } + + /// Write npk to `file`. + pub fn to_file(self, file: &Path) -> Result { + let file = fs::File::create(file) + .with_context(|| format!("failed to create {}", file.display()))?; + self.to_writer(file) + } + + /// Write npk to `dir` with the filename `{name}-{version}.npk` with the values + /// from the manifest. + pub fn to_dir(self, dir: &Path) -> Result<(PathBuf, u64), Error> { + let mut me = self; + // Append filename from manifest if only a directory path was given. + // Otherwise use the given filename. + if Path::is_dir(dir) { + let manifest = me.get_manifest()?; + let mut npk_path = dir.to_path_buf(); + npk_path.push(format!("{}-{}.", &manifest.name, &manifest.version)); + npk_path.set_extension(NPK_EXT); + me.to_file(&npk_path) + .map(|size| (npk_path.to_owned(), size)) + } else { + Err(anyhow!("dir must be a directory").into()) } } -} -/// Create an NPK for the northstar runtime. -/// northstar-sextant collects the artifacts in a given container directory, creates and signs the necessary metadata -/// and packs the results into a zipped NPK file. -/// -/// # Arguments -/// * `manifest` - Path to the container's manifest file -/// * `root` - Path to the container's root directory -/// * `out` - Target directory or filename of the packed NPK -/// * `key` - Path to the key used to sign the package -/// -/// # Example -/// -/// To build the 'hello' example container: -/// -/// northstar-sextant pack \ -/// --manifest examples/hello/manifest.yaml \ -/// --root examples/hello/root \ -/// --out target/northstar/repository \ -/// --key examples/keys/northstar.key \ -pub fn pack( - manifest: &Path, - root: &Path, - out: &Path, - key: Option<&Path>, -) -> Result { - pack_with(manifest, root, out, key, &SquashfsOptions::default()) -} + /// Write npk to `writer`. + pub fn to_writer(self, writer: W) -> Result { + let mut me = self; + + const META: Meta = Meta { version: VERSION }; + let fsimage = me.fsimage; + let root = me.root; + let squashfs_options = me.squashfs_options.cloned().unwrap_or_default(); + let manifest = me.get_manifest()?; + + // Create zip. + let options = + zip::write::FileOptions::default().compression_method(zip::CompressionMethod::Stored); + let mut zip = zip::ZipWriter::new(writer); + + // Write meta data to zip comment. + let meta_str = serde_yaml::to_string(&META).context("failed to serialize meta")?; + zip.set_comment(meta_str); + + // Add manifest. + let manifest_str = manifest.to_string(); + zip.start_file(MANIFEST_NAME, options) + .context("failed to write manifest to NPK")?; + zip.write_all(manifest_str.as_bytes()) + .context("failed to write manifest to NPK")?; + + let (fsimage, fsimage_size, fsimage_tmp) = match (root, fsimage) { + (Some(_), Some(_)) => { + return Err(anyhow!("root and fsimage are mutually exclusive")).map_err(Into::into); + } + (Some(root), None) => { + // Create squashfs image. + let fsimage = + tempfile::NamedTempFile::new().context("failed to create tempfile")?; + let fsimage_size = mksquashfs(manifest, root, fsimage.path(), &squashfs_options)?; + (fsimage.path().to_owned(), fsimage_size, Some(fsimage)) + } + (None, Some(fsimage)) => { + let fsimage_size = fs::metadata(fsimage) + .with_context(|| format!("failed to get metadata of {}", fsimage.display()))? + .len(); + (fsimage.to_path_buf(), fsimage_size, None) + } + (None, None) => return Err(anyhow!("missing root or fsimage")).map_err(Into::into), + }; -/// Create an NPK with special `squashfs` options -/// -/// Returns the path to the created NPK. -/// -/// # Arguments -/// * `manifest` - Path to the container's manifest file -/// * `root` - Path to the container's root directory -/// * `out` - Target directory or filename of the packed NPK -/// * `key` - Path to the key used to sign the package -/// * `squashfs_opts` - Options for `mksquashfs` -/// -pub fn pack_with( - manifest: &Path, - root: &Path, - out: &Path, - key: Option<&Path>, - squashfs_opts: &SquashfsOptions, -) -> Result { - let manifest = read_manifest(manifest)?; - pack_with_manifest(&manifest, root, out, key, squashfs_opts) -} + let mut fsimage = fs::OpenOptions::new() + .read(true) + .write(true) + .append(true) + .open(fsimage) + .context("failed to open fsimage")?; + + if let Some(key) = me.key { + let signature = signature(key, &META, &mut fsimage, fsimage_size, &manifest_str)?; + zip.start_file(SIGNATURE_NAME, options) + .context("failed to add signature file")?; + zip.write_all(signature.as_bytes()) + .context("failed to write signature to NPK")?; + } -/// Create an NPK. -/// Returns the path to the created NPK. -pub fn pack_with_manifest( - manifest: &Manifest, - root: &Path, - out: &Path, - key: Option<&Path>, - squashfs_opts: &SquashfsOptions, -) -> Result { - let name = manifest.name.clone(); - let version = manifest.version.clone(); - let mut builder = Builder::new(root, manifest); - if let Some(key) = key { - builder = builder.key(key); - } - builder = builder.squashfs_opts(squashfs_opts); - - let mut dest = out.to_path_buf(); - // Append filename from manifest if only a directory path was given - if Path::is_dir(out) { - dest.push(format!("{}-{}.", &name, &version)); - dest.set_extension(NPK_EXT); - } - let npk = fs::File::create(&dest) - .with_context(|| format!("failed to create NPK: '{}'", &dest.display()))?; - builder.build(npk)?; - Ok(dest) + // We need to ensure that the fs.img start at an offset of 4096 so we add empty (zeros) ZIP + // 'extra data' to inflate the header of the ZIP file. + // See chapter 4.3.6 of APPNOTE.TXT + // (https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT) + zip.start_file_aligned(FS_IMG_NAME, options, BLOCK_SIZE as u16) + .context("failed to create aligned zip-file")?; + fsimage + .seek(SeekFrom::Start(0)) + .context("failed to seek to start of fs.img")?; + io::copy(&mut fsimage, &mut zip) + .context("failed to write the filesystem image to the archive")?; + drop(fsimage_tmp); + + let mut zip = zip.finish().context("failed to flush zip")?; + let zip_size = zip + .seek(SeekFrom::End(0)) + .context("failed to seek to end of zip")?; + + Ok(zip_size) + } + + fn get_manifest(&mut self) -> Result<&Manifest, Error> { + match self.manifest { + NpkBuilderManifest::None => Err(anyhow!("missing manifest").into()), + NpkBuilderManifest::Manifest(ref manifest) => Ok(manifest), + NpkBuilderManifest::ManifestPath(path) => { + let file = fs::File::open(path) + .with_context(|| format!("failed to open {}", path.display()))?; + let manifest = Manifest::from_reader(&file) + .with_context(|| format!("failed to parse {}", path.display()))?; + self.manifest = NpkBuilderManifest::Manifest(manifest); + self.get_manifest() + } + } + } } /// Extract the npk content to `out` @@ -520,12 +588,28 @@ pub fn unpack(npk: &Path, out: &Path) -> Result<(), Error> { } /// Extract the npk content to `out` with a give unsquashfs binary -pub fn unpack_with(npk: &Path, out: &Path, unsquashfs: &Path) -> Result<(), Error> { - let mut zip = open(npk)?; +pub fn unpack_with(path: &Path, out: &Path, unsquashfs: &Path) -> Result<(), Error> { + // Open zip archive. + let npk = + fs::File::open(path).with_context(|| format!("failed to open {}", &path.display()))?; + let mut zip = ZipArchive::new(BufReader::new(npk)) + .with_context(|| format!("failed to parse ZIP format: {}", &path.display()))?; + + // Extract zip archive. zip.extract(out) - .with_context(|| format!("failed to extract NPK to '{}'", &out.display()))?; + .with_context(|| format!("failed to extract NPK to {}", &out.display()))?; + + // Unpack squashfs image. let fsimg = out.join(FS_IMG_NAME); - unpack_squashfs(&fsimg, out, unsquashfs)?; + let root = out.join("root"); + + let mut cmd = Command::new(unsquashfs); + cmd.arg("-dest") + .arg(&root.display().to_string()) + .arg(&fsimg.display().to_string()); + cmd.output().context("failed to unsquashfs")?; + fs::remove_file(&fsimg).with_context(|| format!("failed to remove {}", &fsimg.display()))?; + Ok(()) } @@ -533,20 +617,12 @@ pub fn unpack_with(npk: &Path, out: &Path, unsquashfs: &Path) -> Result<(), Erro pub fn generate_key(name: &str, out: &Path) -> Result<(), Error> { fn assume_non_existing(path: &Path) -> anyhow::Result<()> { if path.exists() { - bail!("file '{}' already exists", &path.display()) + bail!("file {} already exists", &path.display()) } else { Ok(()) } } - fn write(data: &[u8], path: &Path) -> Result<(), Error> { - let mut file = fs::File::create(path) - .with_context(|| format!("failed to create '{}'", path.display()))?; - file.write_all(data) - .with_context(|| format!("failed to write to '{}'", &path.display()))?; - Ok(()) - } - let mut secret_key_bytes = [0u8; 32]; OsRng.fill_bytes(&mut secret_key_bytes); @@ -559,18 +635,12 @@ pub fn generate_key(name: &str, out: &Path) -> Result<(), Error> { assume_non_existing(&public_key_file)?; assume_non_existing(&secret_key_file)?; - write(&secret_key.to_bytes(), &secret_key_file)?; - write(&public_key.to_bytes(), &public_key_file)?; + fs::write(&secret_key_file, secret_key.to_bytes()).context("failed to write secret key")?; + fs::write(&public_key_file, public_key.to_bytes()).context("failed to write public key")?; Ok(()) } -fn read_manifest(path: &Path) -> Result { - let file = - fs::File::open(path).with_context(|| format!("failed to open '{}'", &path.display()))?; - Manifest::from_reader(&file).with_context(|| format!("failed to parse '{}'", &path.display())) -} - fn read_keypair(key_file: &Path) -> Result { let mut secret_key_bytes = [0u8; SECRET_KEY_LENGTH]; fs::File::open(key_file) @@ -617,16 +687,17 @@ fn hashes_yaml( } /// Try to construct the signature yaml file -fn signature(key: &Path, meta: &Meta, fsimg: &Path, manifest: &Manifest) -> Result { +fn signature( + key: &Path, + meta: &Meta, + fsimg: I, + fsimg_size: u64, + manifest: &str, +) -> Result { let meta_hash = Sha256::digest(serde_yaml::to_string(&meta).context("failed to encode metadata")?); - let manifest_hash = Sha256::digest(manifest.to_string().as_bytes()); + let manifest_hash = Sha256::digest(manifest.as_bytes()); - // The size of the fs image is the offset of the verity block. The verity block - // is appended to the fs.img - let fsimg_size = fs::metadata(fsimg) - .with_context(|| format!("failed to read file size: '{}'", &fsimg.display()))? - .len(); // Calculate verity root hash let fsimg_hash: &[u8] = &append_dm_verity_block(fsimg, fsimg_size) .context("failed to calculate verity root hash")?; @@ -742,18 +813,18 @@ fn pseudo_files(manifest: &Manifest) -> Result { Ok(pseudo_file_entries) } -fn create_squashfs_img( +fn mksquashfs( manifest: &Manifest, root: &Path, image: &Path, squashfs_opts: &SquashfsOptions, -) -> Result<()> { +) -> Result { let pseudo_files = pseudo_files(manifest)?; let mksquashfs = &squashfs_opts.mksquashfs; // Check root if !root.exists() { - bail!("Root directory '{}' does not exist", &root.display()); + bail!("root directory {} does not exist", &root.display()); } // Check mksquashfs version @@ -774,7 +845,7 @@ fn create_squashfs_img( .unwrap_or_default(); let minor = major_minor.next().unwrap_or_default(); let minor = minor.parse::().unwrap_or_else(|_| { - // remove trailing subversion if present (e.g. 4.4-e0485802) + // Remove trailing subversion if present (e.g. 4.4-e0485802) minor .split(|c: char| !c.is_numeric()) .next() @@ -802,9 +873,8 @@ fn create_squashfs_img( let mut cmd = Command::new(mksquashfs); cmd.arg(&root.display().to_string()) .arg(&image.display().to_string()) + .arg("-noappend") .arg("-no-progress") - .arg("-comp") - .arg(squashfs_opts.compression.to_string()) .arg("-info") .arg("-force-uid") .arg(manifest.uid.to_string()) @@ -812,83 +882,32 @@ fn create_squashfs_img( .arg(manifest.gid.to_string()) .arg("-pf") .arg(pseudo_files.path()); + if let Some(block_size) = squashfs_opts.block_size { cmd.arg("-b").arg(format!("{block_size}")); } - cmd.output() - .with_context(|| format!("failed to execute '{}'", mksquashfs.display()))?; - if !image.exists() { - bail!( - "'{}' failed to create '{}'", - mksquashfs.display(), - &image.display() - ); - } - Ok(()) -} - -fn unpack_squashfs(image: &Path, out: &Path, unsquashfs: &Path) -> Result<()> { - let squashfs_root = out.join("squashfs-root"); - - if !image.exists() { - bail!("Squashfs image '{}' does not exist", &image.display()); + match &squashfs_opts.compression { + Compression::None => { + cmd.args(["-noI", "-noD", "-noF", "-noX", "-no-fragments"]); + } + compression => { + cmd.arg("-comp").arg(compression.to_string()); + } } - let mut cmd = Command::new(unsquashfs); - cmd.arg("-dest") - .arg(&squashfs_root.display().to_string()) - .arg(&image.display().to_string()); - cmd.output() - .with_context(|| format!("Error while executing '{}'", unsquashfs.display(),))?; - - Ok(()) -} + match cmd.output() { + Ok(output) if !output.status.success() => { + return Err(anyhow!("mksquashfs failed with {:?}", output.status)) + } + Ok(_) => (), + Err(e) => return Err(anyhow!("mksquashfs failed: {e}")), + } -fn write_npk( - npk: W, - meta: &Meta, - manifest: &Manifest, - fsimg: &Path, - signature: Option<&str>, -) -> Result<()> { - let mut fsimg = - fs::File::open(fsimg).with_context(|| format!("failed to open '{}'", &fsimg.display()))?; - let options = - zip::write::FileOptions::default().compression_method(zip::CompressionMethod::Stored); - let manifest_string = - serde_yaml::to_string(&manifest).context("failed to serialize manifest")?; - let meta_string = serde_yaml::to_string(&meta).context("failed to serialize meta")?; - - let mut zip = zip::ZipWriter::new(npk); - zip.set_comment(&meta_string); - - if let Some(signature) = signature { - zip.start_file(SIGNATURE_NAME, options)?; - zip.write_all(signature.as_bytes()) - .context("failed to write signature to NPK")?; - } - - zip.start_file(MANIFEST_NAME, options) - .context("failed to write manifest to NPK")?; - zip.write_all(manifest_string.as_bytes()) - .context("failed to convert manifest to NPK")?; - - // We need to ensure that the fs.img start at an offset of 4096 so we add empty (zeros) ZIP - // 'extra data' to inflate the header of the ZIP file. - // See chapter 4.3.6 of APPNOTE.TXT - // (https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT) - zip.start_file_aligned(FS_IMG_NAME, options, BLOCK_SIZE as u16) - .context("Could create aligned zip-file")?; - io::copy(&mut fsimg, &mut zip) - .context("failed to write the filesystem image to the archive")?; - Ok(()) -} + let image_size = image + .metadata() + .context("failed to read image metadata")? + .len(); -/// Open a Zip file -fn open(path: &Path) -> Result>> { - let file = - fs::File::open(path).with_context(|| format!("failed to open '{}'", &path.display()))?; - ZipArchive::new(BufReader::new(file)) - .with_context(|| format!("failed to parse ZIP format: '{}'", &path.display())) + Ok(image_size) } diff --git a/northstar-runtime/src/npk/tests.rs b/northstar-runtime/src/npk/tests.rs new file mode 100644 index 000000000..4dcdbf92b --- /dev/null +++ b/northstar-runtime/src/npk/tests.rs @@ -0,0 +1,244 @@ +use crate::npk::{ + manifest::Manifest, + npk::{self, Compression, NpkBuilder, FS_IMG_NAME, MANIFEST_NAME}, +}; +use anyhow::Result; +use rstest::{fixture, rstest}; +use std::{ + fs::{self}, + io, + path::{Path, PathBuf}, + str::FromStr, +}; +use tempfile::TempDir; +use zip::ZipArchive; + +const TEST_KEY_NAME: &str = "test_key"; +const TEST_CONTAINER_NAME: &str = "hello-0.0.2.npk"; +const TEST_MANIFEST: &str = "name: hello +version: 0.0.2 +init: /hello +uid: 100 +gid: 1 +env: + HELLO: north"; + +struct Fixture { + manifest_path: PathBuf, + manifest: Manifest, + root: PathBuf, + dir: PathBuf, + tmpdir: TempDir, + key_prv: PathBuf, +} + +#[fixture] +fn fixture() -> Fixture { + let tmpdir = TempDir::new().expect("failed to create tempdir"); + let dir = tmpdir.path().to_path_buf(); + let root = root(tmpdir.path()).expect("create root"); + let manifest_path = tmpdir.path().join(MANIFEST_NAME); + fs::write(&manifest_path, TEST_MANIFEST).expect("write manifest"); + let manifest = Manifest::from_str(TEST_MANIFEST).expect("parse manifest"); + let (_, key_prv) = generate_test_key(tmpdir.path()); + Fixture { + manifest_path, + manifest, + root, + dir, + tmpdir, + key_prv, + } +} + +#[rstest] +fn pack_with_manifest_file(fixture: Fixture) -> Result<()> { + let npk = NpkBuilder::default() + .manifest_path(&fixture.manifest_path) + .root(&fixture.root, None) + .to_dir(fixture.tmpdir.path())? + .0; + + assert!(fixture.dir.join(TEST_CONTAINER_NAME).exists()); + assert_test_manifest(&npk)?; + assert_root(&npk)?; + + Ok(()) +} + +#[rstest] +fn pack_without_key(fixture: Fixture) -> Result<()> { + let npk = NpkBuilder::default() + .manifest(&fixture.manifest) + .root(&fixture.root, None) + .to_dir(fixture.tmpdir.path())? + .0; + + assert!(fixture.dir.join(TEST_CONTAINER_NAME).exists()); + assert_test_manifest(&npk)?; + assert_root(&npk)?; + + Ok(()) +} + +#[rstest] +fn pack_with_key(fixture: Fixture) -> Result<()> { + let npk = NpkBuilder::default() + .manifest(&fixture.manifest) + .root(&fixture.root, None) + .key(&fixture.key_prv) + .to_dir(&fixture.dir)? + .0; + + assert!(fixture.dir.join(TEST_CONTAINER_NAME).exists()); + assert_test_manifest(&npk)?; + assert_root(&npk)?; + + Ok(()) +} + +#[rstest] +fn pack_with_compression_none( + #[values( + Compression::None, + Compression::Gzip, + Compression::Lzma, + Compression::Lzo, + Compression::Xz, + Compression::Zstd + )] + compression: Compression, + fixture: Fixture, +) -> Result<()> { + let squashfs_options = npk::SquashfsOptions { + compression, + ..Default::default() + }; + + NpkBuilder::default() + .manifest(&fixture.manifest) + .root(&fixture.root, Some(&squashfs_options)) + .key(&fixture.key_prv) + .to_dir(&fixture.dir)?; + + assert!(fixture.dir.join(TEST_CONTAINER_NAME).exists()); + Ok(()) +} + +#[rstest] +fn pack_with_fs_image(fixture: Fixture) -> Result<()> { + // Pack a npk in order to obtain a fs image. + // Do not use a key here. + let npk = NpkBuilder::default() + .manifest(&fixture.manifest) + .root(&fixture.root, None) + .to_dir(&fixture.dir)? + .0; + + // Get fs image from npk. + let mut zip = ZipArchive::new(fs::File::open(&npk)?)?; + let mut fs_img_zip = zip.by_name(FS_IMG_NAME)?; + let fs_img_path = fixture.dir.join(FS_IMG_NAME); + let mut fs_img = fs::File::create(&fs_img_path)?; + io::copy(&mut fs_img_zip, &mut fs_img)?; + + NpkBuilder::default() + .manifest(&fixture.manifest) + .fsimage(&fs_img_path) + .to_file(&npk)?; + + assert!(npk.exists()); + assert_test_manifest(&npk)?; + assert_root(&npk)?; + + Ok(()) +} + +#[rstest] +fn pack_with_manifest_root_and_fsimage_should_fail(fixture: Fixture) -> Result<()> { + let result = NpkBuilder::default() + .manifest(&fixture.manifest) + .root(&fixture.root, None) + .fsimage(&fixture.root) + .to_dir(&fixture.dir); + + assert!(result.is_err()); + + Ok(()) +} + +#[rstest] +fn pack_to_file(fixture: Fixture) -> Result<()> { + let npk = fixture.dir.join("test.npk"); + + NpkBuilder::default() + .manifest(&fixture.manifest) + .root(&fixture.root, None) + .key(&fixture.key_prv) + .to_file(&npk)?; + + assert!(npk.exists()); + assert_test_manifest(&npk)?; + assert_root(&npk)?; + + Ok(()) +} + +#[rstest] +fn pack_to_writer(fixture: Fixture) -> Result<()> { + let npk_path = fixture.dir.join("test.nkp"); + let npk = fs::File::create(&npk_path)?; + + NpkBuilder::default() + .manifest(&fixture.manifest) + .root(&fixture.root, None) + .key(&fixture.key_prv) + .to_writer(npk)?; + + assert!(npk_path.exists()); + assert_test_manifest(&npk_path)?; + assert_root(&npk_path)?; + + Ok(()) +} + +fn generate_test_key(key_dir: &Path) -> (PathBuf, PathBuf) { + npk::generate_key(TEST_KEY_NAME, key_dir).expect("Generate key pair"); + let prv_key = key_dir.join(TEST_KEY_NAME).with_extension("key"); + let pub_key = key_dir.join(TEST_KEY_NAME).with_extension("pub"); + assert!(prv_key.exists()); + assert!(pub_key.exists()); + (pub_key, prv_key) +} + +fn root(tmpdir: &Path) -> Result { + let root = tmpdir.join("root"); + fs::create_dir_all(&root)?; + fs::create_dir(root.join("bin"))?; + fs::create_dir(root.join("etc"))?; + fs::create_dir(root.join("lib"))?; + fs::File::create(root.join("foo"))?; + fs::File::create(root.join("etc").join("hosts"))?; + Ok(root) +} + +fn assert_root(npk: &Path) -> Result<()> { + let tmpdir = TempDir::new()?; + npk::unpack(npk, tmpdir.path())?; + let root = tmpdir.path().join("root"); + assert!(root.join("bin").is_dir()); + assert!(root.join("etc").is_dir()); + assert!(root.join("lib").is_dir()); + assert!(root.join("foo").is_file()); + assert!(root.join("etc").join("hosts").is_file()); + Ok(()) +} + +fn assert_test_manifest(npk: &Path) -> Result<()> { + let tmpdir = TempDir::new()?; + npk::unpack(npk, tmpdir.path())?; + let test_manifest = Manifest::from_str(TEST_MANIFEST)?; + let manifest = Manifest::from_reader(fs::File::open(tmpdir.path().join(MANIFEST_NAME))?)?; + assert_eq!(manifest, test_manifest); + Ok(()) +} diff --git a/northstar-sextant/src/inspect.rs b/northstar-sextant/src/inspect.rs index c9a39a670..490e9e20d 100644 --- a/northstar-sextant/src/inspect.rs +++ b/northstar-sextant/src/inspect.rs @@ -106,7 +106,7 @@ fn print_squashfs(fsimg_path: &Path, unsquashfs: &Path) -> Result<()> { #[cfg(test)] mod test { use super::inspect; - use northstar_runtime::npk::npk::{generate_key, pack}; + use northstar_runtime::npk::npk::{generate_key, NpkBuilder}; use std::{ fs::File, io::Write, @@ -139,8 +139,13 @@ mounts: let key_dir = create_tmp_dir(); let manifest = create_test_manifest(src.path()); let (_pub_key, prv_key) = gen_test_key(key_dir.path()); - pack(&manifest, src.path(), dest, Some(&prv_key)).expect("Pack NPK"); - dest.join("hello-0.0.2.npk") + NpkBuilder::default() + .manifest_path(&manifest) + .root(src.path(), None) + .key(&prv_key) + .to_dir(dest) + .expect("failed to pack npk") + .0 } fn create_test_manifest(src: &Path) -> PathBuf { diff --git a/northstar-sextant/src/main.rs b/northstar-sextant/src/main.rs index 16b6284a6..634016911 100644 --- a/northstar-sextant/src/main.rs +++ b/northstar-sextant/src/main.rs @@ -14,8 +14,10 @@ use std::path::PathBuf; mod inspect; mod pack; -#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum)] +#[derive(Debug, Copy, Clone, Default, PartialEq, Eq, PartialOrd, Ord, ValueEnum)] enum Compression { + None, + #[default] Gzip, Lzma, Lzo, @@ -26,6 +28,7 @@ enum Compression { impl From for NpkCompression { fn from(c: Compression) -> Self { match c { + Compression::None => NpkCompression::None, Compression::Gzip => NpkCompression::Gzip, Compression::Lzma => NpkCompression::Lzma, Compression::Lzo => NpkCompression::Lzo, @@ -119,17 +122,18 @@ fn main() -> Result<()> { let tempdir = tempfile::tempdir()?; (tempdir.path().to_owned(), Some(tempdir)) }; + let squashfs_options = &SquashfsOptions { + compression: compression.into(), + mksquashfs, + block_size, + }; pack::pack( &manifest_path, &root, &out, key.as_deref(), - SquashfsOptions { - compression: compression.into(), - mksquashfs, - block_size, - }, + squashfs_options, clones, )? } diff --git a/northstar-sextant/src/pack.rs b/northstar-sextant/src/pack.rs index 8e05009b5..a9afa8fa8 100644 --- a/northstar-sextant/src/pack.rs +++ b/northstar-sextant/src/pack.rs @@ -1,7 +1,7 @@ use anyhow::{Context, Result}; use northstar_runtime::npk::{ manifest::Manifest, - npk::{pack_with_manifest, SquashfsOptions}, + npk::{NpkBuilder, SquashfsOptions}, }; use std::{convert::TryInto, fs, path::Path}; @@ -11,14 +11,20 @@ pub(crate) fn pack( root: &Path, out: &Path, key: Option<&Path>, - squashfs_options: SquashfsOptions, + squashfs_options: &SquashfsOptions, clones: Option, ) -> Result<()> { - let reader = fs::File::open(manifest).context("failed to open manifest")?; - let mut manifest = Manifest::from_reader(reader).context("failed to read manifest")?; + let builder = NpkBuilder::default().root(root, Some(squashfs_options)); + let builder = if let Some(key) = key { + builder.key(key) + } else { + builder + }; // Create npk clones with the number appended to the name if let Some(clones) = clones { + let reader = fs::File::open(manifest).context("failed to open manifest")?; + let mut manifest = Manifest::from_reader(reader).context("failed to read manifest")?; // Only clone non-resource containers if manifest.init.is_some() { let name = manifest.name.clone(); @@ -27,13 +33,13 @@ pub(crate) fn pack( manifest.name = format!("{name}-{n:0num$}") .try_into() .context("failed to parse name")?; - pack_with_manifest(&manifest, root, out, key, &squashfs_options)?; + builder.clone().manifest(&manifest).to_dir(out)?; } } else { - pack_with_manifest(&manifest, root, out, key, &squashfs_options)?; + builder.manifest(&manifest).to_dir(out)?; } } else { - pack_with_manifest(&manifest, root, out, key, &squashfs_options)?; + builder.manifest_path(manifest).to_dir(out)?; } Ok(()) diff --git a/northstar-tests/Cargo.toml b/northstar-tests/Cargo.toml index decbfb383..bea6567fd 100644 --- a/northstar-tests/Cargo.toml +++ b/northstar-tests/Cargo.toml @@ -21,3 +21,4 @@ regex = "1.7.3" tempfile = "3.5.0" tokio = { version = "1.29.1", features = ["fs", "time"] } url = "2.3.1" +zip = { version = "0.6.6", default-features = false } diff --git a/northstar-tests/src/containers.rs b/northstar-tests/src/containers.rs index 8ba414dab..c2599deac 100644 --- a/northstar-tests/src/containers.rs +++ b/northstar-tests/src/containers.rs @@ -1,13 +1,18 @@ -use std::{fs, io::Read, os::unix::fs::MetadataExt, path::Path}; +use anyhow::anyhow; +use std::{ + fs, + io::{self, Read}, + os::unix::fs::MetadataExt, + path::Path, + str::FromStr, +}; use anyhow::Context; use lazy_static::lazy_static; use northstar_runtime::npk::{ manifest::Manifest, - npk::{self, Compression, SquashfsOptions}, + npk::{Hashes, NpkBuilder, FS_IMG_NAME, MANIFEST_NAME, SIGNATURE_NAME}, }; -use tempfile::tempdir; - macro_rules! npk { ($x:expr) => {{ fs::File::open($x) @@ -92,51 +97,52 @@ pub fn with_manifest(container: &[u8], patch: F) -> anyhow::Result> where F: FnOnce(&mut Manifest), { - let tmpdir = tempdir()?; let key = Path::new("../examples/northstar.key"); - let src = tmpdir.path().join("src.npk"); - let unpacked = tmpdir.path().join("unpacked"); - let manifest = unpacked.join("manifest.yaml"); - let out = tmpdir.path().join("out.npk"); - let root = unpacked.join("squashfs-root"); - // Dump container to disk and unpack it. - fs::write(&src, container)?; - npk::unpack(&src, &unpacked)?; + // Open zip archive. + let mut zip = + zip::ZipArchive::new(io::Cursor::new(container)).context("failed to parse ZIP")?; - // Load manifest - let manifest = fs::File::open(&manifest).context("failed to open manifest")?; + // Apply manifest patch. + let manifest = zip + .by_name(MANIFEST_NAME) + .context("failed to find manifest")?; let mut manifest = Manifest::from_reader(manifest).context("failed to parse manifest")?; + patch(&mut manifest); - // Remove existing mountpoints that are created while packing. - for mount_point in manifest.mounts.keys() { - let mount_point = tmpdir - .path() - .join(mount_point.strip_prefix('/').unwrap_or(mount_point)); - fs::remove_dir_all(mount_point).ok(); - } + // Read hashes to obtain the length of the fs image without the verity block. + let fsimg_size = { + let mut signature = zip + .by_name(SIGNATURE_NAME) + .context("failed to find hashes")?; + let mut content = String::with_capacity(signature.size() as usize); + signature + .read_to_string(&mut content) + .context("failed to read hashes")?; + let mut documents = content.split("---"); + let hashes_str = documents + .next() + .ok_or_else(|| anyhow!("malformed signatures file"))?; + let hashes = Hashes::from_str(hashes_str)?; + hashes.fs_verity_offset + }; - // Apply manifest patch. - patch(&mut manifest); + let fsimg = zip + .by_name(FS_IMG_NAME) + .context("failed to find manifest")?; + let mut fsimage_tmp = tempfile::NamedTempFile::new().context("failed to create tempfile")?; + io::copy(&mut fsimg.take(fsimg_size), &mut fsimage_tmp).context("failed to copy fsimg")?; + + // Output buffer. + let mut npk = io::Cursor::new(Vec::new()); - // Repack - npk::pack_with_manifest( - &manifest, - &root, - &out, - Some(key), - &SquashfsOptions { - compression: Compression::Gzip, - ..Default::default() - }, - ) - .context("failed to pack")?; + // Repack. + NpkBuilder::default() + .fsimage(fsimage_tmp.path()) + .manifest(&manifest) + .key(key) + .to_writer(&mut npk)?; + drop(fsimage_tmp); - // Load repacked container. - let mut buf = Vec::new(); - fs::File::open(&out) - .context("failed to open npk")? - .read_to_end(&mut buf) - .context("failed to read npk") - .map(|_| buf) + Ok(npk.into_inner()) } diff --git a/northstar-tests/tests/npk.rs b/northstar-tests/tests/npk.rs deleted file mode 100644 index 0014bb72e..000000000 --- a/northstar-tests/tests/npk.rs +++ /dev/null @@ -1,129 +0,0 @@ -use northstar_runtime::npk::npk; -use std::{ - fs::{self, File}, - io::Write, - path::{Path, PathBuf}, -}; -use tempfile::TempDir; - -const TEST_KEY_NAME: &str = "test_key"; -const TEST_CONTAINER_NAME: &str = "hello-0.0.2.npk"; -const TEST_MANIFEST: &str = "name: hello -version: 0.0.2 -init: /hello -uid: 100 -gid: 1 -env: - HELLO: north"; -const TEST_MANIFEST_UNPACKED: &str = "name: hello -version: 0.0.2 -init: /hello -env: - HELLO: north -uid: 100 -gid: 1 -"; - -fn tmpdir() -> TempDir { - TempDir::new().expect("failed to create tempdir") -} - -fn create(dest: &Path, manifest_name: Option<&str>) { - let src = tmpdir(); - let key_dir = tmpdir(); - let manifest = create_test_manifest(src.path(), manifest_name); - let (_, prv_key) = generate_test_key(key_dir.path()); - npk::pack(&manifest, src.path(), dest, Some(&prv_key)).expect("Pack NPK"); -} - -fn create_test_manifest(dest: &Path, manifest_name: Option<&str>) -> PathBuf { - let manifest = dest - .join(manifest_name.unwrap_or("manifest")) - .with_extension("yaml"); - File::create(&manifest) - .expect("Create test manifest file") - .write_all(TEST_MANIFEST.as_ref()) - .expect("Write test manifest"); - manifest -} - -fn generate_test_key(key_dir: &Path) -> (PathBuf, PathBuf) { - npk::generate_key(TEST_KEY_NAME, key_dir).expect("Generate key pair"); - let prv_key = key_dir.join(TEST_KEY_NAME).with_extension("key"); - let pub_key = key_dir.join(TEST_KEY_NAME).with_extension("pub"); - assert!(prv_key.exists()); - assert!(pub_key.exists()); - (pub_key, prv_key) -} - -#[test] -fn pack() { - let dest = tmpdir(); - create(dest.path(), None); -} - -#[test] -fn pack_with_manifest() { - let dest = tmpdir(); - create(dest.path(), Some("different_manifest_name")); -} - -#[test] -fn pack_missing_manifest() { - let src = tmpdir(); - let dest = tmpdir(); - let key_dir = tmpdir(); - let manifest = Path::new("invalid"); - let (_pub_key, prv_key) = generate_test_key(key_dir.path()); - npk::pack(manifest, src.path(), dest.path(), Some(&prv_key)).expect_err("invalid manifest"); -} - -#[test] -fn pack_file_as_destination() { - let tmp = tmpdir(); - let dest = tmp.path().join("file.npk"); - create(dest.as_path(), None); -} - -#[test] -fn pack_invalid_key() { - let src = tmpdir(); - let dest = tmpdir(); - let manifest = create_test_manifest(src.path(), None); - let private = Path::new("invalid"); - npk::pack(&manifest, src.path(), dest.path(), Some(private)).expect_err("invalid key dir"); -} - -#[test] -fn unpack() { - let npk_dest = tmpdir(); - create(npk_dest.path(), None); - let npk = npk_dest.path().join(TEST_CONTAINER_NAME); - assert!(npk.exists()); - let unpack_dest = tmpdir(); - npk::unpack(&npk, unpack_dest.path()).expect("Unpack NPK"); - let manifest = unpack_dest.path().join("manifest").with_extension("yaml"); - assert!(manifest.exists()); - let manifest = fs::read_to_string(&manifest).expect("failed to parse manifest"); - - assert_eq!(TEST_MANIFEST_UNPACKED, manifest); -} - -#[test] -fn generate_key_pair() { - let dest = tmpdir(); - generate_test_key(dest.path()); -} - -#[test] -fn generate_key_pair_no_dest() { - npk::generate_key(TEST_KEY_NAME, Path::new("invalid")).expect_err("invalid key dir"); -} - -#[test] -fn do_not_overwrite_keys() -> Result<(), anyhow::Error> { - let dest = tmpdir(); - npk::generate_key(TEST_KEY_NAME, dest.path()).expect("Generate keys"); - npk::generate_key(TEST_KEY_NAME, dest.path()).expect_err("Cannot overwrite keys"); - Ok(()) -}