fix: prevent uploading compressed files which might exhaust the memory

Cargo.toml

@@ -135,7 +135,7 @@ utoipa-redoc = { version = "4.0.0", features = ["actix-web"] }
 utoipa-swagger-ui = "7.1.0"
 uuid = "1.7.0"
 walkdir = "2.5"
-walker-common = "0.9.2"
+walker-common = "0.9.3"
 walker-extras = "0.9.0"
 zip = "2.2.0"
 

common/src/decompress.rs

@@ -2,7 +2,7 @@ use actix_web::http::header;
 use anyhow::anyhow;
 use bytes::Bytes;
 use tokio::{runtime::Handle, task::JoinError};
-use walker_common::compression::{Compression, Detector};
+use walker_common::compression::{Compression, DecompressionOptions, Detector};
 
 #[derive(Debug, thiserror::Error)]
 pub enum Error {
@@ -12,6 +12,8 @@ pub enum Error {
     Detector(anyhow::Error),
     #[error(transparent)]
     Io(#[from] std::io::Error),
+    #[error("payload too large")]
+    PayloadTooLarge,
 }
 
 /// Take some bytes, and an optional content-type header and decompress, if required.
@@ -25,7 +27,11 @@ pub enum Error {
 /// **NOTE:** Depending on the size of the payload, this method might take some time. In an async
 /// context, it might be necessary to run this as a blocking function, or use [`decompress_async`]
 /// instead.
-pub fn decompress(bytes: Bytes, content_type: Option<header::ContentType>) -> Result<Bytes, Error> {
+pub fn decompress(
+    bytes: Bytes,
+    content_type: Option<header::ContentType>,
+    limit: usize,
+) -> Result<Bytes, Error> {
     let content_type = content_type.as_ref().map(|ct| ct.as_ref());
 
     // check what the user has declared
@@ -56,16 +62,22 @@ pub fn decompress(bytes: Bytes, content_type: Option<header::ContentType>) -> Re
 
     // decompress (or not)
 
-    Ok(compression.decompress(bytes)?)
+    compression
+        .decompress_with(bytes, &DecompressionOptions::default().limit(limit))
+        .map_err(|err| match err.kind() {
+            std::io::ErrorKind::WriteZero => Error::PayloadTooLarge,
+            _ => Error::from(err),
+        })
 }
 
 /// An async version of [`decompress`].
 pub async fn decompress_async(
     bytes: Bytes,
     content_type: Option<header::ContentType>,
+    limit: usize,
 ) -> Result<Result<Bytes, Error>, JoinError> {
     Handle::current()
-        .spawn_blocking(|| decompress(bytes, content_type))
+        .spawn_blocking(move || decompress(bytes, content_type, limit))
         .await
 }
 
@@ -81,6 +93,7 @@ mod test {
         let bytes = decompress_async(
             document_bytes_raw("ubi9-9.2-755.1697625012.json").await?,
             None,
+            0,
         )
         .await??;
 
@@ -98,6 +111,7 @@ mod test {
         let bytes = decompress_async(
             document_bytes_raw("openshift-container-storage-4.8.z.json.xz").await?,
             None,
+            0,
         )
         .await??;
 
@@ -115,6 +129,7 @@ mod test {
         let bytes = decompress_async(
             document_bytes_raw("openshift-container-storage-4.8.z.json.xz").await?,
             Some(ContentType::json()),
+            0,
         )
         .await??;
 
@@ -136,6 +151,7 @@ mod test {
         let result = decompress_async(
             document_bytes_raw("openshift-container-storage-4.8.z.json.xz").await?,
             Some(ContentType("application/json+bzip2".parse().unwrap())),
+            0,
         )
         .await?;
 
@@ -153,6 +169,7 @@ mod test {
         let bytes = decompress_async(
             document_bytes_raw("openshift-container-storage-4.8.z.json.xz").await?,
             Some(ContentType("application/json+xz".parse().unwrap())),
+            0,
         )
         .await??;
 

common/src/model/bytesize.rs

@@ -63,6 +63,12 @@ impl From<usize> for BinaryByteSize {
     }
 }
 
+impl From<BinaryByteSize> for usize {
+    fn from(value: BinaryByteSize) -> Self {
+        value.0 .0 as usize
+    }
+}
+
 impl Deref for BinaryByteSize {
     type Target = ByteSize;
 

modules/fundamental/src/advisory/endpoints/config.rs

@@ -0,0 +1,5 @@
+#[derive(Clone, Debug, Eq, PartialEq, Default)]
+pub struct Config {
+    /// An upload limit in bytes. Zero meaning "unlimited".
+    pub upload_limit: usize,
+}

modules/fundamental/src/advisory/endpoints/mod.rs

@@ -1,3 +1,4 @@
+mod config;
 mod label;
 #[cfg(test)]
 mod test;
@@ -6,6 +7,7 @@ use crate::{
     advisory::service::AdvisoryService, purl::service::PurlService, Error, Error::Internal,
 };
 use actix_web::{delete, get, http::header, post, web, HttpResponse, Responder};
+use config::Config;
 use futures_util::TryStreamExt;
 use std::str::FromStr;
 use trustify_common::{
@@ -19,11 +21,12 @@ use trustify_module_ingestor::service::{Format, IngestorService};
 use trustify_module_storage::service::StorageBackend;
 use utoipa::{IntoParams, OpenApi};
 
-pub fn configure(config: &mut web::ServiceConfig, db: Database) {
+pub fn configure(config: &mut web::ServiceConfig, db: Database, upload_limit: usize) {
     let advisory_service = AdvisoryService::new(db);
 
     config
         .app_data(web::Data::new(advisory_service))
+        .app_data(web::Data::new(Config { upload_limit }))
         .service(all)
         .service(get)
         .service(delete)
@@ -171,11 +174,12 @@ struct UploadParams {
 /// Upload a new advisory
 pub async fn upload(
     service: web::Data<IngestorService>,
+    config: web::Data<Config>,
     web::Query(UploadParams { issuer, labels }): web::Query<UploadParams>,
     content_type: Option<web::Header<header::ContentType>>,
     bytes: web::Bytes,
 ) -> Result<impl Responder, Error> {
-    let bytes = decompress_async(bytes, content_type.map(|ct| ct.0)).await??;
+    let bytes = decompress_async(bytes, content_type.map(|ct| ct.0), config.upload_limit).await??;
     let result = service
         .ingest(&bytes, Format::Advisory, labels, issuer)
         .await?;

modules/fundamental/src/advisory/endpoints/test.rs

@@ -1,6 +1,6 @@
 use crate::{
     advisory::model::{AdvisoryDetails, AdvisorySummary},
-    test::{caller, CallService},
+    test::caller,
 };
 use actix_http::StatusCode;
 use actix_web::test::TestRequest;
@@ -18,7 +18,7 @@ use trustify_cvss::cvss3::{
 };
 use trustify_entity::labels::Labels;
 use trustify_module_ingestor::{graph::advisory::AdvisoryInformation, model::IngestResult};
-use trustify_test_context::{document_bytes, TrustifyContext};
+use trustify_test_context::{call::CallService, document_bytes, TrustifyContext};
 use uuid::Uuid;
 
 #[test_context(TrustifyContext)]

modules/fundamental/src/endpoints.rs

@@ -4,27 +4,27 @@ use trustify_module_ingestor::graph::Graph;
 use trustify_module_ingestor::service::IngestorService;
 use trustify_module_storage::service::dispatch::DispatchBackend;
 
+#[derive(Clone, Debug, Eq, PartialEq, Default)]
+pub struct Config {
+    pub sbom_upload_limit: usize,
+    pub advisory_upload_limit: usize,
+}
+
 pub fn configure(
-    config: &mut web::ServiceConfig,
+    svc: &mut web::ServiceConfig,
+    config: Config,
     db: Database,
     storage: impl Into<DispatchBackend>,
 ) {
     let ingestor_service = IngestorService::new(Graph::new(db.clone()), storage);
-    config.app_data(web::Data::new(ingestor_service));
-
-    crate::advisory::endpoints::configure(config, db.clone());
-
-    crate::license::endpoints::configure(config, db.clone());
-
-    crate::organization::endpoints::configure(config, db.clone());
-
-    crate::purl::endpoints::configure(config, db.clone());
-
-    crate::product::endpoints::configure(config, db.clone());
-
-    crate::sbom::endpoints::configure(config, db.clone());
-
-    crate::vulnerability::endpoints::configure(config, db.clone());
-
-    crate::weakness::endpoints::configure(config, db.clone());
+    svc.app_data(web::Data::new(ingestor_service));
+
+    crate::advisory::endpoints::configure(svc, db.clone(), config.advisory_upload_limit);
+    crate::license::endpoints::configure(svc, db.clone());
+    crate::organization::endpoints::configure(svc, db.clone());
+    crate::purl::endpoints::configure(svc, db.clone());
+    crate::product::endpoints::configure(svc, db.clone());
+    crate::sbom::endpoints::configure(svc, db.clone(), config.sbom_upload_limit);
+    crate::vulnerability::endpoints::configure(svc, db.clone());
+    crate::weakness::endpoints::configure(svc, db.clone());
 }

modules/fundamental/src/error.rs

@@ -65,6 +65,9 @@ impl ResponseError for Error {
                 HttpResponse::UnsupportedMediaType()
                     .json(ErrorInformation::new("UnsupportedCompression", self))
             }
+            Error::Compression(decompress::Error::PayloadTooLarge) => {
+                HttpResponse::PayloadTooLarge().json(ErrorInformation::new("PayloadTooLarge", self))
+            }
             Error::Compression(err) => {
                 HttpResponse::BadRequest().json(ErrorInformation::new("CompressionError", err))
             }

modules/fundamental/src/lib.rs

@@ -15,7 +15,7 @@ pub mod openapi;
 pub use openapi::openapi;
 
 pub mod endpoints;
-pub use endpoints::configure;
+pub use endpoints::{configure, Config};
 
 pub mod error;
 

modules/fundamental/src/license/endpoints/test.rs

@@ -1,12 +1,12 @@
 use crate::license::model::{
     LicenseDetailsPurlSummary, LicenseSummary, SpdxLicenseDetails, SpdxLicenseSummary,
 };
-use crate::test::{caller, CallService};
+use crate::test::caller;
 use actix_web::test::TestRequest;
 use test_context::test_context;
 use test_log::test;
 use trustify_common::model::PaginatedResults;
-use trustify_test_context::TrustifyContext;
+use trustify_test_context::{call::CallService, TrustifyContext};
 
 #[test_context(TrustifyContext)]
 #[test(actix_web::test)]

modules/fundamental/src/organization/endpoints/test.rs

@@ -1,4 +1,4 @@
-use crate::test::{caller, CallService};
+use crate::test::caller;
 use actix_web::cookie::time::OffsetDateTime;
 use actix_web::test::TestRequest;
 use jsonpath_rust::JsonPathQuery;
@@ -10,7 +10,7 @@ use trustify_common::db::Transactional;
 use trustify_common::hashing::Digests;
 use trustify_common::model::Paginated;
 use trustify_module_ingestor::graph::advisory::AdvisoryInformation;
-use trustify_test_context::TrustifyContext;
+use trustify_test_context::{call::CallService, TrustifyContext};
 
 #[test_context(TrustifyContext)]
 #[test(actix_web::test)]

modules/fundamental/src/product/endpoints/test.rs

@@ -1,4 +1,4 @@
-use crate::test::{caller, CallService};
+use crate::test::caller;
 use actix_http::StatusCode;
 use actix_web::test::TestRequest;
 use jsonpath_rust::JsonPathQuery;
@@ -8,7 +8,7 @@ use test_log::test;
 use trustify_common::db::query::Query;
 use trustify_common::model::Paginated;
 use trustify_module_ingestor::graph::product::ProductInformation;
-use trustify_test_context::TrustifyContext;
+use trustify_test_context::{call::CallService, TrustifyContext};
 
 #[test_context(TrustifyContext)]
 #[test(actix_web::test)]

modules/fundamental/src/purl/endpoints/test.rs

@@ -4,7 +4,7 @@ use crate::purl::model::details::versioned_purl::VersionedPurlDetails;
 use crate::purl::model::summary::base_purl::{BasePurlSummary, PaginatedBasePurlSummary};
 use crate::purl::model::summary::purl::PaginatedPurlSummary;
 use crate::purl::model::summary::r#type::TypeSummary;
-use crate::test::{caller, CallService};
+use crate::test::caller;
 use actix_web::test::TestRequest;
 use serde_json::Value;
 use std::str::FromStr;
@@ -14,7 +14,7 @@ use trustify_common::db::Transactional;
 use trustify_common::model::PaginatedResults;
 use trustify_common::purl::Purl;
 use trustify_module_ingestor::graph::Graph;
-use trustify_test_context::TrustifyContext;
+use trustify_test_context::{call::CallService, TrustifyContext};
 
 async fn setup(graph: &Graph) -> Result<(), anyhow::Error> {
     let log4j = graph

modules/fundamental/src/sbom/endpoints/config.rs

@@ -0,0 +1,5 @@
+#[derive(Clone, Debug, Eq, PartialEq, Default)]
+pub struct Config {
+    /// An upload limit in bytes. Zero meaning "unlimited".
+    pub upload_limit: usize,
+}

modules/fundamental/src/sbom/endpoints/mod.rs

@@ -1,3 +1,4 @@
+mod config;
 mod label;
 #[cfg(test)]
 mod test;
@@ -11,6 +12,7 @@ use crate::{
     Error::{self, Internal},
 };
 use actix_web::{delete, get, http::header, post, web, HttpResponse, Responder};
+use config::Config;
 use futures_util::TryStreamExt;
 use sea_orm::prelude::Uuid;
 use std::str::FromStr;
@@ -28,11 +30,12 @@ use trustify_module_ingestor::service::{Format, IngestorService};
 use trustify_module_storage::service::StorageBackend;
 use utoipa::OpenApi;
 
-pub fn configure(config: &mut web::ServiceConfig, db: Database) {
+pub fn configure(config: &mut web::ServiceConfig, db: Database, upload_limit: usize) {
     let sbom_service = SbomService::new(db);
 
     config
         .app_data(web::Data::new(sbom_service))
+        .app_data(web::Data::new(Config { upload_limit }))
         .service(all)
         .service(all_related)
         .service(get)
@@ -386,11 +389,12 @@ struct UploadQuery {
 /// Upload a new SBOM
 pub async fn upload(
     service: web::Data<IngestorService>,
+    config: web::Data<Config>,
     web::Query(UploadQuery { labels }): web::Query<UploadQuery>,
     content_type: Option<web::Header<header::ContentType>>,
     bytes: web::Bytes,
 ) -> Result<impl Responder, Error> {
-    let bytes = decompress_async(bytes, content_type.map(|ct| ct.0)).await??;
+    let bytes = decompress_async(bytes, content_type.map(|ct| ct.0), config.upload_limit).await??;
     let result = service.ingest(&bytes, Format::SBOM, labels, None).await?;
     log::info!("Uploaded SBOM: {}", result.id);
     Ok(HttpResponse::Created().json(result))