diff --git a/Cargo.lock b/Cargo.lock index 48105e8..7d9bc6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -187,9 +187,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.82" +version = "0.1.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a27b8a3a6e1a44fa4c8baf1f653e4172e81486d4941f2237e20dc2d0cf4ddff1" +checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", @@ -254,7 +254,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper 1.0.1", "tokio", - "tower 0.5.1", + "tower", "tower-layer", "tower-service", "tracing", @@ -768,9 +768,9 @@ dependencies = [ [[package]] name = "fdeflate" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f9bfee30e4dedf0ab8b422f03af778d9612b63f502710fc500a334ebe2de645" +checksum = "d8090f921a24b04994d9929e204f50b498a33ea6ba559ffaa05e04f7ee7fb5ab" dependencies = [ "simd-adler32", ] @@ -1159,9 +1159,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da62f120a8a37763efb0cf8fdf264b884c7b8b9ac8660b900c8661030c00e6ba" +checksum = "41296eb09f183ac68eec06e03cdbea2e759633d4067b2f6552fc2e009bcad08b" dependencies = [ "bytes", "futures-util", @@ -1170,7 +1170,6 @@ dependencies = [ "hyper", "pin-project-lite", "tokio", - "tower 0.4.13", "tower-service", ] @@ -1400,9 +1399,9 @@ checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8" [[package]] name = "libc" -version = "0.2.158" +version = "0.2.159" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" +checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" [[package]] name = "libdeflate-sys" @@ -1835,26 +1834,6 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" -[[package]] -name = "pin-project" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.77", -] - [[package]] name = "pin-project-lite" version = "0.2.14" @@ -2189,9 +2168,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.4" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0884ad60e090bf1345b93da0a5de8923c93884cd03f40dfcfddd3b4bee661853" +checksum = "355ae415ccd3a04315d3f8246e86d67689ea74d88d915576e1589a351062a13b" dependencies = [ "bitflags 2.6.0", ] @@ -2387,9 +2366,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.7" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb5b1b31579f3811bf615c144393417496f152e12ac8b7663bf664f4a815306d" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" dependencies = [ "serde", ] @@ -2806,9 +2785,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.21" +version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b072cee73c449a636ffd6f32bd8de3a9f7119139aff882f44943ce2986dc5cf" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ "indexmap", "serde", @@ -2817,22 +2796,6 @@ dependencies = [ "winnow", ] -[[package]] -name = "tower" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" -dependencies = [ - "futures-core", - "futures-util", - "pin-project", - "pin-project-lite", - "tokio", - "tower-layer", - "tower-service", - "tracing", -] - [[package]] name = "tower" version = "0.5.1" @@ -2851,9 +2814,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.5.2" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" +checksum = "8437150ab6bbc8c5f0f519e3d5ed4aa883a83dd4cdd3d1b21f9482936046cb97" dependencies = [ "async-compression", "bitflags 2.6.0", @@ -2861,11 +2824,10 @@ dependencies = [ "futures-core", "http", "http-body", - "http-body-util", "pin-project-lite", "tokio", "tokio-util", - "tower 0.4.13", + "tower", "tower-layer", "tower-service", "tracing", @@ -3029,7 +2991,7 @@ dependencies = [ "time", "tokio", "tokio-stream", - "tower 0.5.1", + "tower", "tower-http", "tracing", "tracing-subscriber", @@ -3399,9 +3361,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.6.18" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f" +checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" dependencies = [ "memchr", ] diff --git a/crates/utiles-oxipng/Cargo.toml b/crates/utiles-oxipng/Cargo.toml index 8f41db5..b0fbbff 100644 --- a/crates/utiles-oxipng/Cargo.toml +++ b/crates/utiles-oxipng/Cargo.toml @@ -15,7 +15,6 @@ clap = { workspace = true, features = ["derive", "color", "wrap_help"] } futures.workspace = true indicatif.workspace = true oxipng = { version = "9.1.2", features = [] } -#oxipng = { version = "9.1.2", features = ["parallel"] } size.workspace = true tokio = { workspace = true, features = ["fs"] } tokio-stream.workspace = true diff --git a/crates/utiles/Cargo.toml b/crates/utiles/Cargo.toml index a046bb0..719d709 100644 --- a/crates/utiles/Cargo.toml +++ b/crates/utiles/Cargo.toml @@ -25,17 +25,17 @@ required-features = [ [dependencies] anyhow.workspace = true async-sqlite = { workspace = true, features = ["bundled", "functions", "trace", "blob"] } -async-trait = "0.1.80" -axum = { version = "0.7.5", features = ["tokio", "json", "macros"] } +async-trait = "0.1.83" +axum = { version = "0.7.6", features = ["tokio", "json", "macros"] } chrono = "0.4.38" clap = { workspace = true, features = ["derive", "color", "wrap_help"], optional = true } fnv = "1.0.7" -futures = "0.3.29" +futures = "0.3.30" geo-types.workspace = true geojson.workspace = true globset = "0.4.15" hex = "0.4.3" -image = "0.25.1" +image = "0.25.2" imagesize = "0.13.0" indicatif.workspace = true indoc = { workspace = true } @@ -49,7 +49,7 @@ rusqlite = { workspace = true, features = ["bundled", "blob", "backup", "functio serde.workspace = true serde_json = { workspace = true, features = ["preserve_order"] } size.workspace = true -sqlite-hashes = { version = "0.7.3", default-features = false, features = ["hex", "window", "md5", "fnv", "xxhash"] } +sqlite-hashes = { version = "0.7.5", default-features = false, features = ["hex", "window", "md5", "fnv", "xxhash"] } strum.workspace = true strum_macros.workspace = true thiserror.workspace = true @@ -58,11 +58,11 @@ time = "0.3.36" tokio = { workspace = true, features = ["fs"] } tokio-stream.workspace = true tower = { version = "0.5.1", features = ["timeout"] } -tower-http = { version = "0.5.1", features = ["trace", "timeout", "add-extension", "util", "request-id", "compression-gzip", "compression-zstd", "async-compression"] } +tower-http = { version = "0.6.1", features = ["trace", "timeout", "add-extension", "util", "request-id", "compression-gzip", "compression-zstd", "async-compression"] } tracing.workspace = true tracing-subscriber = { workspace = true, features = ["fmt", "json", "env-filter", "chrono"] } -utiles-core = { path = "../utiles-core", version = "0.7.0-alpha.1" } -walkdir = "2.4.0" +utiles-core = { path = "../utiles-core", version = "0.7.0-alpha.9" } +walkdir = "2.5.0" xxhash-rust = { workspace = true, features = ["const_xxh3", "const_xxh64", "const_xxh32", "xxh3", "xxh64", "xxh32"] } # deadpool = { version = "0.12.1", features = ["managed"], default-features = false } diff --git a/crates/utiles/src/cli/commands/enumerate.rs b/crates/utiles/src/cli/commands/enumerate.rs index 4c3c767..1c4aaee 100644 --- a/crates/utiles/src/cli/commands/enumerate.rs +++ b/crates/utiles/src/cli/commands/enumerate.rs @@ -6,7 +6,7 @@ use std::io::{BufWriter, Write}; use tokio::task::JoinHandle; use tokio_stream::wrappers::ReceiverStream; use tokio_stream::StreamExt; -use tracing::{debug, error}; +use tracing::debug; async fn enumerate_db( fspath: &str, @@ -27,10 +27,10 @@ async fn enumerate_db( let s = mbt.enumerate_rx(Some(&query))?; let mut tiles = ReceiverStream::new(s); while let Some(tile) = tiles.next().await { - // let tile_str = format!("{} {}", fspath, tile.json_arr()); let tile_str = tformatter.fmt_tile(&tile); if let Err(e) = tx.send(tile_str).await { - return Err(crate::UtilesError::Error(format!("enumerate_db: {e:?}"))); + debug!("recv dropped: {:?}", e); + break; } } Ok(()) @@ -55,20 +55,25 @@ pub async fn enumerate_main(args: &EnumerateArgs) -> UtilesResult<()> { let mut buf = BufWriter::with_capacity(32 * 1024, lock); let mut count: usize = 0; while let Some(tile_str) = rx.blocking_recv() { - buf.write_all(tile_str.as_bytes())?; - buf.write_all(b"\n")?; + let tile_str_newline = format!("{tile_str}\n"); + + if let Err(e) = buf.write_all(tile_str_newline.as_bytes()) { + debug!("write_all err: {:?}", e); + break; + } count += 1; if count % 1024 == 0 { - buf.flush()?; if let Err(e) = buf.flush() { - error!("write_task: {:?}", e); + debug!("flushing err: {:?}", e); break; } } } // flush remaining - buf.flush()?; + if let Err(e) = buf.flush() { + debug!("final flush err: {:?}", e); + } Ok(()) }); let tfilter = args.filter_args.tiles_filter_maybe(); diff --git a/crates/utiles/src/mbt/tiles_stream.rs b/crates/utiles/src/mbt/tiles_stream.rs index 63d3d5c..b69a4d3 100644 --- a/crates/utiles/src/mbt/tiles_stream.rs +++ b/crates/utiles/src/mbt/tiles_stream.rs @@ -1,6 +1,6 @@ use rusqlite::Connection; use tokio_stream::wrappers::ReceiverStream; -use tracing::{error, warn}; +use tracing::{debug, error, warn}; use utiles_core::prelude::*; @@ -85,20 +85,32 @@ pub fn make_enumerate_rx( let z_column = s.column_index("zoom_level")?; let x_column = s.column_index("tile_column")?; let y_column = s.column_index("tile_row")?; + let tx = tx.clone(); let tiles_iters = s.query_map(rusqlite::params![], |row| { let z: u8 = row.get(z_column)?; let x: u32 = row.get(x_column)?; let yup: u32 = row.get(y_column)?; let tile = utile_yup!(x, yup, z); - let tx = tx.clone(); if let Err(e) = tx.blocking_send(tile) { - warn!("Blocking send error: {:?}", e); + debug!("Blocking send error: {:?}", e); + Ok(false) + } else { + Ok(true) } - Ok(()) })?; // Consume the iterator for row in tiles_iters { let _ = row; + match row { + Ok(true) => {} + Ok(false) => { + break; + } + Err(e) => { + error!("enum tiles iter error: {:?}", e); + break; + } + } } Ok(()) })