update changelog + agg-hash speedup (#123)

* lint cleanup and webp cleanup

* mbt lint fix clippy

* fix panic

* use stream for agg-hash!

* clippy so angry

* still making clippy happy

* agg hash uses stream is much much faster

* clean copy

* update'

Cargo.toml

@@ -28,7 +28,7 @@ geojson = "0.24.1"
 indoc = "2.0.5"
 pyo3 = "0.22.0"
 pyo3-build-config = "0.22.0"
-rusqlite = { version = "0.31.0", features = ["bundled", "vtab", "blob"] }
+rusqlite = { version = "0.32.0", features = ["bundled", "vtab", "blob"] }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = { version = "1.0.119", features = ["preserve_order"] }
 size = { version = "=0.5.0-preview2", features = ["default"] }

crates/utiles/Cargo.toml

@@ -18,6 +18,9 @@ path = "src/lib.rs"
 [[bin]]
 name = "utiles"
 path = "src/bin.rs"
+required-features = [
+    #    "cli"
+]
 
 [dependencies]
 geo-types.workspace = true
@@ -55,14 +58,22 @@ walkdir = "2.4.0"
 image = "0.25.1"
 fnv = "1.0.7"
 tokio-stream = "0.1.15"
-oxipng = {version = "9.1.2" , features = []}
-pmtiles = { version = "0.10.0", features = ["mmap-async-tokio", "tilejson"] }
+oxipng = { version = "9.1.2", features = [] }
 indicatif = "0.17.8"
 size.workspace = true
+md-5 = "0.10.6"
+noncrypto-digests = "0.3.2"
+hex = "0.4.3"
+pmtiles = { version = "0.10.0", features = ["mmap-async-tokio", "tilejson"], optional = true }
 
 [dev-dependencies]
 criterion = "0.5.1"
 
 [[bench]]
 name = "bench"
 harness = false
+
+[features]
+default = []
+#cli = ["dep:clap"]
+pmtiles = ["dep:pmtiles"]

crates/utiles/src/cli/args.rs

@@ -1,4 +1,5 @@
 use clap::{Args, Parser, Subcommand};
+use std::path::PathBuf;
 use strum_macros::AsRefStr;
 
 use utiles_core::{
@@ -9,6 +10,7 @@ use crate::cli::commands::dev::DevArgs;
 use crate::cli::commands::serve::ServeArgs;
 use crate::cli::commands::shapes::ShapesArgs;
 use crate::cli::commands::{analyze_main, vacuum_main};
+use crate::copy::CopyConfig;
 use crate::errors::UtilesResult;
 use crate::mbt::hash_types::HashType;
 use crate::mbt::{MbtType, TilesFilter};
@@ -169,7 +171,7 @@ pub struct ParentChildrenArgs {
 
 #[derive(Debug, Parser)]
 pub struct SqliteDbCommonArgs {
-    /// mbtiles filepath
+    /// sqlite filepath
     #[arg(required = true)]
     pub filepath: String,
 
@@ -269,12 +271,18 @@ impl DbCommands {
     }
 }
 
+// #[derive(Debug, Parser)]
+// pub struct SqliteSchemaArgs {
+//     #[command(flatten)]
+//     pub common: SqliteDbCommonArgs,
+// }
+
 #[derive(Debug, Parser)]
 pub struct AnalyzeArgs {
     #[command(flatten)]
     pub common: SqliteDbCommonArgs,
 
-    #[arg(required = false, long, action = clap::ArgAction::SetTrue)]
+    #[arg(required = false, long)]
     pub analysis_limit: Option<usize>,
 }
 
@@ -425,6 +433,10 @@ pub struct OxipngArgs {
     /// n-jobs ~ 0=ncpus (default: max(4, ncpus))
     #[arg(required = false, long, short)]
     pub jobs: Option<u8>,
+
+    /// quiet
+    #[arg(required = false, long, short, action = clap::ArgAction::SetTrue)]
+    pub(crate) quiet: bool,
 }
 
 #[derive(Debug, Parser)]
@@ -435,6 +447,14 @@ pub struct WebpifyArgs {
     /// destination dataset fspath (mbtiles, dirpath)
     #[arg(required = true)]
     pub dst: String,
+
+    /// n-jobs ~ 0=ncpus (default: max(4, ncpus))
+    #[arg(required = false, long, short)]
+    pub jobs: Option<u8>,
+
+    /// quiet
+    #[arg(required = false, long, short, action = clap::ArgAction::SetTrue)]
+    pub(crate) quiet: bool,
 }
 
 #[derive(Debug, Subcommand)]
@@ -871,3 +891,24 @@ impl CopyArgs {
         }
     }
 }
+
+impl From<&CopyArgs> for CopyConfig {
+    fn from(args: &CopyArgs) -> CopyConfig {
+        let dbtype = args.dbtype.as_ref().map(|dbtype| dbtype.into());
+        CopyConfig {
+            src: PathBuf::from(&args.src),
+            dst: PathBuf::from(&args.dst),
+            zset: args.zoom_set(),
+            zooms: args.zooms(),
+            verbose: true,
+            bboxes: args.bboxes(),
+            bounds_string: args.bounds(),
+            force: false,
+            dryrun: false,
+            jobs: args.jobs,
+            istrat: InsertStrategy::from(args.conflict),
+            hash: args.hash,
+            dbtype,
+        }
+    }
+}

crates/utiles/src/cli/commands/agg_hash.rs

@@ -1,18 +1,26 @@
 use crate::cli::args::AggHashArgs;
 use crate::errors::UtilesResult;
 use crate::mbt::hash_types::HashType;
-use crate::mbt::mbt_agg_tiles_hash;
-use crate::sqlite::AsyncSqliteConn;
+use crate::mbt::mbt_agg_tiles_hash_stream;
 use crate::utilesqlite::{MbtilesAsync, MbtilesAsyncSqliteClient};
 
+// pub async fn agg_hash_main(args: &AggHashArgs) -> UtilesResult<()> {
+//     let mbt = MbtilesAsyncSqliteClient::open_readonly(&args.common.filepath).await?;
+//     mbt.register_utiles_sqlite_functions().await?;
+//     let hash_type = args.hash.unwrap_or(HashType::Md5);
+//     let filter = args.filter_args.tiles_filter_maybe();
+//     let result = mbt
+//         .conn(move |c| Ok(mbt_agg_tiles_hash(c, hash_type, None, &filter)))
+//         .await??;
+//     println!("{}", serde_json::to_string_pretty(&result)?);
+//     Ok(())
+// }
 pub async fn agg_hash_main(args: &AggHashArgs) -> UtilesResult<()> {
     let mbt = MbtilesAsyncSqliteClient::open_readonly(&args.common.filepath).await?;
     mbt.register_utiles_sqlite_functions().await?;
     let hash_type = args.hash.unwrap_or(HashType::Md5);
     let filter = args.filter_args.tiles_filter_maybe();
-    let result = mbt
-        .conn(move |c| Ok(mbt_agg_tiles_hash(c, hash_type, None, &filter)))
-        .await??;
+    let result = mbt_agg_tiles_hash_stream(&mbt, hash_type, None, &filter).await?;
     println!("{}", serde_json::to_string_pretty(&result)?);
     Ok(())
 }

crates/utiles/src/cli/commands/db.rs

@@ -1,4 +1,3 @@
-#![allow(dead_code)]
 use tracing::{info, trace};
 
 use crate::cli::args::AnalyzeArgs;
@@ -25,3 +24,12 @@ pub async fn analyze_main(args: &AnalyzeArgs) -> UtilesResult<()> {
     info!("Analyze time: {}ms", analyze_time_ms);
     Ok(())
 }
+
+//
+// pub async fn schema_main(args: &SqliteSchemaArgs) -> UtilesResult<()> {
+//     info!("Schema for sqlite file: {}", args.common.filepath);
+//     let db = SqliteDbAsyncClient::open_existing(&args.common.filepath, None).await?;
+//     let schema = db.schema().await?;
+//     println!("{}", schema);
+//     Ok(())
+// }

crates/utiles/src/cli/commands/dev.rs

@@ -2,10 +2,6 @@ use clap::Parser;
 use tracing::{debug, info, warn};
 
 use crate::errors::UtilesResult;
-use crate::mbt::hash_types::HashType;
-use crate::mbt::mbt_agg_tiles_hash;
-use crate::utilesqlite::mbtiles::add_functions;
-use crate::utilesqlite::Mbtiles;
 
 /// ██╗   ██╗████████╗██╗██╗     ███████╗███████╗      ██████╗ ███████╗██╗   ██╗
 /// ██║   ██║╚══██╔══╝██║██║     ██╔════╝██╔════╝      ██╔══██╗██╔════╝██║   ██║
@@ -20,29 +16,29 @@ pub struct DevArgs {
     fspath: Option<String>,
 }
 
-fn _timing_agg_tiles_hash(filepath: &str) -> UtilesResult<()> {
-    let mbt = Mbtiles::open(filepath)?;
-    add_functions(&mbt.conn)?;
-    let hashes = vec![
-        HashType::Xxh3_128,
-        HashType::Xxh3_64,
-        HashType::Xxh64,
-        HashType::Xxh32,
-        HashType::Fnv1a,
-        HashType::Md5,
-    ];
-    for hash in hashes {
-        let start_time = std::time::Instant::now();
-        let agg_tile_hash = mbt_agg_tiles_hash(&mbt.conn, hash, None, &None)?;
-        let elapsed = start_time.elapsed();
-        debug!("---------------------");
-        debug!("hash: {:?}, agg_tile_hash: {:?}", hash, agg_tile_hash);
-        debug!("agg_tile_hash: {:?}", agg_tile_hash);
-        debug!("elapsed: {:?}", elapsed);
-    }
-    Ok(())
-}
-
+// fn _timing_agg_tiles_hash(filepath: &str) -> UtilesResult<()> {
+//     let mbt = Mbtiles::open(filepath)?;
+//     add_functions(&mbt.conn)?;
+//     let hashes = vec![
+//         HashType::Xxh3_128,
+//         HashType::Xxh3_64,
+//         HashType::Xxh64,
+//         HashType::Xxh32,
+//         HashType::Fnv1a,
+//         HashType::Md5,
+//     ];
+//     for hash in hashes {
+//         let start_time = std::time::Instant::now();
+//         let agg_tile_hash = mbt_agg_tiles_hash_stream(&mbt.conn, hash, None, &None)?;
+//         let elapsed = start_time.elapsed();
+//         debug!("---------------------");
+//         debug!("hash: {:?}, agg_tile_hash: {:?}", hash, agg_tile_hash);
+//         debug!("agg_tile_hash: {:?}", agg_tile_hash);
+//         debug!("elapsed: {:?}", elapsed);
+//     }
+//     Ok(())
+// }
+//
 #[allow(clippy::unused_async)]
 async fn dev(args: DevArgs) -> UtilesResult<()> {
     // DEV START

crates/utiles/src/cli/commands/oxipng.rs

@@ -21,7 +21,7 @@ pub async fn oxipng_main(args: OxipngArgs) -> UtilesResult<()> {
     let mbt_metadata = mbt.metadata_rows().await?;
     let dst_mbtiles = Mbtiles::open_new(args.dst, None)?;
     dst_mbtiles.metadata_set_many(&mbt_metadata)?;
-    let tiles_stream = make_tiles_stream(&mbt)?;
+    let tiles_stream = make_tiles_stream(&mbt, None)?;
 
     let (tx_writer, rx_writer) = tokio::sync::mpsc::channel(100);
     let start_time = std::time::Instant::now();
@@ -54,7 +54,8 @@ pub async fn oxipng_main(args: OxipngArgs) -> UtilesResult<()> {
                         Ok(oxipngify_res) => match oxipngify_res {
                             Ok(img_result) => {
                                 let final_size = img_result.len();
-                                let size_diff = initial_size - final_size;
+                                let size_diff =
+                                    (initial_size as i64) - (final_size as i64);
                                 debug!("size_diff: {}", size_diff);
                                 let send_res = tx_writer.send((tile, img_result)).await;
 
@@ -83,6 +84,9 @@ pub async fn oxipng_main(args: OxipngArgs) -> UtilesResult<()> {
         let pb_style = ProgressStyle::with_template(
             "[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}",
         );
+        if args.quiet {
+            pb.set_draw_target(indicatif::ProgressDrawTarget::hidden());
+        }
         match pb_style {
             Err(e) => {
                 warn!("pb_style error: {:?}", e);
@@ -107,7 +111,7 @@ pub async fn oxipng_main(args: OxipngArgs) -> UtilesResult<()> {
     });
 
     let (cruncher_res, writer_res, progress_res) =
-        join!(proc_future, writer.write_batched(), progress_future);
+        join!(proc_future, writer.write(), progress_future);
     let elapsed = start_time.elapsed();
     info!("elapsed: {:?}", elapsed);
     cruncher_res?;

crates/utiles/src/cli/commands/tilejson.rs

@@ -16,3 +16,35 @@ pub async fn tilejson_main(args: &TilejsonArgs) -> UtilesResult<()> {
     println!("{s}");
     Ok(())
 }
+
+// use tracing::debug;
+//
+// use crate::cli::args::TilejsonArgs;
+// use crate::errors::UtilesResult;
+// use crate::pmt::fspath2pmtilejson;
+// use crate::utilejson::tilejson_stringify;
+// use crate::utilesqlite::{MbtilesAsync, MbtilesAsyncSqliteClient};
+//
+// pub async fn tilejson_main(args: &TilejsonArgs) -> UtilesResult<()> {
+//     debug!("tilejson: {}", args.common.filepath);
+//     // if it ends with .pmtiles, use pmtiles else use mbtiles...
+//     let mut tj = if args.common.filepath.ends_with(".pmtiles") {
+//         // pmtiles
+//         fspath2pmtilejson(&args.common.filepath).await?
+//         // let reader = AsyncPmTilesReader::new_with_path(&args.common.filepath).await?;
+//         // reader.parse_tilejson(vec![]).await?
+//     } else {
+//         // mbtiles
+//         let mbt = MbtilesAsyncSqliteClient::open_readonly(&args.common.filepath).await?;
+//         let tj = mbt.tilejson().await?;
+//         tj
+//     };
+//     // let mbt = MbtilesAsyncSqliteClient::open_readonly(&args.common.filepath).await?;
+//     // let mut tj = mbt.tilejson().await?;
+//     if !args.tilestats {
+//         tj.other.remove("tilestats");
+//     }
+//     let s = tilejson_stringify(&tj, Option::from(!args.common.min));
+//     println!("{s}");
+//     Ok(())
+// }