From 7117683b8674b48a5191df9120f632ebbe5fc58d Mon Sep 17 00:00:00 2001 From: jesse Date: Thu, 25 Jul 2024 09:20:36 -0700 Subject: [PATCH] Jkrdev (#124) * fix bbox parsing error for ci in parse_ext * oxi png own bin/crate * main! * moved metadata stuff around more * alrighty oxipng removed from main utiles cli' * update async-sqlite! GO FOWLSTACK --- .gitignore | 19 +- CHANGELOG.md | 2 + Cargo.lock | 24 ++- Cargo.toml | 11 +- crates/utiles-core/Cargo.toml | 1 + crates/utiles-core/src/parsing.rs | 10 +- crates/utiles-oxipng/Cargo.toml | 22 ++ crates/utiles-oxipng/README.md | 4 + crates/utiles-oxipng/src/main.rs | 200 ++++++++++++++++++ crates/utiles/Cargo.toml | 8 +- crates/utiles/src/cli/args.rs | 53 +++-- crates/utiles/src/cli/commands/agg_hash.rs | 6 +- crates/utiles/src/cli/commands/contains.rs | 4 +- crates/utiles/src/cli/commands/copy.rs | 1 - crates/utiles/src/cli/commands/metadata.rs | 6 +- crates/utiles/src/cli/commands/mod.rs | 4 +- crates/utiles/src/cli/commands/optimize.rs | 5 +- crates/utiles/src/cli/commands/oxipng.rs | 5 +- crates/utiles/src/cli/commands/tilejson.rs | 6 +- crates/utiles/src/cli/commands/touch.rs | 5 +- crates/utiles/src/cli/commands/update.rs | 10 +- crates/utiles/src/cli/commands/webpify.rs | 9 +- crates/utiles/src/cli/commands/zxyify.rs | 5 +- crates/utiles/src/cli/entry.rs | 13 +- crates/utiles/src/copy/cfg.rs | 2 +- crates/utiles/src/copy/pasta.rs | 30 +-- crates/utiles/src/copy/pyramid.rs | 2 +- crates/utiles/src/copy/unpyramid.rs | 2 +- crates/utiles/src/errors.rs | 7 +- crates/utiles/src/{mbt => }/hash_types.rs | 0 crates/utiles/src/img.rs | 21 +- crates/utiles/src/lager.rs | 7 +- crates/utiles/src/lib.rs | 7 +- crates/utiles/src/lint/mbt_linter.rs | 28 +-- crates/utiles/src/mbt/agg_tiles_hash.rs | 23 +- crates/utiles/src/mbt/info.rs | 4 +- crates/utiles/src/mbt/mbt_stats.rs | 2 +- .../src/{utilesqlite => mbt}/mbtiles.rs | 0 .../mbtiles_async_sqlite.rs | 50 ++--- crates/utiles/src/mbt/metadata/change.rs | 2 +- .../src/mbt/{ => metadata}/metadata2map.rs | 0 crates/utiles/src/mbt/metadata/mod.rs | 11 +- crates/utiles/src/mbt/mod.rs | 17 +- crates/utiles/src/mbt/stream_writer.rs | 2 +- crates/utiles/src/mbt/tiles_stream.rs | 26 ++- crates/utiles/src/server/mod.rs | 8 +- crates/utiles/src/tile_stream.rs | 4 + crates/utiles/src/utilesqlite/mod.rs | 6 - 48 files changed, 472 insertions(+), 222 deletions(-) create mode 100644 crates/utiles-oxipng/Cargo.toml create mode 100644 crates/utiles-oxipng/README.md create mode 100644 crates/utiles-oxipng/src/main.rs rename crates/utiles/src/{mbt => }/hash_types.rs (100%) rename crates/utiles/src/{utilesqlite => mbt}/mbtiles.rs (100%) rename crates/utiles/src/{utilesqlite => mbt}/mbtiles_async_sqlite.rs (94%) rename crates/utiles/src/mbt/{ => metadata}/metadata2map.rs (100%) create mode 100644 crates/utiles/src/tile_stream.rs delete mode 100644 crates/utiles/src/utilesqlite/mod.rs diff --git a/.gitignore b/.gitignore index 7d2a5ceb..48d303ad 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,9 @@ f.cjs herm.* *.db *.db-journal +*.db-wal +*.sqlite-wal +*.mbtiles-wal ################## @@ -127,8 +130,6 @@ bower_components # Compiled binary addons (https://nodejs.org/api/addons.html) build/Release -# Dependency directories -node_modules/ jspm_packages/ # Optional npm cache directory @@ -140,24 +141,15 @@ jspm_packages/ # Optional REPL history .node_repl_history -# Output of 'npm pack' -*.tgz - # Zip files *.zip # Yarn Integrity file .yarn-integrity -# dotenv environment variables file -.env - # next.js build output .next -# OS X temporary files -.DS_Store - # Rush temporary files common/deploy/ common/temp/ @@ -187,8 +179,8 @@ third-party/ deployment.tar.gz target/ -.hypothesis/ -file.py +.hypothesis/ +file.py *.so # testing data @@ -198,3 +190,4 @@ crates/**/*.mbtiles crates/**/*.pmtiles crates/**/*.sqlite crates/**/*.utiles + diff --git a/CHANGELOG.md b/CHANGELOG.md index cc0c7adc..f6291eb6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ ## TODO - Docs/readme update +- Write/copy from planetiler/tippecanoe fails bc distinction between supported write schema and what is a recognized schema(s) +- Move `utiles-oxipng` to own crate/bin ## Unreleased/Future diff --git a/Cargo.lock b/Cargo.lock index 4e9decb4..4d0824e1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2011,7 +2011,7 @@ dependencies = [ [[package]] name = "pyutiles" -version = "0.7.0-alpha.2" +version = "0.7.0-alpha.3" dependencies = [ "pyo3", "pyo3-build-config", @@ -2933,7 +2933,7 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "utiles" -version = "0.7.0-alpha.2" +version = "0.7.0-alpha.3" dependencies = [ "async-sqlite", "async-trait", @@ -2956,7 +2956,6 @@ dependencies = [ "md-5", "noncrypto-digests", "num_cpus", - "oxipng", "pmtiles", "rusqlite", "serde", @@ -2981,14 +2980,31 @@ dependencies = [ [[package]] name = "utiles-core" -version = "0.7.0-alpha.2" +version = "0.7.0-alpha.3" dependencies = [ "fast_hilbert", + "log", "serde", "serde_json", "thiserror", ] +[[package]] +name = "utiles-oxipng" +version = "0.7.0-alpha.3" +dependencies = [ + "anyhow", + "clap", + "futures", + "indicatif", + "oxipng", + "size", + "tokio", + "tokio-stream", + "tracing", + "utiles", +] + [[package]] name = "uuid" version = "1.10.0" diff --git a/Cargo.toml b/Cargo.toml index a1abf236..8cafc2c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,12 +2,12 @@ resolver = "2" members = [ "crates/utiles", - "crates/utiles-core", + "crates/utiles-core", "crates/utiles-oxipng", "utiles-pyo3", ] [workspace.package] -version = "0.7.0-alpha.2" +version = "0.7.0-alpha.3" authors = [ "Jesse Rubin ", "Dan Costello ", @@ -20,11 +20,13 @@ repository = "https://github.com/jessekrubin/utiles" [workspace.dependencies] anyhow = "1.0.75" -# GIT DEP -async-sqlite = { version = "0.3.0", features = ["bundled", "functions", "trace"] } +async-sqlite = { version = "0.3.1", features = ["bundled", "functions", "trace"] } +clap = { version = "4.5.8", features = ["derive", "color", "wrap_help"] } fast_hilbert = "2.0.0" +futures = "0.3.29" geo-types = "0.7.9" geojson = "0.24.1" +indicatif = "0.17.8" indoc = "2.0.5" pyo3 = "0.22.0" pyo3-build-config = "0.22.0" @@ -37,6 +39,7 @@ strum_macros = "0.26.3" thiserror = "1.0.63" tilejson = "0.4.1" tokio = { version = "1.38.1", features = ["full"] } +tokio-stream = "0.1.15" tracing = "0.1.40" tracing-subscriber = { version = "0.3.17", features = ["serde", "serde_json", "env-filter"] } xxhash-rust = { version = "0.8.10", features = ["xxh32", "xxh64", "xxh3", "const_xxh32", "const_xxh64", "const_xxh3"] } diff --git a/crates/utiles-core/Cargo.toml b/crates/utiles-core/Cargo.toml index c43c211a..d89f3d99 100644 --- a/crates/utiles-core/Cargo.toml +++ b/crates/utiles-core/Cargo.toml @@ -17,3 +17,4 @@ fast_hilbert.workspace = true serde.workspace = true serde_json.workspace = true thiserror.workspace = true +log = "0.4.22" diff --git a/crates/utiles-core/src/parsing.rs b/crates/utiles-core/src/parsing.rs index 6f04b466..c9150f7e 100644 --- a/crates/utiles-core/src/parsing.rs +++ b/crates/utiles-core/src/parsing.rs @@ -1,9 +1,8 @@ //! Parsing util(e)ities -use serde_json::Value; - use crate::bbox::BBox; use crate::errors::UtilesCoreResult; use crate::UtilesCoreError; +use serde_json::Value; /// Parse a string into a `BBox` /// @@ -162,7 +161,10 @@ pub fn parse_bbox(string: &str) -> UtilesCoreResult { pub fn parse_bbox_ext(string: &str) -> UtilesCoreResult { // match 'world' or 'planet' // match string/lower - let str_lower = string.to_lowercase(); + let str_lower = string + .trim() + .trim_matches(|c| c == '\'' || c == '"') + .to_lowercase(); let r = match str_lower.as_str() { "world" | "planet" | "all" | "*" => Ok(BBox::new(-180.0, -90.0, 180.0, 90.0)), "n" | "north" => Ok(BBox::new(-180.0, 0.0, 180.0, 90.0)), @@ -173,7 +175,7 @@ pub fn parse_bbox_ext(string: &str) -> UtilesCoreResult { "nw" | "northwest" => Ok(BBox::new(-180.0, 0.0, 0.0, 90.0)), "se" | "southeast" => Ok(BBox::new(0.0, -90.0, 180.0, 0.0)), "sw" | "southwest" => Ok(BBox::new(-180.0, -90.0, 0.0, 0.0)), - _ => parse_bbox(string), + _ => parse_bbox(&str_lower), }; r } diff --git a/crates/utiles-oxipng/Cargo.toml b/crates/utiles-oxipng/Cargo.toml new file mode 100644 index 00000000..1ba8c218 --- /dev/null +++ b/crates/utiles-oxipng/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "utiles-oxipng" +version.workspace = true +authors.workspace = true +documentation.workspace = true +edition.workspace = true +homepage.workspace = true +license.workspace = true +repository.workspace = true +publish = false + +[dependencies] +anyhow.workspace = true +utiles = { path = "../utiles" } +tokio = { workspace = true, features = ["fs"] } +clap = { workspace = true, features = ["derive", "color", "wrap_help"] } +indicatif.workspace = true +oxipng = { version = "9.1.2", features = [] } +tracing.workspace = true +tokio-stream.workspace = true +futures.workspace = true +size.workspace = true diff --git a/crates/utiles-oxipng/README.md b/crates/utiles-oxipng/README.md new file mode 100644 index 00000000..d3823774 --- /dev/null +++ b/crates/utiles-oxipng/README.md @@ -0,0 +1,4 @@ +# `utiles-oxipng` + +Exe/bin/thingy/apparatus that optimizes pngs using `oxipng` within a db that is +"mbtiles-like" (AKA: it can be queried as if it were `*.mbtiles`) diff --git a/crates/utiles-oxipng/src/main.rs b/crates/utiles-oxipng/src/main.rs new file mode 100644 index 00000000..f1f84786 --- /dev/null +++ b/crates/utiles-oxipng/src/main.rs @@ -0,0 +1,200 @@ +use std::time::Duration; + +use anyhow::Result; +use clap::Parser; +use futures::StreamExt; +use tokio::join; +use tokio_stream::wrappers::ReceiverStream; +use tracing::{debug, error, info, warn}; + +use utiles::{ + lager::{init_tracing, LagerConfig}, + mbt::{MbtStreamWriter, MbtWriterStats, Mbtiles, MbtilesAsync, MbtilesClientAsync}, + tile_type::{tiletype, TileType}, + UtilesResult, +}; + +#[derive(Debug, Parser)] +#[command(name = "utiles-oxipng")] +#[command(version = utiles::VERSION)] +#[command(max_term_width = 120)] +#[command(author)] +#[command(about = "oxipng-ify png-format mbtiles", long_about = None)] +struct Cli { + /// debug + #[arg(long, default_value = "false", action = clap::ArgAction::SetTrue)] + pub debug: bool, + + /// mbtiles-like fspath + #[arg(required = true)] + pub src: String, + + /// destination mbtiles fspath + #[arg(required = true)] + pub dst: String, + + /// optimize level + #[arg(required = false, long, short, default_value = "2")] + pub(crate) opt: u8, + + /// optimize alpha channel (default: false) + #[arg(required = false, long, short, action = clap::ArgAction::SetTrue)] + pub(crate) alpha: bool, + + /// palette-reduction optimization (default: false) + #[arg(required = false, long, short, action = clap::ArgAction::SetTrue)] + pub(crate) palette: bool, + + /// n-jobs ~ 0=ncpus (default: max(4, ncpus)) + #[arg(required = false, long, short)] + pub jobs: Option, + + /// quiet + #[arg(required = false, long, short, action = clap::ArgAction::SetTrue)] + pub(crate) quiet: bool, +} + +pub fn oxipngify(data: &[u8], options: &oxipng::Options) -> Result> { + if let TileType::Png = tiletype(data) { + oxipng::optimize_from_memory(data, options).map_err(|e| e.into()) + } else { + warn!("Unsupported image type"); + Ok(data.to_vec()) + } +} + +async fn oxipng_main(args: Cli) -> UtilesResult<()> { + let mbt = MbtilesClientAsync::open_existing(args.src.as_str()).await?; + mbt.assert_mbtiles().await?; + + let total_count = mbt.tiles_count().await?; + let mbt_metadata = mbt.metadata_rows().await?; + let dst_mbtiles = Mbtiles::open_new(args.dst, None)?; + dst_mbtiles.metadata_set_many(&mbt_metadata)?; + let tiles_stream = mbt.tiles_stream(None)?; + + let (tx_writer, rx_writer) = tokio::sync::mpsc::channel(100); + let start_time = std::time::Instant::now(); + let mut writer = MbtStreamWriter { + stream: ReceiverStream::new(rx_writer), + mbt: dst_mbtiles, + stats: MbtWriterStats::default(), + }; + let jobs: usize = args.jobs.unwrap_or(4) as usize; + + let (tx_progress, mut rx_progress) = tokio::sync::mpsc::channel(100); + + let proc_future = tokio::spawn(async move { + // TODO: cli flag for concurrency + tiles_stream + .for_each_concurrent(jobs, |(tile, tile_data)| { + let tx_writer = tx_writer.clone(); + let tx_progress = tx_progress.clone(); + let mut oxipng_options = oxipng::Options::from_preset(args.opt); + if args.alpha { + oxipng_options.optimize_alpha = true; + } + if args.palette { + oxipng_options.palette_reduction = true; + } + + async move { + let initial_size = tile_data.len(); + let blocking_res = tokio::task::spawn_blocking(move || { + oxipngify(&tile_data, &oxipng_options) + }) + .await; + match blocking_res { + Err(je) => { + warn!("join-error: {:?}", je); + } + Ok(oxipngify_res) => match oxipngify_res { + Ok(img_result) => { + let final_size = img_result.len(); + let size_diff = + (initial_size as i64) - (final_size as i64); + debug!("size_diff: {}", size_diff); + let send_res = tx_writer.send((tile, img_result)).await; + + if let Err(e) = send_res { + warn!("send_res: {:?}", e); + } else if let Err(e) = tx_progress.send(size_diff).await + { + warn!("send error tx_progress: {:?}", e); + } + } + Err(e) => { + warn!("webpify_image: {:?}", e); + } + }, + } + } + }) + .await; + }); + + let progress_future = tokio::spawn(async move { + let mut total_size_diff = 0; + let mut processed = 0; + let pb = indicatif::ProgressBar::new(total_count as u64); + pb.set_message("oxipng-ing"); + let pb_style = indicatif::ProgressStyle::with_template( + "[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}", + ); + if args.quiet { + pb.set_draw_target(indicatif::ProgressDrawTarget::hidden()); + } + match pb_style { + Err(e) => { + warn!("pb_style error: {:?}", e); + } + Ok(s) => { + pb.set_style(s); + } + } + pb.enable_steady_tick(Duration::from_millis(100)); + while let Some(size_diff) = rx_progress.recv().await { + total_size_diff += size_diff; + processed += 1; + pb.inc(1); + + let size_saved = size::Size::from_bytes(total_size_diff as u64).to_string(); + pb.set_message(format!("size-diff: {size_saved}")); + } + let total_size_str = size::Size::from_bytes(total_size_diff as u64).to_string(); + pb.finish_with_message(format!( + "Processed {processed} tiles, saved {total_size_str} ({total_size_diff}b)" + )); + }); + + let (cruncher_res, writer_res, progress_res) = + join!(proc_future, writer.write(), progress_future); + let elapsed = start_time.elapsed(); + info!("elapsed: {:?}", elapsed); + cruncher_res?; + writer_res?; + progress_res?; + Ok(()) +} + +#[tokio::main] +async fn main() -> Result<()> { + debug!("utiles-oxipng"); + let args = Cli::parse(); + debug!("args: {:?}", args); + + init_tracing(&LagerConfig { + debug: args.debug, + ..Default::default() + })?; + let res = oxipng_main(args).await; + + // if let Err(e) = res { + // error!("{}", e); + // } + res.map_err(|e| { + error!("{}", e); + e.into() + }) + // Ok(()) +} diff --git a/crates/utiles/Cargo.toml b/crates/utiles/Cargo.toml index 59fc833b..d54c21df 100644 --- a/crates/utiles/Cargo.toml +++ b/crates/utiles/Cargo.toml @@ -31,14 +31,15 @@ strum.workspace = true strum_macros.workspace = true thiserror.workspace = true tilejson.workspace = true +indicatif.workspace = true +tokio-stream.workspace = true utiles-core = { path = "../utiles-core", version = "0.7.0-alpha.1" } xxhash-rust = { workspace = true, features = ["const_xxh3", "const_xxh64", "const_xxh32", "xxh3", "xxh64", "xxh32"] } -# async-sqlite = { workspace = true, features = ["bundled", "functions", "trace", "blob"] } async-trait = "0.1.80" axum = { version = "0.7.5", features = ["tokio", "json", "macros"] } chrono = "0.4.38" -clap = { version = "4.5.8", features = ["derive", "color", "wrap_help"] } +clap = { workspace = true, features = ["derive", "color", "wrap_help"] } colored = "2.1.0" futures = "0.3.29" globset = "0.4.13" @@ -57,9 +58,6 @@ tracing-subscriber = { workspace = true, features = ["fmt", "json", "env-filter" walkdir = "2.4.0" image = "0.25.1" fnv = "1.0.7" -tokio-stream = "0.1.15" -oxipng = { version = "9.1.2", features = [] } -indicatif = "0.17.8" size.workspace = true md-5 = "0.10.6" noncrypto-digests = "0.3.2" diff --git a/crates/utiles/src/cli/args.rs b/crates/utiles/src/cli/args.rs index 690d7571..b8615748 100644 --- a/crates/utiles/src/cli/args.rs +++ b/crates/utiles/src/cli/args.rs @@ -12,7 +12,7 @@ use crate::cli::commands::shapes::ShapesArgs; use crate::cli::commands::{analyze_main, vacuum_main}; use crate::copy::CopyConfig; use crate::errors::UtilesResult; -use crate::mbt::hash_types::HashType; +use crate::hash_types::HashType; use crate::mbt::{MbtType, TilesFilter}; use crate::sqlite::InsertStrategy; use crate::tile_strfmt::TileStringFormatter; @@ -29,7 +29,7 @@ fn about() -> String { } #[derive(Debug, Parser)] -#[command(name = "ut", about = about(), version = VERSION, author, max_term_width = 88)] +#[command(name = "ut", about = about(), version = VERSION, author, max_term_width = 120)] pub struct Cli { /// debug mode (print/log more) #[arg(long, global = true, default_value = "false", action = clap::ArgAction::SetTrue)] @@ -417,27 +417,27 @@ pub struct OptimizeArgs { pub dst: String, } -#[derive(Debug, Parser)] -pub struct OxipngArgs { - #[command(flatten)] - pub common: SqliteDbCommonArgs, - - /// destination dataset fspath (mbtiles, dirpath) - #[arg(required = true)] - pub dst: String, - - /// optimize level - #[arg(required = false, long, short, default_value = "2")] - pub(crate) opt: u8, - - /// n-jobs ~ 0=ncpus (default: max(4, ncpus)) - #[arg(required = false, long, short)] - pub jobs: Option, - - /// quiet - #[arg(required = false, long, short, action = clap::ArgAction::SetTrue)] - pub(crate) quiet: bool, -} +// #[derive(Debug, Parser)] +// pub struct OxipngArgs { +// #[command(flatten)] +// pub common: SqliteDbCommonArgs, +// +// /// destination dataset fspath (mbtiles, dirpath) +// #[arg(required = true)] +// pub dst: String, +// +// /// optimize level +// #[arg(required = false, long, short, default_value = "2")] +// pub(crate) opt: u8, +// +// /// n-jobs ~ 0=ncpus (default: max(4, ncpus)) +// #[arg(required = false, long, short)] +// pub jobs: Option, +// +// /// quiet +// #[arg(required = false, long, short, action = clap::ArgAction::SetTrue)] +// pub(crate) quiet: bool, +// } #[derive(Debug, Parser)] pub struct WebpifyArgs { @@ -711,10 +711,9 @@ pub enum Commands { )] Webpify(WebpifyArgs), - /// Optimize pngs with `oxipng` for raster tiles-db - #[command(name = "oxipng", hide = true)] - Oxipng(OxipngArgs), - + // /// Optimize pngs with `oxipng` for raster tiles-db + // #[command(name = "oxipng", hide = true)] + // Oxipng(OxipngArgs), /// Convert raster mbtiles to webp format #[command( name = "optimize", diff --git a/crates/utiles/src/cli/commands/agg_hash.rs b/crates/utiles/src/cli/commands/agg_hash.rs index 4dbb6c7d..b99846af 100644 --- a/crates/utiles/src/cli/commands/agg_hash.rs +++ b/crates/utiles/src/cli/commands/agg_hash.rs @@ -1,8 +1,8 @@ use crate::cli::args::AggHashArgs; use crate::errors::UtilesResult; -use crate::mbt::hash_types::HashType; +use crate::hash_types::HashType; use crate::mbt::mbt_agg_tiles_hash_stream; -use crate::utilesqlite::{MbtilesAsync, MbtilesAsyncSqliteClient}; +use crate::mbt::{MbtilesAsync, MbtilesClientAsync}; // pub async fn agg_hash_main(args: &AggHashArgs) -> UtilesResult<()> { // let mbt = MbtilesAsyncSqliteClient::open_readonly(&args.common.filepath).await?; @@ -16,7 +16,7 @@ use crate::utilesqlite::{MbtilesAsync, MbtilesAsyncSqliteClient}; // Ok(()) // } pub async fn agg_hash_main(args: &AggHashArgs) -> UtilesResult<()> { - let mbt = MbtilesAsyncSqliteClient::open_readonly(&args.common.filepath).await?; + let mbt = MbtilesClientAsync::open_readonly(&args.common.filepath).await?; mbt.register_utiles_sqlite_functions().await?; let hash_type = args.hash.unwrap_or(HashType::Md5); let filter = args.filter_args.tiles_filter_maybe(); diff --git a/crates/utiles/src/cli/commands/contains.rs b/crates/utiles/src/cli/commands/contains.rs index c7b441a5..08bdb178 100644 --- a/crates/utiles/src/cli/commands/contains.rs +++ b/crates/utiles/src/cli/commands/contains.rs @@ -2,7 +2,7 @@ use std::path::Path; use tracing::debug; use crate::errors::UtilesResult; -use crate::utilesqlite::{MbtilesAsync, MbtilesAsyncSqliteClient}; +use crate::mbt::{MbtilesAsync, MbtilesClientAsync}; use utiles_core::LngLat; /// Check if a mbtiles file contains a lnglat @@ -22,7 +22,7 @@ pub async fn contains_main(filepath: &str, lnglat: LngLat) -> UtilesResult<()> { "Not a file: {filepath}", filepath = filepath.display() ); - let mbtiles = MbtilesAsyncSqliteClient::open_existing(filepath).await?; + let mbtiles = MbtilesClientAsync::open_existing(filepath).await?; let bbox = mbtiles.bbox().await?; let contains = bbox.contains_lnglat(&lnglat); debug!("contains: {contains}"); diff --git a/crates/utiles/src/cli/commands/copy.rs b/crates/utiles/src/cli/commands/copy.rs index 70e341f5..6dd838dc 100644 --- a/crates/utiles/src/cli/commands/copy.rs +++ b/crates/utiles/src/cli/commands/copy.rs @@ -3,7 +3,6 @@ use crate::copy::{copy, CopyConfig}; use crate::errors::UtilesResult; use tracing::debug; -#[tracing::instrument] pub async fn copy_main(args: CopyArgs) -> UtilesResult<()> { debug!("copy-args: {:?}", args); let copy_cfg = CopyConfig::from(&args); diff --git a/crates/utiles/src/cli/commands/metadata.rs b/crates/utiles/src/cli/commands/metadata.rs index cf848237..91664f6a 100644 --- a/crates/utiles/src/cli/commands/metadata.rs +++ b/crates/utiles/src/cli/commands/metadata.rs @@ -8,8 +8,8 @@ use crate::mbt::{ metadata2map, metadata2map_val, read_metadata_json, MbtilesMetadataJson, MbtilesMetadataRowParsed, MetadataChange, }; +use crate::mbt::{MbtilesAsync, MbtilesClientAsync}; use crate::sqlite::AsyncSqliteConn; -use crate::utilesqlite::{MbtilesAsync, MbtilesAsyncSqliteClient}; use tracing::warn; use tracing::{debug, info}; @@ -17,7 +17,7 @@ pub async fn metadata_main(args: &MetadataArgs) -> UtilesResult<()> { debug!("meta: {}", args.common.filepath); // check that filepath exists and is file let filepath = Path::new(&args.common.filepath); - let mbtiles = MbtilesAsyncSqliteClient::open_existing(filepath).await?; + let mbtiles = MbtilesClientAsync::open_existing(filepath).await?; let metadata_rows = mbtiles.metadata_rows().await?; let json_val = match (args.raw, args.obj) { (true, true) => { @@ -52,7 +52,7 @@ pub async fn metadata_set_main(args: &MetadataSetArgs) -> UtilesResult<()> { // check that filepath exists and is file let filepath = Path::new(&args.common.filepath); file_exists_err(filepath).await?; - let mbtiles = MbtilesAsyncSqliteClient::open_existing(filepath).await?; + let mbtiles = MbtilesClientAsync::open_existing(filepath).await?; let current_metadata_json = mbtiles.metadata_json().await?; let c = match &args.value { Some(value) => { diff --git a/crates/utiles/src/cli/commands/mod.rs b/crates/utiles/src/cli/commands/mod.rs index a16466b6..fc58393a 100644 --- a/crates/utiles/src/cli/commands/mod.rs +++ b/crates/utiles/src/cli/commands/mod.rs @@ -10,7 +10,7 @@ pub use info::info_main; pub use lint::lint_main; pub use metadata::{metadata_main, metadata_set_main}; pub use optimize::optimize_main; -pub use oxipng::oxipng_main; +// pub use oxipng::oxipng_main; pub use rimraf::rimraf_main; pub use serve::serve_main; pub use shapes::shapes_main; @@ -36,7 +36,7 @@ mod info; pub mod lint; mod metadata; mod optimize; -mod oxipng; +// mod oxipng; pub mod rimraf; pub mod serve; pub mod shapes; diff --git a/crates/utiles/src/cli/commands/optimize.rs b/crates/utiles/src/cli/commands/optimize.rs index 13f34c0b..eb496cf4 100644 --- a/crates/utiles/src/cli/commands/optimize.rs +++ b/crates/utiles/src/cli/commands/optimize.rs @@ -6,14 +6,13 @@ use tracing::{info, warn}; use crate::cli::args::OptimizeArgs; -use crate::utilesqlite::{MbtilesAsync, MbtilesAsyncSqliteClient}; +use crate::mbt::{MbtilesAsync, MbtilesClientAsync}; use crate::UtilesResult; pub async fn optimize_main(args: OptimizeArgs) -> UtilesResult<()> { info!("Optimizing mbtiles file: {}", args.common.filepath); warn!("NOT IMPLEMENTED YET"); - let mbt = - MbtilesAsyncSqliteClient::open_existing(args.common.filepath.as_str()).await?; + let mbt = MbtilesClientAsync::open_existing(args.common.filepath.as_str()).await?; mbt.assert_mbtiles().await?; Ok(()) } diff --git a/crates/utiles/src/cli/commands/oxipng.rs b/crates/utiles/src/cli/commands/oxipng.rs index 6a7c8b06..06182233 100644 --- a/crates/utiles/src/cli/commands/oxipng.rs +++ b/crates/utiles/src/cli/commands/oxipng.rs @@ -1,7 +1,7 @@ use crate::cli::args::OxipngArgs; use crate::img::oxipngify; use crate::mbt::{make_tiles_stream, MbtStreamWriter, MbtWriterStats}; -use crate::utilesqlite::{Mbtiles, MbtilesAsync, MbtilesAsyncSqliteClient}; +use crate::mbt::{Mbtiles, MbtilesAsync, MbtilesClientAsync}; use crate::UtilesResult; use futures::StreamExt; use indicatif; @@ -12,8 +12,7 @@ use tokio_stream::wrappers::ReceiverStream; use tracing::{debug, info, warn}; pub async fn oxipng_main(args: OxipngArgs) -> UtilesResult<()> { - let mbt = - MbtilesAsyncSqliteClient::open_existing(args.common.filepath.as_str()).await?; + let mbt = MbtilesClientAsync::open_existing(args.common.filepath.as_str()).await?; mbt.assert_mbtiles().await?; let total_count = mbt.tiles_count().await?; diff --git a/crates/utiles/src/cli/commands/tilejson.rs b/crates/utiles/src/cli/commands/tilejson.rs index e4bf344c..7fb0ae4b 100644 --- a/crates/utiles/src/cli/commands/tilejson.rs +++ b/crates/utiles/src/cli/commands/tilejson.rs @@ -2,12 +2,12 @@ use tracing::debug; use crate::cli::args::TilejsonArgs; use crate::errors::UtilesResult; +use crate::mbt::{MbtilesAsync, MbtilesClientAsync}; use crate::utilejson::tilejson_stringify; -use crate::utilesqlite::{MbtilesAsync, MbtilesAsyncSqliteClient}; pub async fn tilejson_main(args: &TilejsonArgs) -> UtilesResult<()> { debug!("tilejson: {}", args.common.filepath); - let mbt = MbtilesAsyncSqliteClient::open_readonly(&args.common.filepath).await?; + let mbt = MbtilesClientAsync::open_readonly(&args.common.filepath).await?; let mut tj = mbt.tilejson().await?; if !args.tilestats { tj.other.remove("tilestats"); @@ -23,7 +23,7 @@ pub async fn tilejson_main(args: &TilejsonArgs) -> UtilesResult<()> { // use crate::errors::UtilesResult; // use crate::pmt::fspath2pmtilejson; // use crate::utilejson::tilejson_stringify; -// use crate::utilesqlite::{MbtilesAsync, MbtilesAsyncSqliteClient}; +// use crate::mbt::{MbtilesAsync, MbtilesAsyncSqliteClient}; // // pub async fn tilejson_main(args: &TilejsonArgs) -> UtilesResult<()> { // debug!("tilejson: {}", args.common.filepath); diff --git a/crates/utiles/src/cli/commands/touch.rs b/crates/utiles/src/cli/commands/touch.rs index 7cce620e..5a3f62a4 100644 --- a/crates/utiles/src/cli/commands/touch.rs +++ b/crates/utiles/src/cli/commands/touch.rs @@ -6,9 +6,9 @@ use tracing::{debug, info}; use crate::cli::args::TouchArgs; use crate::errors::UtilesResult; use crate::mbt::MbtType; +use crate::mbt::{MbtilesAsync, MbtilesClientAsync}; use crate::sqlite::is_valid_page_size; use crate::sqlite::{Sqlike3Async, SqliteError}; -use crate::utilesqlite::{MbtilesAsync, MbtilesAsyncSqliteClient}; use crate::UtilesError; fn check_page_size(page_size: i64) -> UtilesResult { @@ -50,8 +50,7 @@ pub async fn touch_main(args: &TouchArgs) -> UtilesResult<()> { } }; let mbtiles = - MbtilesAsyncSqliteClient::open_new(filepath, Some(dbtype)) - .await?; + MbtilesClientAsync::open_new(filepath, Some(dbtype)).await?; info!("Created mbtiles: {:?}", filepath); mbtiles.metadata_set("name", &filename_no_ext).await?; mbtiles.metadata_set("mbtype", dbtype_str.as_str()).await?; diff --git a/crates/utiles/src/cli/commands/update.rs b/crates/utiles/src/cli/commands/update.rs index 8c8590a1..f99303a0 100644 --- a/crates/utiles/src/cli/commands/update.rs +++ b/crates/utiles/src/cli/commands/update.rs @@ -4,12 +4,10 @@ use tracing::{debug, warn}; use crate::cli::args::UpdateArgs; use crate::errors::UtilesResult; +use crate::mbt::mbtiles::{query_distinct_tilesize_fast, query_distinct_tiletype_fast}; use crate::mbt::{DbChangeType, DbChangeset, MetadataChange, MetadataChangeFromTo}; +use crate::mbt::{MbtilesAsync, MbtilesClientAsync}; use crate::sqlite::AsyncSqliteConn; -use crate::utilesqlite::mbtiles::{ - query_distinct_tilesize_fast, query_distinct_tiletype_fast, -}; -use crate::utilesqlite::{MbtilesAsync, MbtilesAsyncSqliteClient}; use crate::UtilesError; pub async fn update_mbtiles( @@ -18,9 +16,9 @@ pub async fn update_mbtiles( ) -> UtilesResult { // check that filepath exists and is file let mbt = if dryrun { - MbtilesAsyncSqliteClient::open_readonly(filepath).await? + MbtilesClientAsync::open_readonly(filepath).await? } else { - MbtilesAsyncSqliteClient::open_existing(filepath).await? + MbtilesClientAsync::open_existing(filepath).await? }; // check if tiles is empty... diff --git a/crates/utiles/src/cli/commands/webpify.rs b/crates/utiles/src/cli/commands/webpify.rs index 96546d83..61804135 100644 --- a/crates/utiles/src/cli/commands/webpify.rs +++ b/crates/utiles/src/cli/commands/webpify.rs @@ -8,13 +8,12 @@ use tracing::{info, warn}; use crate::cli::args::WebpifyArgs; use crate::img::webpify_image; -use crate::mbt::{make_tiles_stream, MbtStreamWriter, MbtWriterStats}; -use crate::utilesqlite::{Mbtiles, MbtilesAsync, MbtilesAsyncSqliteClient}; +use crate::mbt::{MbtStreamWriter, MbtWriterStats}; +use crate::mbt::{Mbtiles, MbtilesAsync, MbtilesClientAsync}; use crate::UtilesResult; pub async fn webpify_main(args: WebpifyArgs) -> UtilesResult<()> { - let mbt = - MbtilesAsyncSqliteClient::open_existing(args.common.filepath.as_str()).await?; + let mbt = MbtilesClientAsync::open_existing(args.common.filepath.as_str()).await?; mbt.assert_mbtiles().await?; let total_count = mbt.tiles_count().await?; @@ -22,7 +21,7 @@ pub async fn webpify_main(args: WebpifyArgs) -> UtilesResult<()> { let dst_mbtiles = Mbtiles::open_new(args.dst, None)?; dst_mbtiles.metadata_set_many(&mbt_metadata)?; dst_mbtiles.metadata_set("format", "webp")?; - let tiles_stream = make_tiles_stream(&mbt, None)?; + let tiles_stream = mbt.tiles_stream(None)?; let (tx_progress, mut rx_progress) = tokio::sync::mpsc::channel(100); let (tx_writer, rx_writer) = tokio::sync::mpsc::channel(100); let start_time = std::time::Instant::now(); diff --git a/crates/utiles/src/cli/commands/zxyify.rs b/crates/utiles/src/cli/commands/zxyify.rs index c5a9cdd6..c1ab4b3b 100644 --- a/crates/utiles/src/cli/commands/zxyify.rs +++ b/crates/utiles/src/cli/commands/zxyify.rs @@ -1,13 +1,12 @@ use crate::cli::args::ZxyifyArgs; use crate::errors::UtilesResult; use crate::mbt::zxyify::unzxyify; +use crate::mbt::{MbtilesAsync, MbtilesClientAsync}; use crate::sqlite::AsyncSqliteConn; -use crate::utilesqlite::{MbtilesAsync, MbtilesAsyncSqliteClient}; #[tracing::instrument] pub async fn zxyify_main(args: ZxyifyArgs) -> UtilesResult<()> { - let mbt = - MbtilesAsyncSqliteClient::open_existing(args.common.filepath.as_str()).await?; + let mbt = MbtilesClientAsync::open_existing(args.common.filepath.as_str()).await?; mbt.assert_mbtiles().await?; if args.rm { diff --git a/crates/utiles/src/cli/entry.rs b/crates/utiles/src/cli/entry.rs index 28857b6d..cdfbad6a 100644 --- a/crates/utiles/src/cli/entry.rs +++ b/crates/utiles/src/cli/entry.rs @@ -2,12 +2,12 @@ use crate::cli::args::{Cli, Commands}; use crate::cli::commands::{ about_main, agg_hash_main, bounding_tile_main, children_main, contains_main, copy_main, dev_main, fmtstr_main, info_main, lint_main, metadata_main, - metadata_set_main, neighbors_main, optimize_main, oxipng_main, parent_main, - pmtileid_main, quadkey_main, rimraf_main, serve_main, shapes_main, tilejson_main, - tiles_main, touch_main, update_main, vacuum_main, webpify_main, zxyify_main, + metadata_set_main, neighbors_main, optimize_main, parent_main, pmtileid_main, + quadkey_main, rimraf_main, serve_main, shapes_main, tilejson_main, tiles_main, + touch_main, update_main, vacuum_main, webpify_main, zxyify_main, }; use crate::errors::UtilesResult; -use crate::lager::{init_tracing, LogConfig}; +use crate::lager::{init_tracing, LagerConfig}; use crate::signal::shutdown_signal; use crate::UtilesError; use clap::{CommandFactory, FromArgMatches}; @@ -83,13 +83,13 @@ pub async fn cli_main_inner(cliopts: Option) -> UtilesResult { // if the command is "dev" init tracing w/ debug let logcfg = if let Commands::Dev(_) = args.command { - LogConfig { + LagerConfig { trace: false, debug: true, json: args.log_json, } } else { - LogConfig { + LagerConfig { trace: args.trace, debug: args.debug, json: args.log_json, @@ -132,7 +132,6 @@ pub async fn cli_main_inner(cliopts: Option) -> UtilesResult { Commands::Shapes(args) => shapes_main(args), Commands::Optimize(args) => optimize_main(args).await, Commands::Webpify(args) => webpify_main(args).await, - Commands::Oxipng(args) => oxipng_main(args).await, // server WIP Commands::Serve(args) => serve_main(args).await, }; diff --git a/crates/utiles/src/copy/cfg.rs b/crates/utiles/src/copy/cfg.rs index 1e11e965..8924d106 100644 --- a/crates/utiles/src/copy/cfg.rs +++ b/crates/utiles/src/copy/cfg.rs @@ -7,7 +7,7 @@ use utiles_core::BBox; use crate::errors::UtilesCopyError; use crate::errors::UtilesResult; -use crate::mbt::hash_types::HashType; +use crate::hash_types::HashType; use crate::mbt::{MbtType, TilesFilter}; use crate::sqlite::InsertStrategy; diff --git a/crates/utiles/src/copy/pasta.rs b/crates/utiles/src/copy/pasta.rs index e61eccc7..95f1297f 100644 --- a/crates/utiles/src/copy/pasta.rs +++ b/crates/utiles/src/copy/pasta.rs @@ -6,8 +6,8 @@ use crate::copy::CopyConfig; use crate::errors::UtilesCopyError; use crate::errors::UtilesResult; use crate::mbt::MbtType; +use crate::mbt::{MbtilesAsync, MbtilesClientAsync}; use crate::sqlite::{AsyncSqliteConn, Sqlike3Async}; -use crate::utilesqlite::{MbtilesAsync, MbtilesAsyncSqliteClient}; use crate::UtilesError; #[derive(Debug)] @@ -18,10 +18,10 @@ pub struct CopyPasta { #[derive(Debug)] pub struct CopyPastaPreflightAnalysis { pub dst_db_type: MbtType, - pub dst_db: MbtilesAsyncSqliteClient, + pub dst_db: MbtilesClientAsync, pub src_db_type: MbtType, - pub src_db: MbtilesAsyncSqliteClient, + pub src_db: MbtilesClientAsync, pub dst_is_new: bool, pub check_conflict: bool, @@ -34,9 +34,9 @@ impl CopyPasta { Ok(Self { cfg }) } - pub async fn get_src_db(&self) -> UtilesResult { + pub async fn get_src_db(&self) -> UtilesResult { // do the thing - let src_db = MbtilesAsyncSqliteClient::open_existing(&self.cfg.src).await?; + let src_db = MbtilesClientAsync::open_existing(&self.cfg.src).await?; debug!("src_db: {:?}", src_db); Ok(src_db) } @@ -45,9 +45,9 @@ impl CopyPasta { pub async fn get_dst_db( &self, dst_db_type: Option, - ) -> UtilesResult<(MbtilesAsyncSqliteClient, bool, MbtType)> { + ) -> UtilesResult<(MbtilesClientAsync, bool, MbtType)> { // if the dst is a file... we gotta get it... - let dst_db_res = MbtilesAsyncSqliteClient::open_existing(&self.cfg.dst).await; + let dst_db_res = MbtilesClientAsync::open_existing(&self.cfg.dst).await; let (dst_db, is_new) = match dst_db_res { Ok(db) => (db, false), Err(e) => { @@ -55,8 +55,8 @@ impl CopyPasta { debug!("Creating new db... {:?}", self.cfg.dst); // type is debug!("dbtype: {:?}", self.cfg.dbtype); - let db = MbtilesAsyncSqliteClient::open_new(&self.cfg.dst, dst_db_type) - .await?; + let db = + MbtilesClientAsync::open_new(&self.cfg.dst, dst_db_type).await?; (db, true) } }; @@ -67,7 +67,7 @@ impl CopyPasta { pub async fn copy_metadata( &self, - dst_db: &MbtilesAsyncSqliteClient, + dst_db: &MbtilesClientAsync, ) -> UtilesResult { let src_db = self.get_src_db().await?; let metadata_rows = src_db.metadata_json().await?.as_obj(); @@ -89,7 +89,7 @@ impl CopyPasta { pub async fn copy_tiles_zbox_flat( &self, - dst_db: &MbtilesAsyncSqliteClient, + dst_db: &MbtilesClientAsync, ) -> UtilesResult { let src_db_name = "src"; let where_clause = self.cfg.mbtiles_sql_where()?; @@ -119,7 +119,7 @@ impl CopyPasta { pub async fn copy_tiles_zbox_hash( &self, - dst_db: &MbtilesAsyncSqliteClient, + dst_db: &MbtilesClientAsync, ) -> UtilesResult { let src_db_name = "src"; let where_clause = self.cfg.mbtiles_sql_where()?; @@ -149,7 +149,7 @@ impl CopyPasta { pub async fn copy_tiles_zbox_norm( &self, - dst_db: &MbtilesAsyncSqliteClient, + dst_db: &MbtilesClientAsync, ) -> UtilesResult { let src_db_name = "src"; let where_clause = self.cfg.mbtiles_sql_where()?; @@ -209,7 +209,7 @@ ON pub async fn copy_tiles_zbox( &self, - dst_db: &MbtilesAsyncSqliteClient, + dst_db: &MbtilesClientAsync, ) -> UtilesResult { debug!("copy tiles zbox"); @@ -271,7 +271,7 @@ ON pub async fn check_conflict( &self, - dst_db: &MbtilesAsyncSqliteClient, + dst_db: &MbtilesClientAsync, ) -> UtilesResult { // do the thing debug!("Check overlapping: {:?}", self.cfg); diff --git a/crates/utiles/src/copy/pyramid.rs b/crates/utiles/src/copy/pyramid.rs index 26a03b0a..74a08dca 100644 --- a/crates/utiles/src/copy/pyramid.rs +++ b/crates/utiles/src/copy/pyramid.rs @@ -12,7 +12,7 @@ use crate::copy::CopyConfig; use crate::errors::UtilesError; use crate::errors::UtilesResult; use crate::mbt::MbtTileRow; -use crate::utilesqlite::Mbtiles; +use crate::mbt::Mbtiles; #[derive(Debug)] pub struct WriterStats { diff --git a/crates/utiles/src/copy/unpyramid.rs b/crates/utiles/src/copy/unpyramid.rs index 721ebf18..589e6ca5 100644 --- a/crates/utiles/src/copy/unpyramid.rs +++ b/crates/utiles/src/copy/unpyramid.rs @@ -11,7 +11,7 @@ use utiles_core::Tile; use crate::copy::CopyConfig; use crate::errors::UtilesResult; use crate::mbt::parse_metadata_json; -use crate::utilesqlite::Mbtiles; +use crate::mbt::Mbtiles; use crate::UtilesError; fn fspath2xyz(path: &Path) -> UtilesResult<(u32, u32, u8)> { diff --git a/crates/utiles/src/errors.rs b/crates/utiles/src/errors.rs index b89b3667..e8f4514e 100644 --- a/crates/utiles/src/errors.rs +++ b/crates/utiles/src/errors.rs @@ -109,10 +109,9 @@ pub enum UtilesError { #[error("tokio::task::JoinError - {0}")] TokioJoinError(#[from] tokio::task::JoinError), - /// Error from `oxipng` - #[error("oxipng::PngError: {0}")] - OxipngError(#[from] oxipng::PngError), - + // /// Error from `oxipng` + // #[error("oxipng::PngError: {0}")] + // OxipngError(#[from] oxipng::PngError), /// Error from `pmtiles` #[cfg(feature = "pmtiles")] #[error("pmtiles error: {0}")] diff --git a/crates/utiles/src/mbt/hash_types.rs b/crates/utiles/src/hash_types.rs similarity index 100% rename from crates/utiles/src/mbt/hash_types.rs rename to crates/utiles/src/hash_types.rs diff --git a/crates/utiles/src/img.rs b/crates/utiles/src/img.rs index b54f7058..be94053b 100644 --- a/crates/utiles/src/img.rs +++ b/crates/utiles/src/img.rs @@ -1,8 +1,11 @@ -use crate::{UtilesError, UtilesResult}; use std::io::Cursor; + use tracing::warn; + use utiles_core::tile_type::{tiletype, TileType}; +use crate::UtilesResult; + pub fn webpify_image(data: &[u8]) -> UtilesResult> { match tiletype(data) { TileType::Webp => Ok(data.to_vec()), @@ -21,14 +24,14 @@ pub fn webpify_image(data: &[u8]) -> UtilesResult> { } } -pub fn oxipngify(data: &[u8], options: &oxipng::Options) -> UtilesResult> { - if let TileType::Png = tiletype(data) { - oxipng::optimize_from_memory(data, options).map_err(UtilesError::OxipngError) - } else { - warn!("Unsupported image type"); - Ok(data.to_vec()) - } -} +// pub fn oxipngify(data: &[u8], options: &oxipng::Options) -> UtilesResult> { +// if let TileType::Png = tiletype(data) { +// oxipng::optimize_from_memory(data, options).map_err(UtilesError::OxipngError) +// } else { +// warn!("Unsupported image type"); +// Ok(data.to_vec()) +// } +// } // TODO: Implement pngify_image // fn pngify_image(data: &Vec) -> UtilesResult> { diff --git a/crates/utiles/src/lager.rs b/crates/utiles/src/lager.rs index a72961dc..71b58b3b 100644 --- a/crates/utiles/src/lager.rs +++ b/crates/utiles/src/lager.rs @@ -7,14 +7,14 @@ use tracing_subscriber::EnvFilter; use crate::errors::UtilesResult; -#[derive(Parser, Debug)] -pub struct LogConfig { +#[derive(Parser, Debug, Default)] +pub struct LagerConfig { pub debug: bool, pub trace: bool, pub json: bool, } -pub fn init_tracing(log_config: &LogConfig) -> UtilesResult<()> { +pub fn init_tracing(log_config: &LagerConfig) -> UtilesResult<()> { let filter = if log_config.trace { EnvFilter::new("TRACE") } else if log_config.debug { @@ -39,6 +39,7 @@ pub fn init_tracing(log_config: &LogConfig) -> UtilesResult<()> { } false => { let subscriber = fmt::Subscriber::builder() + .without_time() .with_env_filter(filter) .with_writer(io::stderr) .with_target(debug_or_trace) diff --git a/crates/utiles/src/lib.rs b/crates/utiles/src/lib.rs index cb0e62e9..4c59ce6b 100644 --- a/crates/utiles/src/lib.rs +++ b/crates/utiles/src/lib.rs @@ -23,9 +23,7 @@ pub use core::*; pub use errors::UtilesError; pub use errors::UtilesResult; -pub use lager::init_tracing; pub use tile_strfmt::TileStringFormatter; - pub mod cli; mod config; mod copy; @@ -36,7 +34,7 @@ mod fs_async; pub mod gj; mod globster; mod img; -mod lager; +pub mod lager; mod lint; pub mod mbt; mod pmt; @@ -46,9 +44,10 @@ pub mod sqlite; pub mod sqlite_utiles; mod tile_strfmt; pub mod utilejson; -pub mod utilesqlite; +pub mod hash_types; #[cfg(test)] mod tests; +mod tile_stream; pub const VERSION: &str = env!("CARGO_PKG_VERSION"); diff --git a/crates/utiles/src/lint/mbt_linter.rs b/crates/utiles/src/lint/mbt_linter.rs index 292551ea..a26a58a1 100644 --- a/crates/utiles/src/lint/mbt_linter.rs +++ b/crates/utiles/src/lint/mbt_linter.rs @@ -1,14 +1,16 @@ +use std::path::{Path, PathBuf}; + +use tracing::warn; + use crate::errors::UtilesResult; use crate::lint::MbtLint; -use crate::mbt::metadata2duplicates; -use crate::sqlite::AsyncSqliteConn; -use crate::utilesqlite::mbtiles::{ +use crate::mbt::mbtiles::{ has_unique_index_on_metadata, metadata_table_name_is_primary_key, }; -use crate::utilesqlite::{MbtilesAsync, MbtilesAsyncSqliteClient}; -use crate::{utilesqlite, UtilesError}; -use std::path::{Path, PathBuf}; -use tracing::warn; +use crate::mbt::metadata2duplicates; +use crate::mbt::{MbtilesAsync, MbtilesClientAsync}; +use crate::sqlite::AsyncSqliteConn; +use crate::UtilesError; #[derive(Debug)] pub struct MbtilesLinter { @@ -24,17 +26,17 @@ impl MbtilesLinter { fix, } } - async fn open_mbtiles(&self) -> UtilesResult { + async fn open_mbtiles(&self) -> UtilesResult { let pth = self.path.to_str().map_or_else( || Err(UtilesError::PathConversionError("path".to_string())), Ok, )?; - let mbtiles = utilesqlite::MbtilesAsyncSqliteClient::open_readonly(pth).await?; + let mbtiles = MbtilesClientAsync::open_readonly(pth).await?; Ok(mbtiles) } pub async fn check_magic_number( - mbt: &MbtilesAsyncSqliteClient, + mbt: &MbtilesClientAsync, ) -> UtilesResult> { let magic_number_res = mbt.magic_number().await; match magic_number_res { @@ -52,7 +54,7 @@ impl MbtilesLinter { } pub async fn check_encoding( - mbt: &MbtilesAsyncSqliteClient, + mbt: &MbtilesClientAsync, ) -> UtilesResult> { let encoding = mbt.pragma_encoding().await?; if encoding.to_lowercase() != "utf-8" { @@ -62,7 +64,7 @@ impl MbtilesLinter { } pub async fn check_metadata_rows( - mbt: &MbtilesAsyncSqliteClient, + mbt: &MbtilesClientAsync, ) -> UtilesResult> { let metadata_rows = mbt.metadata_rows().await?; let metadata_keys = metadata_rows @@ -82,7 +84,7 @@ impl MbtilesLinter { } pub async fn check_metadata( - mbt: &MbtilesAsyncSqliteClient, + mbt: &MbtilesClientAsync, ) -> UtilesResult> { // that metadata table exists let has_unique_index_on_metadata_name = mbt diff --git a/crates/utiles/src/mbt/agg_tiles_hash.rs b/crates/utiles/src/mbt/agg_tiles_hash.rs index 0fdda383..97651db5 100644 --- a/crates/utiles/src/mbt/agg_tiles_hash.rs +++ b/crates/utiles/src/mbt/agg_tiles_hash.rs @@ -1,15 +1,18 @@ -use crate::errors::UtilesResult; -use crate::mbt::hash_types::HashType; -use crate::mbt::{make_tiles_stream, TilesFilter}; -use crate::utilesqlite::MbtilesAsyncSqliteClient; -use futures::StreamExt; +use futures::{Stream, StreamExt}; use hex::ToHex; use md5::Digest; use noncrypto_digests::Fnv; use serde::Serialize; use tokio_stream::wrappers::ReceiverStream; use tracing::debug; -use utiles_core::{Tile, TileLike}; + +use utiles_core::TileLike; + +use crate::errors::UtilesResult; +use crate::hash_types::HashType; +use crate::mbt::MbtilesClientAsync; +use crate::mbt::TilesFilter; +use crate::tile_stream::TileReceiverStream; #[derive(Debug, Serialize)] pub struct AggHashResult { @@ -75,7 +78,7 @@ pub struct AggHashResult { // } pub async fn hash_stream( - mut data: ReceiverStream>, + mut data: impl Stream> + Unpin, ) -> (String, usize) { let mut hasher = T::new(); let mut count = 0; @@ -87,7 +90,7 @@ pub async fn hash_stream( (hasher.finalize().to_vec().encode_hex_upper(), count) } pub fn tile_stream_to_bytes_stream( - mut data: ReceiverStream<(Tile, Vec)>, + mut data: TileReceiverStream, ) -> ReceiverStream> { let (tx, rx) = tokio::sync::mpsc::channel(100); @@ -123,7 +126,7 @@ impl HashType { } pub async fn mbt_agg_tiles_hash_stream( - mbt: &MbtilesAsyncSqliteClient, + mbt: &MbtilesClientAsync, hash_type: HashType, prefix: Option<&str>, filter: &Option, @@ -136,7 +139,7 @@ pub async fn mbt_agg_tiles_hash_stream( let query = format!( "SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles {where_clause} ORDER BY zoom_level, tile_column, tile_row;" ); - let stream = make_tiles_stream(mbt, Some(query.as_str()))?; + let stream = mbt.tiles_stream(Some(query.as_str()))?; let bstream = tile_stream_to_bytes_stream(stream); let ti = std::time::Instant::now(); let (agg_tiles_hash_str, ntiles) = hash_type.hash_stream(bstream).await; diff --git a/crates/utiles/src/mbt/info.rs b/crates/utiles/src/mbt/info.rs index 28e1dc41..16a5aeff 100644 --- a/crates/utiles/src/mbt/info.rs +++ b/crates/utiles/src/mbt/info.rs @@ -4,13 +4,13 @@ use std::path::Path; use crate::errors::UtilesResult; use crate::fs_async::file_exists; use crate::mbt::MbtilesStats; -use crate::utilesqlite::{MbtilesAsync, MbtilesAsyncSqliteClient}; +use crate::mbt::{MbtilesAsync, MbtilesClientAsync}; pub async fn mbinfo(filepath: &str) -> UtilesResult { let fspath = Path::new(filepath); let is_file = file_exists(filepath).await; if is_file { - let mbt = MbtilesAsyncSqliteClient::open_existing(filepath).await?; + let mbt = MbtilesClientAsync::open_existing(filepath).await?; let stats = mbt.mbt_stats(None).await?; Ok(stats) } else { diff --git a/crates/utiles/src/mbt/mbt_stats.rs b/crates/utiles/src/mbt/mbt_stats.rs index 2063e4ca..a955f059 100644 --- a/crates/utiles/src/mbt/mbt_stats.rs +++ b/crates/utiles/src/mbt/mbt_stats.rs @@ -3,10 +3,10 @@ use serde::Serialize; use tracing::debug; use crate::errors::UtilesResult; +use crate::mbt::mbtiles::{zoom_stats, zoom_stats_full}; use crate::mbt::query::query_mbtiles_type; use crate::mbt::MbtType; use crate::sqlite::{pragma_freelist_count, pragma_page_count, pragma_page_size}; -use crate::utilesqlite::mbtiles::{zoom_stats, zoom_stats_full}; #[derive(Debug, Serialize)] pub struct MbtilesZoomStats { diff --git a/crates/utiles/src/utilesqlite/mbtiles.rs b/crates/utiles/src/mbt/mbtiles.rs similarity index 100% rename from crates/utiles/src/utilesqlite/mbtiles.rs rename to crates/utiles/src/mbt/mbtiles.rs diff --git a/crates/utiles/src/utilesqlite/mbtiles_async_sqlite.rs b/crates/utiles/src/mbt/mbtiles_async_sqlite.rs similarity index 94% rename from crates/utiles/src/utilesqlite/mbtiles_async_sqlite.rs rename to crates/utiles/src/mbt/mbtiles_async_sqlite.rs index eecb75d8..77c4c94d 100644 --- a/crates/utiles/src/utilesqlite/mbtiles_async_sqlite.rs +++ b/crates/utiles/src/mbt/mbtiles_async_sqlite.rs @@ -13,6 +13,11 @@ use tracing::{debug, error, info, warn}; use utiles_core::BBox; use crate::errors::UtilesResult; +use crate::mbt::mbtiles::{ + add_functions, has_metadata_table_or_view, has_tiles_table_or_view, + has_zoom_row_col_index, init_mbtiles, mbtiles_metadata, mbtiles_metadata_row, + metadata_json, minzoom_maxzoom, query_zxy, tiles_count, tiles_is_empty, +}; use crate::mbt::mbtiles_async::MbtilesAsync; use crate::mbt::query::query_mbtiles_type; use crate::mbt::zxyify::zxyify; @@ -26,29 +31,24 @@ use crate::sqlite::{ }; use crate::sqlite::{pathlike2dbpath, DbPath, DbPathTrait}; use crate::utilejson::metadata2tilejson; -use crate::utilesqlite::mbtiles::{ - add_functions, has_metadata_table_or_view, has_tiles_table_or_view, - has_zoom_row_col_index, init_mbtiles, mbtiles_metadata, mbtiles_metadata_row, - metadata_json, minzoom_maxzoom, query_zxy, tiles_count, tiles_is_empty, -}; use crate::UtilesError; #[derive(Clone)] -pub struct MbtilesAsyncSqliteClient { +pub struct MbtilesClientAsync { pub dbpath: DbPath, pub mbtype: MbtType, pub client: Client, } #[derive(Clone)] -pub struct MbtilesAsyncSqlitePool { +pub struct MbtilesPoolAsync { pub dbpath: DbPath, pub mbtype: MbtType, pub pool: Pool, } #[allow(clippy::missing_fields_in_debug)] -impl Debug for MbtilesAsyncSqlitePool { +impl Debug for MbtilesPoolAsync { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { // use the dbpath to debug f.debug_struct("MbtilesAsyncSqlitePool") @@ -58,7 +58,7 @@ impl Debug for MbtilesAsyncSqlitePool { } #[allow(clippy::missing_fields_in_debug)] -impl Debug for MbtilesAsyncSqliteClient { +impl Debug for MbtilesClientAsync { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("MbtilesAsyncSqliteClient") .field("fspath", &self.dbpath.fspath) @@ -67,7 +67,7 @@ impl Debug for MbtilesAsyncSqliteClient { } #[async_trait] -impl AsyncSqliteConn for MbtilesAsyncSqliteClient { +impl AsyncSqliteConn for MbtilesClientAsync { async fn conn(&self, func: F) -> Result where F: FnOnce(&Connection) -> Result + Send + 'static, @@ -78,7 +78,7 @@ impl AsyncSqliteConn for MbtilesAsyncSqliteClient { } #[async_trait] -impl AsyncSqliteConn for MbtilesAsyncSqlitePool { +impl AsyncSqliteConn for MbtilesPoolAsync { async fn conn(&self, func: F) -> Result where F: FnOnce(&Connection) -> Result + Send + 'static, @@ -89,7 +89,7 @@ impl AsyncSqliteConn for MbtilesAsyncSqlitePool { } #[async_trait] -impl AsyncSqliteConnMut for MbtilesAsyncSqliteClient { +impl AsyncSqliteConnMut for MbtilesClientAsync { async fn conn_mut(&self, func: F) -> Result where F: FnOnce(&mut Connection) -> Result + Send + 'static, @@ -100,7 +100,7 @@ impl AsyncSqliteConnMut for MbtilesAsyncSqliteClient { } #[async_trait] -impl AsyncSqliteConnMut for MbtilesAsyncSqlitePool { +impl AsyncSqliteConnMut for MbtilesPoolAsync { async fn conn_mut(&self, func: F) -> Result where F: FnOnce(&mut Connection) -> Result + Send + 'static, @@ -110,11 +110,11 @@ impl AsyncSqliteConnMut for MbtilesAsyncSqlitePool { } } -impl MbtilesAsyncSqliteClient { +impl MbtilesClientAsync { pub async fn new(dbpath: DbPath, client: Client) -> UtilesResult { let mbtype = client.conn(query_mbtiles_type).await?; - Ok(MbtilesAsyncSqliteClient { + Ok(MbtilesClientAsync { dbpath, mbtype, client, @@ -152,14 +152,14 @@ impl MbtilesAsyncSqliteClient { }) .await?; - MbtilesAsyncSqliteClient::new(dbpath, client).await + MbtilesClientAsync::new(dbpath, client).await } } pub async fn open>(path: P) -> UtilesResult { let dbpath = pathlike2dbpath(path)?; debug!("Opening mbtiles file with client: {}", dbpath); let client = ClientBuilder::new().path(&dbpath.fspath).open().await?; - MbtilesAsyncSqliteClient::new(dbpath, client).await + MbtilesClientAsync::new(dbpath, client).await } pub async fn open_existing>(path: P) -> UtilesResult { @@ -174,7 +174,7 @@ impl MbtilesAsyncSqliteClient { ) .open() .await?; - MbtilesAsyncSqliteClient::new(dbpath, client).await + MbtilesClientAsync::new(dbpath, client).await } pub async fn open_readonly>(path: P) -> UtilesResult { @@ -188,7 +188,7 @@ impl MbtilesAsyncSqliteClient { .flags(flags) .open() .await?; - MbtilesAsyncSqliteClient::new(dbpath, client).await + MbtilesClientAsync::new(dbpath, client).await } pub async fn journal_mode_wal(self) -> UtilesResult { @@ -204,10 +204,10 @@ impl MbtilesAsyncSqliteClient { Ok(jm) } } -impl MbtilesAsyncSqlitePool { +impl MbtilesPoolAsync { pub async fn new(dbpath: DbPath, pool: Pool) -> UtilesResult { let mbtype = pool.conn(query_mbtiles_type).await?; - Ok(MbtilesAsyncSqlitePool { + Ok(MbtilesPoolAsync { dbpath, mbtype, pool, @@ -225,7 +225,7 @@ impl MbtilesAsyncSqlitePool { .num_conns(2) .open() .await?; - MbtilesAsyncSqlitePool::new(dbpath, pool).await + MbtilesPoolAsync::new(dbpath, pool).await } pub async fn open_existing>(path: P) -> UtilesResult { @@ -241,7 +241,7 @@ impl MbtilesAsyncSqlitePool { .num_conns(2) .open() .await?; - MbtilesAsyncSqlitePool::new(dbpath, pool).await + MbtilesPoolAsync::new(dbpath, pool).await } pub async fn journal_mode_wal(self) -> UtilesResult { @@ -258,13 +258,13 @@ impl MbtilesAsyncSqlitePool { } } -impl DbPathTrait for MbtilesAsyncSqliteClient { +impl DbPathTrait for MbtilesClientAsync { fn db_path(&self) -> &DbPath { &self.dbpath } } -impl DbPathTrait for MbtilesAsyncSqlitePool { +impl DbPathTrait for MbtilesPoolAsync { fn db_path(&self) -> &DbPath { &self.dbpath } diff --git a/crates/utiles/src/mbt/metadata/change.rs b/crates/utiles/src/mbt/metadata/change.rs index e1bb204b..35ffad2d 100644 --- a/crates/utiles/src/mbt/metadata/change.rs +++ b/crates/utiles/src/mbt/metadata/change.rs @@ -1,4 +1,4 @@ -use crate::utilesqlite::mbtiles::{metadata_delete, metadata_set}; +use crate::mbt::mbtiles::{metadata_delete, metadata_set}; use json_patch::Patch; use serde::{Deserialize, Serialize}; use serde_json::Value; diff --git a/crates/utiles/src/mbt/metadata2map.rs b/crates/utiles/src/mbt/metadata/metadata2map.rs similarity index 100% rename from crates/utiles/src/mbt/metadata2map.rs rename to crates/utiles/src/mbt/metadata/metadata2map.rs diff --git a/crates/utiles/src/mbt/metadata/mod.rs b/crates/utiles/src/mbt/metadata/mod.rs index c7e1861d..1822b384 100644 --- a/crates/utiles/src/mbt/metadata/mod.rs +++ b/crates/utiles/src/mbt/metadata/mod.rs @@ -1,6 +1,11 @@ -mod change; -mod parse; -mod read_fspath; pub use change::{DbChangeType, DbChangeset, MetadataChange, MetadataChangeFromTo}; +pub use metadata2map::{ + metadata2duplicates, metadata2map, metadata2map_val, metadata_vec_has_duplicates, +}; pub use parse::{parse_metadata_json, parse_metadata_json_value}; pub use read_fspath::read_metadata_json; + +mod change; +mod metadata2map; +mod parse; +mod read_fspath; diff --git a/crates/utiles/src/mbt/mod.rs b/crates/utiles/src/mbt/mod.rs index d9c7f759..4f0b7897 100644 --- a/crates/utiles/src/mbt/mod.rs +++ b/crates/utiles/src/mbt/mod.rs @@ -1,31 +1,32 @@ pub use agg_tiles_hash::mbt_agg_tiles_hash_stream; +pub use info::mbinfo; +pub use mbtiles::Mbtiles; +pub use mbtiles_async::MbtilesAsync; +pub use mbtiles_async_sqlite::{MbtilesClientAsync, MbtilesPoolAsync}; pub use mbtype::MbtType; pub use metadata::*; pub use stream_writer::{MbtStreamWriter, MbtWriterStats}; pub use tiles_row::MbtTileRow; pub use crate::mbt::mbt_stats::{query_mbt_stats, MbtilesStats, MbtilesZoomStats}; -pub use crate::mbt::metadata2map::{ - metadata2duplicates, metadata2map, metadata2map_val, metadata_vec_has_duplicates, -}; pub use crate::mbt::metadata_row::{ MbtMetadataRow, MbtilesMetadataJson, MbtilesMetadataJsonRaw, MbtilesMetadataRowParsed, MbtilesMetadataRows, }; pub use crate::mbt::minzoom_maxzoom::MinZoomMaxZoom; pub use crate::mbt::tiles_filter::TilesFilter; -pub use info::mbinfo; -pub use tiles_stream::{make_tiles_rx, make_tiles_stream}; + mod agg_tiles_hash; -pub mod hash_types; mod info; mod mbt_stats; -pub mod mbtiles_async; +mod mbtiles_async; mod mbtype; mod metadata; -mod metadata2map; mod metadata_row; mod minzoom_maxzoom; + +pub mod mbtiles; +pub mod mbtiles_async_sqlite; pub mod query; mod stream_writer; mod tiles_filter; diff --git a/crates/utiles/src/mbt/stream_writer.rs b/crates/utiles/src/mbt/stream_writer.rs index b9ac9871..39b2b993 100644 --- a/crates/utiles/src/mbt/stream_writer.rs +++ b/crates/utiles/src/mbt/stream_writer.rs @@ -6,7 +6,7 @@ use tracing::{debug, warn}; use utiles_core::{Tile, TileLike}; -use crate::utilesqlite::Mbtiles; +use crate::mbt::Mbtiles; use crate::UtilesResult; #[derive(Default)] diff --git a/crates/utiles/src/mbt/tiles_stream.rs b/crates/utiles/src/mbt/tiles_stream.rs index ee1cc3c7..0ad54383 100644 --- a/crates/utiles/src/mbt/tiles_stream.rs +++ b/crates/utiles/src/mbt/tiles_stream.rs @@ -4,13 +4,13 @@ use tracing::{error, warn}; use utiles_core::prelude::*; +use crate::mbt::MbtilesClientAsync; use crate::sqlite::{AsyncSqliteConn, RusqliteResult}; -use crate::utilesqlite::MbtilesAsyncSqliteClient; +use crate::tile_stream::TileReceiverStream; use crate::UtilesResult; pub fn make_tiles_rx( - mbt: &MbtilesAsyncSqliteClient, - // options: Option, + mbt: &MbtilesClientAsync, query_override: Option<&str>, ) -> UtilesResult)>> { let (tx, rx) = tokio::sync::mpsc::channel(100); @@ -63,10 +63,18 @@ pub fn make_tiles_rx( Ok(rx) } -pub fn make_tiles_stream( - mbt: &MbtilesAsyncSqliteClient, - query_override: Option<&str>, -) -> UtilesResult)>> { - let rx = make_tiles_rx(mbt, query_override)?; - Ok(ReceiverStream::new(rx)) +impl MbtilesClientAsync { + pub fn tiles_rx( + &self, + query_override: Option<&str>, + ) -> UtilesResult)>> { + make_tiles_rx(self, query_override) + } + pub fn tiles_stream( + &self, + query_override: Option<&str>, + ) -> UtilesResult { + let rx = self.tiles_rx(query_override)?; + Ok(ReceiverStream::new(rx)) + } } diff --git a/crates/utiles/src/server/mod.rs b/crates/utiles/src/server/mod.rs index 4eadd2a7..9d34aef7 100644 --- a/crates/utiles/src/server/mod.rs +++ b/crates/utiles/src/server/mod.rs @@ -35,9 +35,9 @@ use utiles_core::{quadkey2tile, utile, Tile}; use crate::errors::UtilesResult; use crate::globster::find_filepaths; -use crate::mbt::mbtiles_async::MbtilesAsync; +use crate::mbt::MbtilesAsync; +use crate::mbt::MbtilesClientAsync; use crate::signal::shutdown_signal; -use crate::utilesqlite::MbtilesAsyncSqliteClient; pub mod radix36; mod request_id; @@ -45,7 +45,7 @@ mod request_id; //============================================================================= pub struct MbtilesDataset { - pub mbtiles: MbtilesAsyncSqliteClient, + pub mbtiles: MbtilesClientAsync, pub tilejson: TileJSON, } @@ -93,7 +93,7 @@ async fn preflight(config: &UtilesServerConfig) -> UtilesResult { // let mut tilejsons = HashMap::new(); for fspath in &filepaths { // let pool = MbtilesAsyncSqlitePool::open_readonly(fspath).await.unwrap(); - let pool = MbtilesAsyncSqliteClient::open_readonly(fspath).await?; + let pool = MbtilesClientAsync::open_readonly(fspath).await?; debug!("sanity check: {:?}", pool.filepath()); let is_valid = pool.is_mbtiles().await; // if error or not a valid mbtiles file, skip it diff --git a/crates/utiles/src/tile_stream.rs b/crates/utiles/src/tile_stream.rs new file mode 100644 index 00000000..774f13dc --- /dev/null +++ b/crates/utiles/src/tile_stream.rs @@ -0,0 +1,4 @@ +use tokio_stream::wrappers::ReceiverStream; +use utiles_core::Tile; + +pub type TileReceiverStream = ReceiverStream<(Tile, Vec)>; diff --git a/crates/utiles/src/utilesqlite/mod.rs b/crates/utiles/src/utilesqlite/mod.rs deleted file mode 100644 index ce468644..00000000 --- a/crates/utiles/src/utilesqlite/mod.rs +++ /dev/null @@ -1,6 +0,0 @@ -pub mod mbtiles; -pub mod mbtiles_async_sqlite; -pub use crate::mbt::mbtiles_async::MbtilesAsync; -pub use mbtiles::Mbtiles; -pub use mbtiles_async_sqlite::MbtilesAsyncSqliteClient; -pub use mbtiles_async_sqlite::MbtilesAsyncSqlitePool;