Skip to content

Commit

Permalink
refactor: use sparse crates index
Browse files Browse the repository at this point in the history
  • Loading branch information
robjtede committed May 26, 2024
1 parent 85a077e commit c00b380
Show file tree
Hide file tree
Showing 6 changed files with 61 additions and 170 deletions.
117 changes: 3 additions & 114 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

11 changes: 6 additions & 5 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,12 @@ badge = { path = "./libs/badge" }

anyhow = "1"
cadence = "1"
crates-index = { version = "2", default-features = false, features = ["git"] }
crates-index = "2"
derive_more = "0.99"
dotenvy = "0.15"
font-awesome-as-a-crate = "0.3"
futures-util = { version = "0.3", default-features = false, features = ["std"] }
http = "1"
hyper = { version = "0.14.10", features = ["full"] }
error_reporter = "1"
indexmap = { version = "2", features = ["serde"] }
Expand All @@ -30,7 +31,7 @@ once_cell = "1"
parking_lot = "0.12"
pulldown-cmark = "0.11"
relative-path = { version = "1", features = ["serde"] }
reqwest = { version = "0.12", features = ["json"] }
reqwest = { version = "0.12", default-features = false, features = ["json", "http2", "rustls-tls", "gzip"] }
route-recognizer = "0.3"
rustsec = "0.29"
semver = { version = "1.0", features = ["serde"] }
Expand All @@ -41,9 +42,9 @@ toml = "0.8"
tracing = "0.1.30"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }

[target.'cfg(any())'.dependencies]
gix = { version = "0.63", default-features = false, features = ["blocking-http-transport-reqwest-rust-tls"] }

[build-dependencies]
grass = { version = "0.13", default-features = false }
sha-1 = "0.10"

[dev-dependencies]
static_assertions = "1"
6 changes: 3 additions & 3 deletions src/interactors/crates.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ use futures_util::FutureExt as _;
use hyper::service::Service;
use semver::{Version, VersionReq};
use serde::Deserialize;
use tokio::task::spawn_blocking;

use crate::{
models::crates::{CrateDep, CrateDeps, CrateName, CratePath, CrateRelease},
Expand Down Expand Up @@ -69,8 +68,9 @@ impl QueryCrate {
crate_name: CrateName,
) -> anyhow::Result<QueryCrateResponse> {
let crate_name2 = crate_name.clone();
let krate = spawn_blocking(move || index.crate_(crate_name2))
.await?
let krate = index
.crate_(crate_name2)
.await
.ok_or_else(|| anyhow!("crate '{}' not found", crate_name.as_ref()))?;

convert_pkgs(krate)
Expand Down
11 changes: 3 additions & 8 deletions src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -79,15 +79,10 @@ async fn main() {

let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), port);

let index = ManagedIndex::new();
let index = ManagedIndex::new(client.clone());

{
let index = index.clone();

tokio::spawn(async move {
index.refresh_at_interval(Duration::from_secs(20)).await;
});
}
// index health check
index.crate_("libc".parse().unwrap()).await.unwrap();

let mut engine = Engine::new(client.clone(), index);
engine.set_metrics(metrics);
Expand Down
6 changes: 6 additions & 0 deletions src/models/crates.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,12 @@ impl CratePath {
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct CrateName(String);

impl CrateName {
pub(crate) fn as_str(&self) -> &str {
&self.0
}
}

impl From<CrateName> for String {
fn from(crate_name: CrateName) -> String {
crate_name.0
Expand Down
80 changes: 40 additions & 40 deletions src/utils/index.rs
Original file line number Diff line number Diff line change
@@ -1,54 +1,54 @@
use std::{sync::Arc, time::Duration};

use anyhow::Result;
use crates_index::{Crate, GitIndex};
use parking_lot::Mutex;
use tokio::{
task::spawn_blocking,
time::{self, MissedTickBehavior},
};
use std::sync::Arc;

use crate::models::crates::CrateName;
use crates_index::Crate;
use crates_index::SparseIndex;

#[derive(Clone)]
pub struct ManagedIndex {
index: Arc<Mutex<GitIndex>>,
index: Arc<SparseIndex>,
client: reqwest::Client,
}

#[cfg(test)]
static_assertions::assert_impl_all!(ManagedIndex: Send, Sync);

impl ManagedIndex {
pub fn new() -> Self {
pub fn new(client: reqwest::Client) -> Self {
// the index path is configurable through the `CARGO_HOME` env variable
let index = Arc::new(Mutex::new(GitIndex::new_cargo_default().unwrap()));

Self { index }
}
let index = Arc::new(SparseIndex::new_cargo_default().unwrap());

pub fn crate_(&self, crate_name: CrateName) -> Option<Crate> {
self.index.lock().crate_(crate_name.as_ref())
Self { index, client }
}

pub async fn refresh_at_interval(&self, update_interval: Duration) {
let mut update_interval = time::interval(update_interval);
update_interval.set_missed_tick_behavior(MissedTickBehavior::Delay);

loop {
if let Err(err) = self.refresh().await {
tracing::error!(
"failed refreshing the crates.io-index, the operation will be retried: {}",
error_reporter::Report::new(err),
);
}
update_interval.tick().await;
}
}

async fn refresh(&self) -> Result<(), crates_index::Error> {
let index = Arc::clone(&self.index);

spawn_blocking(move || index.lock().update())
.await
.expect("blocking index update task should never panic")?;

Ok(())
pub async fn crate_(&self, crate_name: CrateName) -> Option<Crate> {
let req = self
.index
.make_cache_request(crate_name.as_str())
.unwrap()
.body(())
.unwrap()
.into_parts()
.0;
let req = http::Request::from_parts(req, Vec::new());
let req = reqwest::Request::try_from(req).unwrap();

let res = self.client.execute(req).await.unwrap();

let mut builder = http::Response::builder()
.status(res.status())
.version(res.version());

builder
.headers_mut()
.unwrap()
.extend(res.headers().iter().map(|(k, v)| (k.clone(), v.clone())));

let body = res.bytes().await.unwrap();
let res = builder.body(body.to_vec()).unwrap();

self.index
.parse_cache_response(crate_name.as_str(), res, true)
.unwrap()
}
}

0 comments on commit c00b380

Please sign in to comment.