thread dump service rust (#1270)

* thread dumb service rust

* fix bug

* fix routing

* change to elfutils

* install lidw-dev

* fix install

* fix install

* set-up in workflows

* train set-up

* more dockerfile

* switch to using Makefiles for system deps

* more centralization

* retrigger CI

* switch to JSON

* format to Trace

* proper formatting

* use match[

* we need libwd-dev at run time too (dynamic lib)

* tipo

* renamed options

---------

Co-authored-by: Tarek Ziadé <[email protected]>

.github/workflows/nucliadb_node.yml

@@ -58,6 +58,15 @@ env:
   IMAGE_NAME_NODE_SIDECAR: node_sidecar
 
 jobs:
+  pre-checks-rust:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the repository
+        uses: actions/checkout@v3
+
+      - name: Check system
+        run: make -C nucliadb_node/ check-system
+
   pre-checks-python:
     runs-on: ubuntu-latest
     strategy:
@@ -112,6 +121,7 @@ jobs:
   clippy-rust:
     name: Clippy lint
     runs-on: ubuntu-latest
+    needs: pre-check-rust
     steps:
       - uses: actions/checkout@v3
 
@@ -131,6 +141,7 @@ jobs:
   prebuild-tests-rust:
     name: Prebuild rust tests
     runs-on: ubuntu-latest
+    needs: pre-check-rust
 
     steps:
       - uses: actions/checkout@v3

.github/workflows/nucliadb_node_release.yml

@@ -81,6 +81,10 @@ jobs:
 
       - run: pip install -U twine 'black>=22.3.0,<23' typing_extensions
 
+      - name: Check system deps
+        if: ${{ matrix.os != 'windows' }}
+        run: make -C nucliadb check-system
+
       - name: build sdist
         if: ${{ matrix.os == 'ubuntu' && matrix.target == 'x86_64' && matrix.manylinux == 'auto' }}
         uses: messense/maturin-action@v1
@@ -91,7 +95,10 @@ jobs:
 
       - name: build wheels
         uses: messense/maturin-action@v1
+        env:
+          RUSTFLAGS: --cfg tokio_unstable
         with:
+          before-script-linux: scripts/install-system-deps.sh
           target: ${{ matrix.target }}
           manylinux: ${{ matrix.manylinux || 'auto' }}
           container: ${{ matrix.container }}

Dockerfile.node

@@ -29,6 +29,10 @@ ARG CARGO_FEATURES=release-feature-set
 ARG CARGO_PROFILE=release
 ENV RUSTFLAGS="--cfg prometheus_metrics --cfg tokio_unstable"
 
+# Install system dependencies
+COPY scripts/install-system-deps.sh /tmp/
+RUN /tmp/install-system-deps.sh && rm /tmp/install-system-deps.sh && rm -rf /var/lib/apt/lists/*
+
 COPY --from=planner /nucliadb/recipe.json recipe.json
 
 # Build dependencies (this is the cached docker layer)
@@ -54,7 +58,6 @@ COPY nucliadb_texts /nucliadb/nucliadb_texts
 COPY nucliadb_vectors /nucliadb/nucliadb_vectors
 COPY vectors_benchmark /nucliadb/vectors_benchmark
 
-# Build application (only run when sources are modified)
 RUN echo "Building workspace with feature(s) '$CARGO_FEATURES' and profile '$CARGO_PROFILE'" \
     && cargo build \
     --locked \
@@ -78,6 +81,7 @@ RUN apt-get -y update \
     && apt-get -y install ca-certificates curl \
     libssl1.1 \
     lmdb-utils \
+    libdw-dev \
     && rm -rf /var/lib/apt/lists/*
 
 COPY --from=eu.gcr.io/stashify-218417/basenode:latest /bin/grpc_health_probe /bin/grpc_health_probe

Dockerfile.node_local

@@ -5,6 +5,10 @@ LABEL org.opencontainers.image.vendor="Nuclia Inc."
 
 WORKDIR /nucliadb
 
+# Install system dependencies
+COPY scripts/install-system-deps.sh /tmp/
+RUN /tmp/install-system-deps.sh && rm /tmp/install-system-deps.sh
+
 RUN apt-get -y update \
     && apt-get -y install ca-certificates \
     cmake \

Dockerfile.rust

@@ -25,6 +25,10 @@ RUN set -eux; \
     cargo --version; \
     rustc --version;
 
+# Install system dependencies
+COPY scripts/install-system-deps.sh /tmp/
+RUN /tmp/install-system-deps.sh && rm /tmp/install-system-deps.sh
+
 RUN apt-get update -y && apt-get install --yes --no-install-recommends patchelf cmake && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 RUN pip install maturin

Dockerfile.withbinding

@@ -26,6 +26,10 @@ RUN set -eux; \
     cargo --version; \
     rustc --version;
 
+# Install system dependencies
+COPY scripts/install-system-deps.sh /tmp/
+RUN /tmp/install-system-deps.sh && rm /tmp/install-system-deps.sh
+
 RUN apt-get update -y && apt-get install --yes --no-install-recommends patchelf cmake && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 RUN pip install maturin

nucliadb/Makefile

@@ -1,5 +1,5 @@
 .PHONY: install-dev
-install-dev:
+install-dev: check-system
 	pip install --upgrade pip wheel
 	cd .. && pip install \
 		-r test-requirements.txt \
@@ -9,6 +9,10 @@ install-dev:
 		-r nucliadb/requirements-test.txt
 	pip install -e .
 
+.PHONY: check-system
+check-system:
+	../scripts/install-system-deps.sh
+
 freeze-requirements:
 # create venv only for freezing requirements
 	python -m venv .freeze-venv

nucliadb_node/Cargo.toml

@@ -68,10 +68,10 @@ rand = "0.8.4"
 # nucliadb dependencies
 nucliadb_telemetry = { path = "../nucliadb_telemetry" }
 nucliadb_core = { path = "../nucliadb_core" }
-nucliadb_texts= { path = "../nucliadb_texts" }
-nucliadb_paragraphs= { path = "../nucliadb_paragraphs" }
-nucliadb_vectors= { path = "../nucliadb_vectors" }
-nucliadb_relations= { path = "../nucliadb_relations" }
+nucliadb_texts = { path = "../nucliadb_texts" }
+nucliadb_paragraphs = { path = "../nucliadb_paragraphs" }
+nucliadb_vectors = { path = "../nucliadb_vectors" }
+nucliadb_relations = { path = "../nucliadb_relations" }
 
 # sentry sdk
 sentry = "0.26.0"
@@ -86,6 +86,9 @@ tracing-log = { version = "0.1.3", features = ["env_logger"] }
 opentelemetry-zipkin = "0.15.0"
 sentry-tracing = "0.27.0"
 
+[target.'cfg(all(target_arch = "x86_64", target_os = "linux", not(target_env = "musl")))'.dependencies]
+rstack-self = { version = "0.3.0", features = ["dw"], default-features = false }
+
 # indra
 # indradb-lib = { version = "1", features = ["rocksdb-datastore"] }
 

nucliadb_node/Makefile

@@ -1,5 +1,5 @@
 .PHONY: install-dev
-install-dev:
+install-dev: check-system
 	pip install --upgrade pip wheel
 	cd .. && pip install \
 		-r test-requirements.txt \
@@ -8,6 +8,11 @@ install-dev:
 		-r nucliadb_node/requirements.txt
 	pip install -e .
 
+.PHONY: check-system
+check-system:
+	../scripts/install-system-deps.sh
+
+
 .PHONY: format
 format:
 	cd .. && isort --profile black nucliadb_node

nucliadb_node/src/bin/reader.rs

@@ -28,7 +28,7 @@ use nucliadb_node::grpc::middleware::{
     GrpcDebugLogsLayer, GrpcInstrumentorLayer, GrpcTasksMetricsLayer,
 };
 use nucliadb_node::grpc::reader::NodeReaderGRPCDriver;
-use nucliadb_node::http_server::{run_http_metrics_server, MetricsServerOptions};
+use nucliadb_node::http_server::{run_http_server, ServerOptions};
 use nucliadb_node::lifecycle;
 use nucliadb_node::settings::providers::env::EnvSettingsProvider;
 use nucliadb_node::settings::providers::SettingsProvider;
@@ -64,7 +64,7 @@ async fn main() -> NodeResult<()> {
         grpc_driver,
         settings.reader_listen_address(),
     ));
-    let metrics_task = tokio::spawn(run_http_metrics_server(MetricsServerOptions {
+    let metrics_task = tokio::spawn(run_http_server(ServerOptions {
         default_http_port: 3031,
     }));
 

nucliadb_node/src/bin/writer.rs

@@ -29,7 +29,7 @@ use nucliadb_node::grpc::middleware::{
     GrpcDebugLogsLayer, GrpcInstrumentorLayer, GrpcTasksMetricsLayer,
 };
 use nucliadb_node::grpc::writer::{NodeWriterEvent, NodeWriterGRPCDriver};
-use nucliadb_node::http_server::{run_http_metrics_server, MetricsServerOptions};
+use nucliadb_node::http_server::{run_http_server, ServerOptions};
 use nucliadb_node::node_metadata::NodeMetadata;
 use nucliadb_node::settings::providers::env::EnvSettingsProvider;
 use nucliadb_node::settings::providers::SettingsProvider;
@@ -90,7 +90,7 @@ async fn main() -> NodeResult<()> {
             None => tokio::spawn(task),
         };
     }
-    let metrics_task = tokio::spawn(run_http_metrics_server(MetricsServerOptions {
+    let metrics_task = tokio::spawn(run_http_server(ServerOptions {
         default_http_port: 3032,
     }));
 

nucliadb_node/src/http_server/mod.rs

@@ -21,6 +21,7 @@
 //! HTTP serving utilities
 
 mod metrics_service;
+mod traces_service;
 
 use std::net::SocketAddr;
 
@@ -29,17 +30,17 @@ use axum::Router;
 
 use crate::env::metrics_http_port;
 
-pub struct MetricsServerOptions {
+pub struct ServerOptions {
     pub default_http_port: u16,
 }
 
-pub async fn run_http_metrics_server(options: MetricsServerOptions) {
+pub async fn run_http_server(options: ServerOptions) {
     // Add routes to services
     let addr = SocketAddr::from(([0, 0, 0, 0], metrics_http_port(options.default_http_port)));
-    let metrics = Router::new().route("/metrics", get(metrics_service::metrics_service));
+    let router = Router::new().route("/metrics", get(metrics_service::metrics_service));
+    let router = router.route("/__dump", get(traces_service::thread_dump_service));
     axum_server::bind(addr)
-        // Services will be added here
-        .serve(metrics.into_make_service())
+        .serve(router.into_make_service())
         .await
         .expect("Error starting the HTTP server");
 }