feat(infra): setup for trace infrastructure (#34)

* feat: add tempo * chore: disable tempo multi tenant * chore: unuse grafana agent * chore: add reverse proxy for tempo * chore: open receiver protocol http
skkuding · Feb 26, 2024 · 69452a2 · 69452a2
1 parent 8f958ca
commit 69452a2
Show file tree

Hide file tree

Showing 3 changed files with 100 additions and 24 deletions.
diff --git a/Caddyfile b/Caddyfile
@@ -16,22 +16,28 @@ grafana.codedang.com {
 		reverse_proxy 127.0.0.1:3199
 	}
 
-	handle /agent* {
-		handle /agent/metric* {
-			uri strip_prefix /agent/metric
-			reverse_proxy 127.0.0.1:4318
-		}
-
-		handle {
-			uri strip_prefix /agent
-			reverse_proxy 127.0.0.1:12345
-		}
-	}
+	# handle /agent* {
+	# 	handle /agent/metric* {
+	# 		uri strip_prefix /agent/metric
+	# 		reverse_proxy 127.0.0.1:4318
+	# 	}
+
+	# 	handle {
+	# 		uri strip_prefix /agent
+	# 		reverse_proxy 127.0.0.1:12345
+	# 	}
+	# }
 
 	handle /prometheus/* {
+		uri strip_prefix /prometheus
 		reverse_proxy 127.0.0.1:9090
 	}
 
+	handle /tempo/* {
+		uri strip_prefix /tempo
+		reverse_proxy 127.0.0.1:4318
+	}
+
 	handle {
 		reverse_proxy 127.0.0.1:3000
 	}

diff --git a/config/tempo/tempo.yml b/config/tempo/tempo.yml
@@ -0,0 +1,54 @@
+stream_over_http_enabled: true
+
+server:
+  http_listen_port: 3200
+  log_level: info
+
+query_frontend:
+  search:
+    duration_slo: 5s
+    throughput_bytes_slo: 1.073741824e+09
+  trace_by_id:
+    duration_slo: 5s
+
+distributor:
+  receivers:                           # this configuration will listen on all ports and protocols that tempo is capable of.
+    otlp:
+      protocols:
+        http:
+        grpc:
+
+ingester:
+  max_block_duration: 5m               # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally
+
+compactor:
+  compaction:
+    block_retention: 1h                # overall Tempo trace retention. set for demo purposes
+
+# metrics_generator:
+#   registry:
+#     external_labels:
+#       source: tempo
+#       cluster: docker-compose
+#   storage:
+#     path: /tmp/tempo/generator/wal
+#     remote_write:
+#       - url: http://prometheus:9090/api/v1/write
+#         send_exemplars: true
+
+storage:
+  trace:
+    backend: s3                     # backend configuration to use
+    wal:
+      path: /tmp/tempo/wal             # where to store the the wal locally
+    s3:
+      bucket: tempo                    # how to store data in s3
+      endpoint: minio:9000
+      access_key: ${MINIO_ROOT_USER}
+      secret_key: ${MINIO_ROOT_USER}
+      insecure: true
+
+overrides:
+  defaults:
+    metrics_generator:
+      processors: [service-graphs, span-metrics] # enables metrics generator
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -27,7 +27,7 @@ services:
     shm_size: "1gb"
 
   createbuckets:
-    profiles: ["log"]
+    profiles: ["log", "trace"]
     image: minio/mc
     volumes:
       - minio_data_volume:/data
@@ -39,6 +39,8 @@ services:
       /usr/bin/mc mb data/loki-aws;
       /usr/bin/mc mb data/loki-stage;
       /usr/bin/mc mb data/loki-test;
+      /usr/bin/mc mb data/tempo;
+      /usr/bin/mc anonymous set public data/tempo;
       /usr/bin/mc anonymous set public data/loki-aws;
       /usr/bin/mc anonymous set public data/loki-stage;
       /usr/bin/mc anonymous set public data/loki-test;
@@ -97,19 +99,33 @@ services:
       - '--storage.tsdb.path=/prometheus'
       - '--web.enable-remote-write-receiver'
 
-
-  agent:
-    profiles: ["metric", "trace"]
-    image: grafana/agent:latest
-    container_name: agent
-    environment:
-      - AGENT_MODE=flow
+  tempo:
+    prifiles: ["trace"]
+    image: grafana/tempo:latest
+    container_name: tempo
     ports:
-      - "12345:12345"
-      - "4318:4318"   # OTLP over HTTP receiver
-    volumes: 
-      - $PWD/config/agent/config.river:/etc/agent/config.river
-    command: run --server.http.listen-addr=0.0.0.0:12345 /etc/agent/config.river
+      - "3200:3200"   # tempo
+      - "4317:4317"  # otlp grpc
+      - "4318:4318"  # otlp http
+    volumes:
+      - $PWD/config/tempo/tempo.yml:/etc/tempo.yml
+    command: 
+      - "-config.file=/etc/tempo.yml -config.expand-env=true" 
+    depends_on:
+      - minio
+
+  # agent:
+  #   profiles: ["metric", "trace"]
+  #   image: grafana/agent:latest
+  #   container_name: agent
+  #   environment:
+  #     - AGENT_MODE=flow
+  #   ports:
+  #     - "12345:12345"
+  #     - "4318:4318"   # OTLP over HTTP receiver
+  #   volumes: 
+  #     - $PWD/config/agent/config.river:/etc/agent/config.river
+  #   command: run --server.http.listen-addr=0.0.0.0:12345 /etc/agent/config.river
 
   grafana:
     profiles: ["log", "metric", "trace"]