Debug and Stabilize Logging and Loki

Makefile

@@ -24,7 +24,8 @@ else
 endif
 
 WITHOUT_JOBS := $(shell echo $(ALL_MODULES) | tr ' ' '\n' | grep -v -e "nomad" | awk '{print "-target=module." $$0 ""}' | xargs)
-ALL_MODULES_ARGS := $(shell echo $(ALL_MODULES) | tr ' ' '\n' | awk '{print "-target=module." $$0 ""}' | xargs)
+# TEMPORARILY DISABLED FOR DEBUG ALL_MODULES_ARGS := $(shell echo $(ALL_MODULES) | tr ' ' '\n' | awk '{print "-target=module." $$0 ""}' | xargs)
+ALL_MODULES_ARGS := -target=module.nomad.nomad_job.loki -target=module.nomad.nomad_job.logs-collector
 DESTROY_TARGETS := $(shell terraform state list | grep module | cut -d'.' -f1,2 | grep -v -e "fc_envs_disk" -e "buckets" | uniq | awk '{print "-target=" $$0 ""}' | xargs)
 
 # Login for Packer and Docker (uses gcloud user creds)

packages/cluster/network/main.tf

@@ -252,8 +252,8 @@ resource "google_compute_global_forwarding_rule" "https" {
   name                  = "${var.prefix}forwarding-rule-https"
   target                = google_compute_target_https_proxy.default.self_link
   load_balancing_scheme = "EXTERNAL_MANAGED"
-  port_range = "443"
-  labels     = var.labels
+  port_range            = "443"
+  labels                = var.labels
 }
 
 

packages/nomad/logs-collector.hcl

@@ -136,21 +136,22 @@ teamID = "{{ teamID }}"
 envID = "{{ envID }}"
 sandboxID = "{{ sandboxID }}"
 
-[sinks.grafana]
-type = "loki"
-inputs = [ "add_source_envd" ]
-endpoint = "${var.grafana_logs_endpoint}"
-encoding.codec = "json"
-auth.strategy = "basic"
-auth.user = "${var.grafana_logs_username}"
-auth.password = "${var.grafana_api_key}"
-
-[sinks.grafana.labels]
-source = "logs-collector"
-service = "{{ service }}"
-teamID = "{{ teamID }}"
-envID = "{{ envID }}"
-sandboxID = "{{ sandboxID }}"
+# [sinks.grafana]
+# type = "loki"
+# inputs = [ "add_source_envd" ]
+# endpoint = "${var.grafana_logs_endpoint}"
+# encoding.codec = "json"
+# auth.strategy = "basic"
+# auth.user = "${var.grafana_logs_username}"
+# auth.password = "${var.grafana_api_key}"
+# request.rate_limit_num = 20
+# 
+# [sinks.grafana.labels]
+# source = "logs-collector"
+# service = "{{ service }}"
+# teamID = "{{ teamID }}"
+# envID = "{{ envID }}"
+# sandboxID = "{{ sandboxID }}"
 
         EOH
       }

packages/nomad/loki.hcl

@@ -54,9 +54,9 @@ job "loki" {
       }
 
       resources {
-        memory_max = 2048
-        memory = 1024
-        cpu    = 512
+        memory_max = 4096
+        memory = 2046
+        cpu    = 1024
       }
 
       template {
@@ -79,6 +79,8 @@ common:
 storage_config:
   gcs:
     bucket_name: "${var.loki_bucket_name}"
+    chunk_buffer_size: 2097152  # 2MB
+
   tsdb_shipper:
     active_index_directory: /loki/tsdb-shipper-active
     cache_location: /loki/tsdb-shipper-cache
@@ -89,7 +91,7 @@ chunk_store_config:
   chunk_cache_config:
     embedded_cache:
       enabled: true
-      max_size_mb: 256
+      max_size_mb: 512
       ttl: 1h
 
 query_range:
@@ -111,6 +113,7 @@ ingester_client:
 ingester:
   chunk_idle_period: 30m
   chunk_encoding: snappy
+  chunk_target_size: 1048576  # 1MB 
   wal:
     dir: /loki/wal
     flush_on_shutdown: true
@@ -137,9 +140,9 @@ compactor:
 limits_config:
   retention_period: 168h
   ingestion_rate_mb: 100
-  ingestion_burst_size_mb: 200
-  per_stream_rate_limit: "48MB"
-  per_stream_rate_limit_burst: "120MB"
+  ingestion_burst_size_mb: 500
+  per_stream_rate_limit: "80MB"
+  per_stream_rate_limit_burst: "240MB"
   max_streams_per_user: 0
   max_global_streams_per_user: 10000