diff --git a/Makefile b/Makefile index 3bc706c2..2fae3a3a 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,8 @@ else endif WITHOUT_JOBS := $(shell echo $(ALL_MODULES) | tr ' ' '\n' | grep -v -e "nomad" | awk '{print "-target=module." $$0 ""}' | xargs) -ALL_MODULES_ARGS := $(shell echo $(ALL_MODULES) | tr ' ' '\n' | awk '{print "-target=module." $$0 ""}' | xargs) +ALL_MODULES_ARGS := $(shell echo $(ALL_MODULES) | tr ' ' '\n' | awk '{print "-target=module." $$0 ""}' | xargs) +DEBUG_ALL_MODULES_ARGS := -target=module.nomad.nomad_job.loki -target=module.nomad.nomad_job.logs-collector -target=module.nomad.nomad_job.otel-collector DESTROY_TARGETS := $(shell terraform state list | grep module | cut -d'.' -f1,2 | grep -v -e "fc_envs_disk" -e "buckets" | uniq | awk '{print "-target=" $$0 ""}' | xargs) # Login for Packer and Docker (uses gcloud user creds) @@ -62,6 +63,17 @@ apply: -compact-warnings \ -parallelism=20 $(ALL_MODULES_ARGS) +.PHONY: debug-apply +debug-apply: + @ printf "Applying Terraform for env: `tput setaf 2``tput bold`$(ENV)`tput sgr0`\n\n" + ./scripts/confirm.sh $(ENV) + $(tf_vars) \ + terraform apply \ + -auto-approve \ + -input=false \ + -compact-warnings \ + -parallelism=20 $(DEBUG_ALL_MODULES_ARGS) + .PHONY: plan-without-jobs plan-without-jobs: @ printf "Planning Terraform for env: `tput setaf 2``tput bold`$(ENV)`tput sgr0`\n\n" diff --git a/packages/cluster/network/main.tf b/packages/cluster/network/main.tf index a0f0aea2..d6241736 100644 --- a/packages/cluster/network/main.tf +++ b/packages/cluster/network/main.tf @@ -252,8 +252,8 @@ resource "google_compute_global_forwarding_rule" "https" { name = "${var.prefix}forwarding-rule-https" target = google_compute_target_https_proxy.default.self_link load_balancing_scheme = "EXTERNAL_MANAGED" - port_range = "443" - labels = var.labels + port_range = "443" + labels = var.labels } diff --git a/packages/nomad/logs-collector.hcl b/packages/nomad/logs-collector.hcl index acb8991e..943010fb 100644 --- a/packages/nomad/logs-collector.hcl +++ b/packages/nomad/logs-collector.hcl @@ -134,22 +134,6 @@ source = "logs-collector" service = "{{ service }}" teamID = "{{ teamID }}" envID = "{{ envID }}" -sandboxID = "{{ sandboxID }}" - -[sinks.grafana] -type = "loki" -inputs = [ "add_source_envd" ] -endpoint = "${var.grafana_logs_endpoint}" -encoding.codec = "json" -auth.strategy = "basic" -auth.user = "${var.grafana_logs_username}" -auth.password = "${var.grafana_api_key}" - -[sinks.grafana.labels] -source = "logs-collector" -service = "{{ service }}" -teamID = "{{ teamID }}" -envID = "{{ envID }}" sandboxID = "{{ sandboxID }}" EOH diff --git a/packages/nomad/loki.hcl b/packages/nomad/loki.hcl index 7967d4e1..723a61dc 100644 --- a/packages/nomad/loki.hcl +++ b/packages/nomad/loki.hcl @@ -54,9 +54,9 @@ job "loki" { } resources { - memory_max = 2048 - memory = 1024 - cpu = 512 + memory_max = 4096 + memory = 2046 + cpu = 1024 } template { @@ -79,6 +79,8 @@ common: storage_config: gcs: bucket_name: "${var.loki_bucket_name}" + chunk_buffer_size: 2097152 # 2MB + tsdb_shipper: active_index_directory: /loki/tsdb-shipper-active cache_location: /loki/tsdb-shipper-cache @@ -89,7 +91,7 @@ chunk_store_config: chunk_cache_config: embedded_cache: enabled: true - max_size_mb: 256 + max_size_mb: 512 ttl: 1h query_range: @@ -111,6 +113,7 @@ ingester_client: ingester: chunk_idle_period: 30m chunk_encoding: snappy + chunk_target_size: 1048576 # 1MB wal: dir: /loki/wal flush_on_shutdown: true @@ -137,9 +140,9 @@ compactor: limits_config: retention_period: 168h ingestion_rate_mb: 100 - ingestion_burst_size_mb: 200 - per_stream_rate_limit: "48MB" - per_stream_rate_limit_burst: "120MB" + ingestion_burst_size_mb: 500 + per_stream_rate_limit: "80MB" + per_stream_rate_limit_burst: "240MB" max_streams_per_user: 0 max_global_streams_per_user: 10000