start implementing data-hoarder registry

tlm-solutions · Nov 25, 2023 · 0c8b910 · 0c8b910
1 parent ed4107a
commit 0c8b910
Show file tree

Hide file tree

Showing 18 changed files with 177 additions and 149 deletions.
diff --git a/flake.nix b/flake.nix
@@ -241,6 +241,7 @@
       // (import ./pkgs/deployment.nix { inherit self pkgs lib; })
       // (lib.foldl (x: y: lib.mergeAttrs x { "${y.config.system.name}-vm" = y.config.system.build.vm; }) { } (lib.attrValues self.nixosConfigurations));
 
+      registry = import ./registry;
     in
     {
 
@@ -252,7 +253,7 @@
 
         data-hoarder = nixpkgs.lib.nixosSystem {
           system = "x86_64-linux";
-          specialArgs = { inherit inputs self; };
+          specialArgs = { inherit inputs self; registry = registry.data-hoarder; };
           modules = [
             microvm.nixosModules.microvm
             ./hosts/data-hoarder
@@ -261,7 +262,7 @@
 
         staging-data-hoarder = nixpkgs.lib.nixosSystem {
           system = "x86_64-linux";
-          specialArgs = { inherit inputs self; };
+          specialArgs = { inherit inputs self; registry = registry.data-hoarder; };
           modules = [
             ./hosts/staging-data-hoarder
             microvm.nixosModules.microvm

diff --git a/modules/data-hoarder/bureaucrat.nix b/modules/data-hoarder/bureaucrat.nix
@@ -1,25 +1,15 @@
-{ config, ... }:
-let
-  service_number = 6;
-in
-{
+{ config, registry, ... }: {
   TLMS.bureaucrat = {
     enable = true;
-    grpc = {
-      host = "127.0.0.1";
-      port = 50050 + service_number;
-    };
-    redis = {
-      host = config.services.redis.servers."state".bind;
-      port = config.services.redis.servers."state".port;
-    };
+    grpc = registry.grpc-chemo-bureaucrat;
+    redis = registry.redis-bureaucrat-lizard;
   };
 
   services = {
-    redis.servers."state" = {
+    redis.servers."state" = with registry.redis-bureaucrat-lizard; {
+      inherit port;
       enable = true;
-      bind = "127.0.0.1";
-      port = 5314;
+      bind = host;
     };
   };
 }
diff --git a/modules/data-hoarder/chemo.nix b/modules/data-hoarder/chemo.nix
@@ -1,12 +1,7 @@
-{ config, ... }:
-let
-  service_number = 3;
-in
-{
+{ config, registry, ... }: {
   TLMS.chemo = {
+    inherit (registry.grpc-data_accumulator-chemo) host port;
     enable = true;
-    host = "127.0.0.1";
-    port = 50050 + service_number;
     database = {
       host = "127.0.0.1";
       port = config.services.postgresql.port;
@@ -16,14 +11,12 @@ in
     };
     GRPC = [
       {
+        inherit (registry.grpc-chemo-bureaucrat) host port;
         name = "BUREAUCRAT";
-        host = config.TLMS.bureaucrat.grpc.host;
-        port = config.TLMS.bureaucrat.grpc.port;
       }
       {
+        inherit (registry.grpc-chemo-funnel) host port;
         name = "FUNNEL";
-        host = config.TLMS.funnel.GRPC.host;
-        port = config.TLMS.funnel.GRPC.port;
       }
     ];
   };

diff --git a/modules/data-hoarder/data-accumulator.nix b/modules/data-hoarder/data-accumulator.nix
@@ -1,23 +1,18 @@
-{ config, ... }:
-{
+{ config, registry, ... }: {
   TLMS.dataAccumulator = {
+    inherit (registry.port-data_accumulator) host port;
     enable = true;
-    host = "0.0.0.0";
-    port = 8080;
     database = {
       host = "127.0.0.1";
       port = config.services.postgresql.port;
       passwordFile = config.sops.secrets.postgres_password.path;
       user = "tlms";
       database = "tlms";
     };
-    GRPC = [
-      {
-        name = "CHEMO";
-        host = config.TLMS.chemo.host;
-        port = config.TLMS.chemo.port;
-      }
-    ];
+    GRPC = [{
+      inherit (registry.grpc-data_accumulator-chemo) host port;
+      name = "CHEMO";
+    }];
   };
   systemd.services."data-accumulator" = {
     after = [ "postgresql.service" ];
@@ -29,7 +24,10 @@
       enable = true;
       recommendedProxySettings = true;
       virtualHosts = {
-        "dump.${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] config.deployment-TLMS.domain)}" = {
+        "dump.${
+          (builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ]
+            config.deployment-TLMS.domain)
+        }" = {
           enableACME = true;
           forceSSL = true;
           extraConfig = ''
@@ -41,7 +39,8 @@
           enableACME = true;
           locations = {
             "/" = {
-              proxyPass = with config.TLMS.dataAccumulator; "http://${host}:${toString port}/";
+              proxyPass = with registry.port-data_accumulator;
+                "http://${host}:${toString port}/";
             };
           };
         };

diff --git a/modules/data-hoarder/datacare.nix b/modules/data-hoarder/datacare.nix
@@ -1,10 +1,7 @@
-{ config, ... }: {
+{ config, registry, ... }: {
   TLMS.datacare = {
     enable = true;
-    http = {
-      host = "127.0.0.1";
-      port = 8070;
-    };
+    http = registry.port-datacare;
     database = {
       host = "127.0.0.1";
       port = config.services.postgresql.port;
@@ -22,13 +19,15 @@
     wants = [ "postgresql.service" ];
   };
 
-
   services = {
     nginx = {
       enable = true;
       recommendedProxySettings = true;
       virtualHosts = {
-        "datacare.${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] config.deployment-TLMS.domain)}" = {
+        "datacare.${
+          (builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ]
+            config.deployment-TLMS.domain)
+        }" = {
           enableACME = true;
           forceSSL = true;
           extraConfig = ''
@@ -40,7 +39,8 @@
           enableACME = true;
           locations = {
             "/" = {
-              proxyPass = with config.TLMS.datacare.http; "http://${host}:${toString port}/";
+              proxyPass = with registry.port-data_accumulator;
+                "http://${host}:${toString port}/";
               proxyWebsockets = true;
               extraConfig = ''
                 more_set_headers "Access-Control-Allow-Credentials: true";

diff --git a/modules/data-hoarder/documentation.nix b/modules/data-hoarder/documentation.nix
@@ -1,11 +1,13 @@
-{ pkgs, config, ... }:
-{
+{ pkgs, config, ... }: {
   services = {
     nginx = {
       enable = true;
       recommendedProxySettings = true;
       virtualHosts = {
-        "docs.${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] config.deployment-TLMS.domain)}" = {
+        "docs.${
+          (builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ]
+            config.deployment-TLMS.domain)
+        }" = {
           enableACME = true;
           forceSSL = true;
           extraConfig = ''

diff --git a/modules/data-hoarder/file_sharing.nix b/modules/data-hoarder/file_sharing.nix
@@ -4,7 +4,10 @@
       enable = true;
       recommendedProxySettings = true;
       virtualHosts = {
-        "files.${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] config.deployment-TLMS.domain)}" = {
+        "files.${
+          (builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ]
+            config.deployment-TLMS.domain)
+        }" = {
           enableACME = true;
           forceSSL = true;
           extraConfig = ''

diff --git a/modules/data-hoarder/kindergarten.nix b/modules/data-hoarder/kindergarten.nix
@@ -2,7 +2,10 @@
   services.nginx = {
     enable = true;
     virtualHosts = {
-      "kid.${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] config.deployment-TLMS.domain)}" = {
+      "kid.${
+        (builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ]
+          config.deployment-TLMS.domain)
+      }" = {
         enableACME = true;
         forceSSL = true;
         extraConfig = ''
@@ -13,7 +16,10 @@
         enableACME = true;
         forceSSL = true;
         locations."~ ^/(de|en)" = {
-          root = if (config.deployment-TLMS.domain == "tlm.solutions") then "${pkgs.kindergarten}" else "${pkgs.kindergarten-staging}";
+          root = if (config.deployment-TLMS.domain == "tlm.solutions") then
+            "${pkgs.kindergarten}"
+          else
+            "${pkgs.kindergarten-staging}";
           # index = "index.html";
           tryFiles = "$uri /$1/index.html =404";
           extraConfig = ''

diff --git a/modules/data-hoarder/lizard.nix b/modules/data-hoarder/lizard.nix
@@ -1,34 +1,22 @@
-{ config, ... }:
-let
-  service_number = 1;
-in
-{
+{ config, registry, ... }: {
   TLMS.lizard = {
     enable = true;
-    http = {
-      host = "127.0.0.1";
-      port = 9000 + service_number;
-    };
+    http = { inherit (registry.port-lizard) host port; };
 
-    redis = {
-      host = config.services.redis.servers."state".bind;
-      port = config.services.redis.servers."state".port;
-    };
+    redis = registry.redis-bureaucrat-lizard;
     logLevel = "debug";
     workerCount = 6;
   };
 
   services = {
-    redis.servers."state" = {
-      enable = true;
-      bind = "127.0.0.1";
-      port = 5314;
-    };
     nginx = {
       enable = true;
       recommendedProxySettings = true;
       virtualHosts = {
-        "lizard.${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] config.deployment-TLMS.domain)}" = {
+        "lizard.${
+          (builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ]
+            config.deployment-TLMS.domain)
+        }" = {
           enableACME = true;
           forceSSL = true;
           extraConfig = ''
@@ -40,7 +28,8 @@ in
           enableACME = true;
           locations = {
             "/" = {
-              proxyPass = with config.TLMS.lizard.http; "http://${host}:${toString port}/";
+              proxyPass = with registry.port-lizard;
+                "http://${host}:${toString port}/";
               proxyWebsockets = true;
             };
           };

diff --git a/modules/data-hoarder/map.nix b/modules/data-hoarder/map.nix
@@ -4,7 +4,10 @@
       enable = true;
       recommendedProxySettings = true;
       virtualHosts = {
-        "map.${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] config.deployment-TLMS.domain)}" = {
+        "map.${
+          (builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ]
+            config.deployment-TLMS.domain)
+        }" = {
           enableACME = true;
           forceSSL = true;
           extraConfig = ''

diff --git a/modules/data-hoarder/nginx.nix b/modules/data-hoarder/nginx.nix
@@ -20,8 +20,7 @@ let
     # STS
     add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
   '';
-in
-{
+in {
   networking.firewall.allowedTCPPorts = [ 80 443 ];
 
   security.acme.acceptTerms = true;

diff --git a/modules/data-hoarder/postgres.nix b/modules/data-hoarder/postgres.nix
@@ -4,22 +4,19 @@
     enable = true;
     enableTCPIP = true;
     port = 5432;
-    authentication =
-      let
-        senpai-ip = self.nixosConfigurations.notice-me-senpai.config.deployment-TLMS.net.wg.addr4;
-      in
-      pkgs.lib.mkOverride 10 ''
-        local	all	all	trust
-        host	all	all	127.0.0.1/32	trust
-        host	all	all	::1/128	trust
-        host	tlms	grafana	${senpai-ip}/32	scram-sha-256
-      '';
+    authentication = let
+      senpai-ip =
+        self.nixosConfigurations.notice-me-senpai.config.deployment-TLMS.net.wg.addr4;
+    in pkgs.lib.mkOverride 10 ''
+      local	all	all	trust
+      host	all	all	127.0.0.1/32	trust
+      host	all	all	::1/128	trust
+      host	tlms	grafana	${senpai-ip}/32	scram-sha-256
+    '';
     package = pkgs.postgresql_14;
     ensureDatabases = [ "tlms" ];
     ensureUsers = [
-      {
-        name = "grafana";
-      }
+      { name = "grafana"; }
       {
         name = "tlms";
         ensurePermissions = {
@@ -30,15 +27,12 @@
     ];
   };
 
-  environment.systemPackages = [ inputs.tlms-rs.packages.x86_64-linux.run-migration-based ];
+  environment.systemPackages =
+    [ inputs.tlms-rs.packages.x86_64-linux.run-migration-based ];
 
   systemd.services.postgresql = {
-    unitConfig = {
-      TimeoutStartSec = 3000;
-    };
-    serviceConfig = {
-      TimeoutSec = lib.mkForce 3000;
-    };
+    unitConfig = { TimeoutStartSec = 3000; };
+    serviceConfig = { TimeoutSec = lib.mkForce 3000; };
     postStart = lib.mkAfter ''
       # set pw for the users
       $PSQL -c "ALTER ROLE tlms WITH PASSWORD '$(cat ${config.sops.secrets.postgres_password.path})';"
@@ -63,9 +57,7 @@
 
   systemd.services.dump-csv = {
     path = [ config.services.postgresql.package ];
-    serviceConfig = {
-      User = "postgres";
-    };
+    serviceConfig = { User = "postgres"; };
     script = ''
       TMPFILE=$(mktemp)
       OUT_FOLDER=/var/lib/pub-files/postgres-dumps/$(date -d"$(date) - 1 day" +"%Y-%m")