diff --git a/flake.nix b/flake.nix index 95f7ee4..6f96fca 100644 --- a/flake.nix +++ b/flake.nix @@ -241,6 +241,7 @@ // (import ./pkgs/deployment.nix { inherit self pkgs lib; }) // (lib.foldl (x: y: lib.mergeAttrs x { "${y.config.system.name}-vm" = y.config.system.build.vm; }) { } (lib.attrValues self.nixosConfigurations)); + registry = import ./registry; in { @@ -252,7 +253,7 @@ data-hoarder = nixpkgs.lib.nixosSystem { system = "x86_64-linux"; - specialArgs = { inherit inputs self; }; + specialArgs = { inherit inputs self; registry = registry.data-hoarder; }; modules = [ microvm.nixosModules.microvm ./hosts/data-hoarder @@ -261,7 +262,7 @@ staging-data-hoarder = nixpkgs.lib.nixosSystem { system = "x86_64-linux"; - specialArgs = { inherit inputs self; }; + specialArgs = { inherit inputs self; registry = registry.data-hoarder; }; modules = [ ./hosts/staging-data-hoarder microvm.nixosModules.microvm diff --git a/modules/data-hoarder/bureaucrat.nix b/modules/data-hoarder/bureaucrat.nix index 9adcb2d..464adb2 100644 --- a/modules/data-hoarder/bureaucrat.nix +++ b/modules/data-hoarder/bureaucrat.nix @@ -1,25 +1,15 @@ -{ config, ... }: -let - service_number = 6; -in -{ +{ config, registry, ... }: { TLMS.bureaucrat = { enable = true; - grpc = { - host = "127.0.0.1"; - port = 50050 + service_number; - }; - redis = { - host = config.services.redis.servers."state".bind; - port = config.services.redis.servers."state".port; - }; + grpc = registry.grpc-chemo-bureaucrat; + redis = registry.redis-bureaucrat-lizard; }; services = { - redis.servers."state" = { + redis.servers."state" = with registry.redis-bureaucrat-lizard; { + inherit port; enable = true; - bind = "127.0.0.1"; - port = 5314; + bind = host; }; }; } diff --git a/modules/data-hoarder/chemo.nix b/modules/data-hoarder/chemo.nix index 878beda..5ad399b 100644 --- a/modules/data-hoarder/chemo.nix +++ b/modules/data-hoarder/chemo.nix @@ -1,12 +1,7 @@ -{ config, ... }: -let - service_number = 3; -in -{ +{ config, registry, ... }: { TLMS.chemo = { + inherit (registry.grpc-data_accumulator-chemo) host port; enable = true; - host = "127.0.0.1"; - port = 50050 + service_number; database = { host = "127.0.0.1"; port = config.services.postgresql.port; @@ -16,14 +11,12 @@ in }; GRPC = [ { + inherit (registry.grpc-chemo-bureaucrat) host port; name = "BUREAUCRAT"; - host = config.TLMS.bureaucrat.grpc.host; - port = config.TLMS.bureaucrat.grpc.port; } { + inherit (registry.grpc-chemo-funnel) host port; name = "FUNNEL"; - host = config.TLMS.funnel.GRPC.host; - port = config.TLMS.funnel.GRPC.port; } ]; }; diff --git a/modules/data-hoarder/data-accumulator.nix b/modules/data-hoarder/data-accumulator.nix index 9acaf6d..c3b5ebb 100644 --- a/modules/data-hoarder/data-accumulator.nix +++ b/modules/data-hoarder/data-accumulator.nix @@ -1,9 +1,7 @@ -{ config, ... }: -{ +{ config, registry, ... }: { TLMS.dataAccumulator = { + inherit (registry.port-data_accumulator) host port; enable = true; - host = "0.0.0.0"; - port = 8080; database = { host = "127.0.0.1"; port = config.services.postgresql.port; @@ -11,13 +9,10 @@ user = "tlms"; database = "tlms"; }; - GRPC = [ - { - name = "CHEMO"; - host = config.TLMS.chemo.host; - port = config.TLMS.chemo.port; - } - ]; + GRPC = [{ + inherit (registry.grpc-data_accumulator-chemo) host port; + name = "CHEMO"; + }]; }; systemd.services."data-accumulator" = { after = [ "postgresql.service" ]; @@ -29,7 +24,10 @@ enable = true; recommendedProxySettings = true; virtualHosts = { - "dump.${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] config.deployment-TLMS.domain)}" = { + "dump.${ + (builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] + config.deployment-TLMS.domain) + }" = { enableACME = true; forceSSL = true; extraConfig = '' @@ -41,7 +39,8 @@ enableACME = true; locations = { "/" = { - proxyPass = with config.TLMS.dataAccumulator; "http://${host}:${toString port}/"; + proxyPass = with registry.port-data_accumulator; + "http://${host}:${toString port}/"; }; }; }; diff --git a/modules/data-hoarder/datacare.nix b/modules/data-hoarder/datacare.nix index 2041d58..381c912 100644 --- a/modules/data-hoarder/datacare.nix +++ b/modules/data-hoarder/datacare.nix @@ -1,10 +1,7 @@ -{ config, ... }: { +{ config, registry, ... }: { TLMS.datacare = { enable = true; - http = { - host = "127.0.0.1"; - port = 8070; - }; + http = registry.port-datacare; database = { host = "127.0.0.1"; port = config.services.postgresql.port; @@ -22,13 +19,15 @@ wants = [ "postgresql.service" ]; }; - services = { nginx = { enable = true; recommendedProxySettings = true; virtualHosts = { - "datacare.${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] config.deployment-TLMS.domain)}" = { + "datacare.${ + (builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] + config.deployment-TLMS.domain) + }" = { enableACME = true; forceSSL = true; extraConfig = '' @@ -40,7 +39,8 @@ enableACME = true; locations = { "/" = { - proxyPass = with config.TLMS.datacare.http; "http://${host}:${toString port}/"; + proxyPass = with registry.port-data_accumulator; + "http://${host}:${toString port}/"; proxyWebsockets = true; extraConfig = '' more_set_headers "Access-Control-Allow-Credentials: true"; diff --git a/modules/data-hoarder/documentation.nix b/modules/data-hoarder/documentation.nix index fd6d1b8..d782924 100644 --- a/modules/data-hoarder/documentation.nix +++ b/modules/data-hoarder/documentation.nix @@ -1,11 +1,13 @@ -{ pkgs, config, ... }: -{ +{ pkgs, config, ... }: { services = { nginx = { enable = true; recommendedProxySettings = true; virtualHosts = { - "docs.${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] config.deployment-TLMS.domain)}" = { + "docs.${ + (builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] + config.deployment-TLMS.domain) + }" = { enableACME = true; forceSSL = true; extraConfig = '' diff --git a/modules/data-hoarder/file_sharing.nix b/modules/data-hoarder/file_sharing.nix index b11e474..e802ec9 100644 --- a/modules/data-hoarder/file_sharing.nix +++ b/modules/data-hoarder/file_sharing.nix @@ -4,7 +4,10 @@ enable = true; recommendedProxySettings = true; virtualHosts = { - "files.${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] config.deployment-TLMS.domain)}" = { + "files.${ + (builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] + config.deployment-TLMS.domain) + }" = { enableACME = true; forceSSL = true; extraConfig = '' diff --git a/modules/data-hoarder/kindergarten.nix b/modules/data-hoarder/kindergarten.nix index e64ce4d..d1b3585 100644 --- a/modules/data-hoarder/kindergarten.nix +++ b/modules/data-hoarder/kindergarten.nix @@ -2,7 +2,10 @@ services.nginx = { enable = true; virtualHosts = { - "kid.${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] config.deployment-TLMS.domain)}" = { + "kid.${ + (builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] + config.deployment-TLMS.domain) + }" = { enableACME = true; forceSSL = true; extraConfig = '' @@ -13,7 +16,10 @@ enableACME = true; forceSSL = true; locations."~ ^/(de|en)" = { - root = if (config.deployment-TLMS.domain == "tlm.solutions") then "${pkgs.kindergarten}" else "${pkgs.kindergarten-staging}"; + root = if (config.deployment-TLMS.domain == "tlm.solutions") then + "${pkgs.kindergarten}" + else + "${pkgs.kindergarten-staging}"; # index = "index.html"; tryFiles = "$uri /$1/index.html =404"; extraConfig = '' diff --git a/modules/data-hoarder/lizard.nix b/modules/data-hoarder/lizard.nix index b4888ba..9116986 100644 --- a/modules/data-hoarder/lizard.nix +++ b/modules/data-hoarder/lizard.nix @@ -1,34 +1,22 @@ -{ config, ... }: -let - service_number = 1; -in -{ +{ config, registry, ... }: { TLMS.lizard = { enable = true; - http = { - host = "127.0.0.1"; - port = 9000 + service_number; - }; + http = { inherit (registry.port-lizard) host port; }; - redis = { - host = config.services.redis.servers."state".bind; - port = config.services.redis.servers."state".port; - }; + redis = registry.redis-bureaucrat-lizard; logLevel = "debug"; workerCount = 6; }; services = { - redis.servers."state" = { - enable = true; - bind = "127.0.0.1"; - port = 5314; - }; nginx = { enable = true; recommendedProxySettings = true; virtualHosts = { - "lizard.${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] config.deployment-TLMS.domain)}" = { + "lizard.${ + (builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] + config.deployment-TLMS.domain) + }" = { enableACME = true; forceSSL = true; extraConfig = '' @@ -40,7 +28,8 @@ in enableACME = true; locations = { "/" = { - proxyPass = with config.TLMS.lizard.http; "http://${host}:${toString port}/"; + proxyPass = with registry.port-lizard; + "http://${host}:${toString port}/"; proxyWebsockets = true; }; }; diff --git a/modules/data-hoarder/map.nix b/modules/data-hoarder/map.nix index 9c58d89..bb13776 100644 --- a/modules/data-hoarder/map.nix +++ b/modules/data-hoarder/map.nix @@ -4,7 +4,10 @@ enable = true; recommendedProxySettings = true; virtualHosts = { - "map.${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] config.deployment-TLMS.domain)}" = { + "map.${ + (builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] + config.deployment-TLMS.domain) + }" = { enableACME = true; forceSSL = true; extraConfig = '' diff --git a/modules/data-hoarder/nginx.nix b/modules/data-hoarder/nginx.nix index 45f194d..bc9ec0f 100644 --- a/modules/data-hoarder/nginx.nix +++ b/modules/data-hoarder/nginx.nix @@ -20,8 +20,7 @@ let # STS add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; ''; -in -{ +in { networking.firewall.allowedTCPPorts = [ 80 443 ]; security.acme.acceptTerms = true; diff --git a/modules/data-hoarder/postgres.nix b/modules/data-hoarder/postgres.nix index a0b5253..feedd18 100644 --- a/modules/data-hoarder/postgres.nix +++ b/modules/data-hoarder/postgres.nix @@ -4,22 +4,19 @@ enable = true; enableTCPIP = true; port = 5432; - authentication = - let - senpai-ip = self.nixosConfigurations.notice-me-senpai.config.deployment-TLMS.net.wg.addr4; - in - pkgs.lib.mkOverride 10 '' - local all all trust - host all all 127.0.0.1/32 trust - host all all ::1/128 trust - host tlms grafana ${senpai-ip}/32 scram-sha-256 - ''; + authentication = let + senpai-ip = + self.nixosConfigurations.notice-me-senpai.config.deployment-TLMS.net.wg.addr4; + in pkgs.lib.mkOverride 10 '' + local all all trust + host all all 127.0.0.1/32 trust + host all all ::1/128 trust + host tlms grafana ${senpai-ip}/32 scram-sha-256 + ''; package = pkgs.postgresql_14; ensureDatabases = [ "tlms" ]; ensureUsers = [ - { - name = "grafana"; - } + { name = "grafana"; } { name = "tlms"; ensurePermissions = { @@ -30,15 +27,12 @@ ]; }; - environment.systemPackages = [ inputs.tlms-rs.packages.x86_64-linux.run-migration-based ]; + environment.systemPackages = + [ inputs.tlms-rs.packages.x86_64-linux.run-migration-based ]; systemd.services.postgresql = { - unitConfig = { - TimeoutStartSec = 3000; - }; - serviceConfig = { - TimeoutSec = lib.mkForce 3000; - }; + unitConfig = { TimeoutStartSec = 3000; }; + serviceConfig = { TimeoutSec = lib.mkForce 3000; }; postStart = lib.mkAfter '' # set pw for the users $PSQL -c "ALTER ROLE tlms WITH PASSWORD '$(cat ${config.sops.secrets.postgres_password.path})';" @@ -63,9 +57,7 @@ systemd.services.dump-csv = { path = [ config.services.postgresql.package ]; - serviceConfig = { - User = "postgres"; - }; + serviceConfig = { User = "postgres"; }; script = '' TMPFILE=$(mktemp) OUT_FOLDER=/var/lib/pub-files/postgres-dumps/$(date -d"$(date) - 1 day" +"%Y-%m") diff --git a/modules/data-hoarder/secrets.nix b/modules/data-hoarder/secrets.nix index 463ec74..7b00dd2 100644 --- a/modules/data-hoarder/secrets.nix +++ b/modules/data-hoarder/secrets.nix @@ -4,14 +4,19 @@ let data-accumulator-user = config.TLMS.dataAccumulator.user; trekkie-user = config.TLMS.trekkie.user; chemo-user = config.TLMS.chemo.user; -in -{ +in { sops.age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ]; users.groups = { postgres-tlms = { name = "postgres-tlms"; - members = [ datacare-user data-accumulator-user trekkie-user chemo-user "postgres" ]; + members = [ + datacare-user + data-accumulator-user + trekkie-user + chemo-user + "postgres" + ]; }; password-salt = { @@ -28,9 +33,7 @@ in }; sops.secrets = { - wg-seckey = { - owner = config.users.users.systemd-network.name; - }; + wg-seckey = { owner = config.users.users.systemd-network.name; }; postgres_password_hash_salt = { group = config.users.groups.password-salt.name; mode = "0440"; diff --git a/modules/data-hoarder/socket.nix b/modules/data-hoarder/socket.nix index 7473bc0..0e0657d 100644 --- a/modules/data-hoarder/socket.nix +++ b/modules/data-hoarder/socket.nix @@ -1,20 +1,10 @@ -{ config, ... }: -let - service_number = 2; -in -{ +{ config, registry, ... }: { TLMS.funnel = { enable = true; - GRPC = { - host = "127.0.0.1"; - port = 50050 + service_number; - }; - defaultWebsocket = { - host = "127.0.0.1"; - port = 9000 + service_number; - }; + GRPC = registry.grpc-chemo-funnel; + defaultWebsocket = { inherit (registry.port-funnel) host port; }; metrics = { - port = 10010 + service_number; + inherit (registry.port-funnel-metrics) port; host = config.deployment-TLMS.net.wg.addr4; }; }; @@ -23,12 +13,16 @@ in enable = true; recommendedProxySettings = true; virtualHosts = { - "socket.${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] config.deployment-TLMS.domain)}" = { + "socket.${ + (builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] + config.deployment-TLMS.domain) + }" = { enableACME = true; forceSSL = true; locations."/" = { proxyWebsockets = true; - proxyPass = with config.TLMS.funnel.defaultWebsocket; "http://${host}:${toString port}/"; + proxyPass = with registry.port-funnel; + "http://${host}:${toString port}/"; }; }; "socket.${config.deployment-TLMS.domain}" = { @@ -36,7 +30,8 @@ in enableACME = true; locations = { "/" = { - proxyPass = with config.TLMS.funnel.defaultWebsocket; "http://${host}:${toString port}/"; + proxyPass = with registry.port-funnel; + "http://${host}:${toString port}/"; proxyWebsockets = true; }; }; diff --git a/modules/data-hoarder/trekkie.nix b/modules/data-hoarder/trekkie.nix index 93482cf..430ead5 100644 --- a/modules/data-hoarder/trekkie.nix +++ b/modules/data-hoarder/trekkie.nix @@ -1,24 +1,16 @@ -{ config, ... }: -{ +{ config, registry, ... }: { TLMS.trekkie = { + inherit (registry.port-trekkie) host port; enable = true; - host = "0.0.0.0"; saltPath = config.sops.secrets.postgres_password_hash_salt.path; - port = 8060; database = { host = "127.0.0.1"; port = config.services.postgresql.port; passwordFile = config.sops.secrets.postgres_password.path; user = "tlms"; }; - redis = { - port = 6379; - host = "localhost"; - }; - grpc = { - host = config.TLMS.chemo.host; - port = config.TLMS.chemo.port; - }; + redis = registry.redis-trekkie; + grpc = registry.grpc-trekkie-chemo; logLevel = "info"; }; systemd.services."trekkie" = { @@ -27,17 +19,20 @@ }; services = { - redis.servers."trekkie" = { + redis.servers."trekkie" = with registry.redis-trekkie; { + inherit port; enable = true; - bind = config.TLMS.trekkie.redis.host; - port = config.TLMS.trekkie.redis.port; + bind = host; }; nginx = { enable = true; recommendedProxySettings = true; virtualHosts = { - "trekkie.${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] config.deployment-TLMS.domain)}" = { + "trekkie.${ + (builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] + config.deployment-TLMS.domain) + }" = { enableACME = true; forceSSL = true; extraConfig = '' @@ -49,7 +44,8 @@ enableACME = true; locations = { "/" = { - proxyPass = with config.TLMS.trekkie; "http://${host}:${toString port}/"; + proxyPass = with registry.port-trekkie; + "http://${host}:${toString port}/"; }; }; }; diff --git a/modules/data-hoarder/website.nix b/modules/data-hoarder/website.nix index 35e545b..964007f 100644 --- a/modules/data-hoarder/website.nix +++ b/modules/data-hoarder/website.nix @@ -2,13 +2,14 @@ services.nginx = { enable = true; virtualHosts = { - "${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] config.deployment-TLMS.domain)}" = { - enableACME = true; - forceSSL = true; - extraConfig = '' - rewrite ^ https://kid.${config.deployment-TLMS.domain}/ permanent; - ''; - }; + "${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] + config.deployment-TLMS.domain)}" = { + enableACME = true; + forceSSL = true; + extraConfig = '' + rewrite ^ https://kid.${config.deployment-TLMS.domain}/ permanent; + ''; + }; "${config.deployment-TLMS.domain}" = { enableACME = true; forceSSL = true; diff --git a/registry/data-hoarder/default.nix b/registry/data-hoarder/default.nix new file mode 100644 index 0000000..7c62b44 --- /dev/null +++ b/registry/data-hoarder/default.nix @@ -0,0 +1,55 @@ +rec { + redis-bureaucrat-lizard = { + host = "127.0.0.1"; + port = 5314; + }; + + grpc-chemo-bureaucrat = { + host = "127.0.0.1"; + port = 50056; + }; + + grpc-chemo-funnel = { + host = "127.0.0.1"; + port = 50052; + }; + + grpc-data_accumulator-chemo = { + host = "127.0.0.1"; + port = 50053; + }; + + grpc-trekkie-chemo = grpc-data_accumulator-chemo; + + port-data_accumulator = { + host = "0.0.0.0"; + port = 8080; + }; + + port-datacare = { + host = "127.0.0.1"; + port = 8070; + }; + + port-lizard = { + host = "127.0.0.1"; + port = 9001; + }; + + port-funnel = { + host = "127.0.0.1"; + port = 9002; + }; + + port-funnel-metrics = { port = 10012; }; + + port-trekkie = { + host = "0.0.0.0"; + port = 8060; + }; + + redis-trekkie = { + host = "localhost"; + port = 6379; + }; +} diff --git a/registry/default.nix b/registry/default.nix new file mode 100644 index 0000000..66911eb --- /dev/null +++ b/registry/default.nix @@ -0,0 +1 @@ +{ data-hoarder = import ./data-hoarder; }