From e70e21c0f5974ff1ead57c2a0f485671f1e120e6 Mon Sep 17 00:00:00 2001 From: Ivan Nikolaenko Date: Thu, 21 Sep 2023 14:36:17 +0300 Subject: [PATCH] Refactor VM networking *Introduce new naming pattern for VMs All VMs' names/hostnames are created using '-vm' pattern. All VMs' network interfaces named using 'tap-' pattern. NetVM's bridge is configured to include all 'tap-*' interfaces. This approach makes no distinction between service VMs and application VMs, but it allows to easily guess the hostname without using a network toolkit. *Introduce .ghaf local domain. All VMs except of NetVM now get their IPs from NetVM's DHCP server. The VMs are accessible by their hostnames with .ghaf postfix added, for instance, 'ssh gui-vm.ghaf'. *All VMs use separate subnet from now on, which is different from the debug-subnet. *The network configuration which is common for every VM is now extracted into the separate 'vm-networking.nix' file. Signed-off-by: Ivan Nikolaenko --- docs/src/ref_impl/creating_appvm.md | 6 +- modules/host/networking.nix | 2 +- modules/virtualization/microvm/appvm.nix | 64 ++----------------- .../microvm/common/vm-networking.nix | 52 +++++++++++++++ modules/virtualization/microvm/guivm.nix | 52 ++------------- modules/virtualization/microvm/netvm.nix | 54 ++++++++-------- targets/lenovo-x1-carbon.nix | 9 +-- 7 files changed, 96 insertions(+), 143 deletions(-) create mode 100644 modules/virtualization/microvm/common/vm-networking.nix diff --git a/docs/src/ref_impl/creating_appvm.md b/docs/src/ref_impl/creating_appvm.md index 3dee5e500..efa4d1a19 100644 --- a/docs/src/ref_impl/creating_appvm.md +++ b/docs/src/ref_impl/creating_appvm.md @@ -28,7 +28,6 @@ vms = with pkgs; [ { name = "chromium"; packages = [chromium]; - ipAddress = "192.168.101.5/24"; macAddress = "02:00:00:03:03:05"; ramMb = 3072; cores = 4; @@ -36,7 +35,6 @@ vms = with pkgs; [ { name = "gala"; packages = [(pkgs.callPackage ../user-apps/gala {})]; - ipAddress = "192.168.101.6/24"; macAddress = "02:00:00:03:03:06"; ramMb = 1536; cores = 2; @@ -44,7 +42,6 @@ vms = with pkgs; [ { name = "zathura"; packages = [zathura]; - ipAddress = "192.168.101.7/24"; macAddress = "02:00:00:03:03:07"; ramMb = 512; cores = 1; @@ -57,9 +54,8 @@ Each VM has the following properties: | **Property** | **Type** | **Unique** | **Description** | **Example** | | -------------- | --------------------------- | ------------ | --------------------------------------------------------------------------------------------------------------- | --------------------- | -| name | str | yes | This name is prefixed with `vm-` and will be shown in microvm list. The prefixed name - e.g. `vm-chromium` will be also the VM hostname. | “chromium” | +| name | str | yes | This name is postfixed with `-vm` and will be shown in microvm list. The name - e.g. `chromium-vm` will be also the VM hostname. The lenght of the name must be 8 characters or less. | “chromium” | | packages | list of types.package | no | Packages to include in a VM. It is possible to make it empty or add several packages. | [chromium top] | -| ipAddress | str | yes | This IP will be used to access a VM from the host. Should has the same subnetwork, as other VMs: Net, GUI VMs. | "192.168.101.5/24" | | macAddress | str | yes | Needed for network configuration. | "02:00:00:03:03:05" | | ramMb | int, [1, …, host memory] | no | Memory in MB. | 3072 | | cores | int, [1, …, host cores] | no | Virtual CPU cores. | 4 | diff --git a/modules/host/networking.nix b/modules/host/networking.nix index 45b765ea3..6f089416b 100644 --- a/modules/host/networking.nix +++ b/modules/host/networking.nix @@ -38,7 +38,7 @@ in # Connect VM tun/tap device to the bridge # TODO configure this based on IF the netvm is enabled networks."11-netvm" = { - matchConfig.Name = "vm-*"; + matchConfig.Name = "tap-*"; networkConfig.Bridge = "virbr0"; }; }; diff --git a/modules/virtualization/microvm/appvm.nix b/modules/virtualization/microvm/appvm.nix index 61f346529..668f252a5 100644 --- a/modules/virtualization/microvm/appvm.nix +++ b/modules/virtualization/microvm/appvm.nix @@ -8,19 +8,21 @@ }: let configHost = config; cfg = config.ghaf.virtualization.microvm.appvm; - waypipe-ssh = pkgs.callPackage ../../../user-apps/waypipe-ssh {}; - makeVm = { vm, index, }: let - hostname = "vm-" + vm.name; + vmName = "${vm.name}-vm"; cid = if vm.cid > 0 then vm.cid else cfg.vsockBaseCID + index; appvmConfiguration = { imports = [ + (import ./common/vm-networking.nix { + inherit vmName; + macAddress = vm.macAddress; + }) ({ lib, config, @@ -39,22 +41,13 @@ }; }; - users.users.${configHost.ghaf.users.accounts.user}.openssh.authorizedKeys.keyFiles = ["${waypipe-ssh}/keys/waypipe-ssh.pub"]; + users.users.${configHost.ghaf.users.accounts.user}.openssh.authorizedKeys.keyFiles = ["${pkgs.waypipe-ssh}/keys/waypipe-ssh.pub"]; - networking.hostName = hostname; system.stateVersion = lib.trivial.release; nixpkgs.buildPlatform.system = configHost.nixpkgs.buildPlatform.system; nixpkgs.hostPlatform.system = configHost.nixpkgs.hostPlatform.system; - networking = { - enableIPv6 = false; - interfaces.ethint0.useDHCP = false; - firewall.allowedTCPPorts = [22]; - firewall.allowedUDPPorts = [67]; - useNetworkd = true; - }; - environment.systemPackages = [ pkgs.waypipe ]; @@ -63,7 +56,6 @@ mem = vm.ramMb; vcpu = vm.cores; hypervisor = "qemu"; - shares = [ { tag = "ro-store"; @@ -73,13 +65,6 @@ ]; writableStoreOverlay = lib.mkIf config.ghaf.development.debug.tools.enable "/nix/.rw-store"; - interfaces = [ - { - type = "tap"; - id = hostname; - mac = vm.macAddress; - } - ]; qemu.extraArgs = [ "-M" "q35,accel=kvm:tcg,mem-merge=on,sata=off" @@ -88,35 +73,6 @@ ]; }; - networking.nat = { - enable = true; - internalInterfaces = ["ethint0"]; - }; - - # Set internal network's interface name to ethint0 - systemd.network.links."10-ethint0" = { - matchConfig.PermanentMACAddress = vm.macAddress; - linkConfig.Name = "ethint0"; - }; - - systemd.network = { - enable = true; - networks."10-ethint0" = { - matchConfig.MACAddress = vm.macAddress; - addresses = [ - { - # IP-address for debugging subnet - addressConfig.Address = vm.ipAddress; - } - ]; - routes = [ - {routeConfig.Gateway = "192.168.101.1";} - ]; - linkConfig.RequiredForOnline = "routable"; - linkConfig.ActivationPolicy = "always-up"; - }; - }; - imports = import ../../module-list.nix; }) ]; @@ -149,12 +105,6 @@ in { type = types.listOf package; default = []; }; - ipAddress = mkOption { - description = '' - AppVM's IP address in the inter-vm network - ''; - type = str; - }; macAddress = mkOption { description = '' AppVM's network interface MAC address @@ -216,7 +166,7 @@ in { config = lib.mkIf cfg.enable { microvm.vms = ( let - vms = lib.imap0 (index: vm: {"appvm-${vm.name}" = makeVm {inherit index vm;};}) cfg.vms; + vms = lib.imap0 (index: vm: {"${vm.name}-vm" = makeVm {inherit index vm;};}) cfg.vms; in lib.foldr lib.recursiveUpdate {} vms ); diff --git a/modules/virtualization/microvm/common/vm-networking.nix b/modules/virtualization/microvm/common/vm-networking.nix new file mode 100644 index 000000000..ff0b59556 --- /dev/null +++ b/modules/virtualization/microvm/common/vm-networking.nix @@ -0,0 +1,52 @@ +# Copyright 2022-2023 TII (SSRC) and the Ghaf contributors +# SPDX-License-Identifier: Apache-2.0 +{ + vmName, + macAddress, + ... +}: let + networkName = "ethint0"; +in { + networking = { + hostName = vmName; + enableIPv6 = false; + firewall.allowedTCPPorts = [22]; + firewall.allowedUDPPorts = [67]; + useNetworkd = true; + nat = { + enable = true; + internalInterfaces = [networkName]; + }; + }; + + microvm.interfaces = [ + { + type = "tap"; + # The interface names must have maximum length of 15 characters + id = "tap-${vmName}"; + mac = macAddress; + } + ]; + + systemd.network = { + enable = true; + # Set internal network's interface name to networkName + links."10-${networkName}" = { + matchConfig.PermanentMACAddress = macAddress; + linkConfig.Name = networkName; + }; + networks."10-${networkName}" = { + matchConfig.MACAddress = macAddress; + DHCP = "yes"; + linkConfig.RequiredForOnline = "routable"; + linkConfig.ActivationPolicy = "always-up"; + }; + }; + + # systemd-resolved does not support local names resolution + # without configuring a local domain. With the local domain, + # one would need also to disable DNSSEC for the clients. + # Disabling DNSSEC for other VM then NetVM is + # completely safe since they use NetVM as DNS proxy. + services.resolved.dnssec = "false"; +} diff --git a/modules/virtualization/microvm/guivm.nix b/modules/virtualization/microvm/guivm.nix index da7bd2cdf..07468ec96 100644 --- a/modules/virtualization/microvm/guivm.nix +++ b/modules/virtualization/microvm/guivm.nix @@ -7,9 +7,12 @@ ... }: let configHost = config; + vmName = "gui-vm"; + macAddress = "02:00:00:02:02:02"; waypipe-ssh = pkgs.callPackage ../../../user-apps/waypipe-ssh {}; guivmBaseConfiguration = { imports = [ + (import ./common/vm-networking.nix {inherit vmName macAddress;}) ({ lib, pkgs, @@ -37,24 +40,14 @@ ]; }; - networking.hostName = "guivm"; system.stateVersion = lib.trivial.release; nixpkgs.buildPlatform.system = configHost.nixpkgs.buildPlatform.system; nixpkgs.hostPlatform.system = configHost.nixpkgs.hostPlatform.system; - networking = { - enableIPv6 = false; - interfaces.ethint0.useDHCP = false; - firewall.allowedTCPPorts = [22]; - firewall.allowedUDPPorts = [67]; - useNetworkd = true; - }; - microvm = { mem = 2048; hypervisor = "qemu"; - shares = [ { tag = "ro-store"; @@ -64,49 +57,12 @@ ]; writableStoreOverlay = lib.mkIf config.ghaf.development.debug.tools.enable "/nix/.rw-store"; - interfaces = [ - { - type = "tap"; - id = "vm-guivm"; - mac = "02:00:00:02:02:02"; - } - ]; - qemu.extraArgs = [ "-device" "vhost-vsock-pci,guest-cid=${toString cfg.vsockCID}" ]; }; - networking.nat = { - enable = true; - internalInterfaces = ["ethint0"]; - }; - - # Set internal network's interface name to ethint0 - systemd.network.links."10-ethint0" = { - matchConfig.PermanentMACAddress = "02:00:00:02:02:02"; - linkConfig.Name = "ethint0"; - }; - - systemd.network = { - enable = true; - networks."10-ethint0" = { - matchConfig.MACAddress = "02:00:00:02:02:02"; - addresses = [ - { - # IP-address for debugging subnet - addressConfig.Address = "192.168.101.3/24"; - } - ]; - routes = [ - {routeConfig.Gateway = "192.168.101.1";} - ]; - linkConfig.RequiredForOnline = "routable"; - linkConfig.ActivationPolicy = "always-up"; - }; - }; - imports = import ../../module-list.nix; # Waypipe service runs in the GUIVM and listens for incoming connections from AppVMs @@ -173,7 +129,7 @@ in { }; config = lib.mkIf cfg.enable { - microvm.vms."guivm" = { + microvm.vms."${vmName}" = { autostart = true; config = guivmBaseConfiguration diff --git a/modules/virtualization/microvm/netvm.nix b/modules/virtualization/microvm/netvm.nix index e21df6fee..5f2165b78 100644 --- a/modules/virtualization/microvm/netvm.nix +++ b/modules/virtualization/microvm/netvm.nix @@ -3,11 +3,15 @@ { config, lib, + pkgs, ... }: let configHost = config; + vmName = "net-vm"; + macAddress = "02:00:00:01:01:01"; netvmBaseConfiguration = { imports = [ + (import ./common/vm-networking.nix {inherit vmName macAddress;}) ({lib, ...}: { ghaf = { users.accounts.enable = lib.mkDefault configHost.ghaf.users.accounts.enable; @@ -19,7 +23,6 @@ }; }; - networking.hostName = "netvm"; system.stateVersion = lib.trivial.release; nixpkgs.buildPlatform.system = configHost.nixpkgs.buildPlatform.system; @@ -28,42 +31,41 @@ microvm.hypervisor = "qemu"; networking = { - enableIPv6 = false; - interfaces.ethint0.useDHCP = false; - firewall.allowedTCPPorts = [22]; - firewall.allowedUDPPorts = [67]; - useNetworkd = true; + firewall.allowedTCPPorts = [53]; + firewall.allowedUDPPorts = [53]; }; - microvm.interfaces = [ - { - type = "tap"; - id = "vm-netvm"; - mac = "02:00:00:01:01:01"; - } - ]; - - networking.nat = { + # Dnsmasq is used as a DHCP/DNS server inside the NetVM + services.dnsmasq = { enable = true; - internalInterfaces = ["ethint0"]; + resolveLocalQueries = true; + settings = { + server = ["8.8.8.8"]; + dhcp-range = ["192.168.100.2,192.168.100.254"]; + dhcp-sequential-ip = true; + dhcp-authoritative = true; + domain = "ghaf"; + listen-address = ["127.0.0.1,192.168.100.1"]; + dhcp-option = [ + "option:router,192.168.100.1" + "6,192.168.100.1" + ]; + expand-hosts = true; + domain-needed = true; + bogus-priv = true; + }; }; - # Set internal network's interface name to ethint0 - systemd.network.links."10-ethint0" = { - matchConfig.PermanentMACAddress = "02:00:00:01:01:01"; - linkConfig.Name = "ethint0"; - }; + # Disable resolved since we are using Dnsmasq + services.resolved.enable = false; systemd.network = { enable = true; networks."10-ethint0" = { - matchConfig.MACAddress = "02:00:00:01:01:01"; + matchConfig.MACAddress = macAddress; networkConfig.DHCPServer = true; dhcpServerConfig.ServerAddress = "192.168.100.1/24"; addresses = [ - { - addressConfig.Address = "192.168.100.1/24"; - } { # IP-address for debugging subnet addressConfig.Address = "192.168.101.1/24"; @@ -103,7 +105,7 @@ in { }; config = lib.mkIf cfg.enable { - microvm.vms."netvm" = { + microvm.vms."${vmName}" = { autostart = true; config = netvmBaseConfiguration diff --git a/targets/lenovo-x1-carbon.nix b/targets/lenovo-x1-carbon.nix index 5c5e33745..292b26e49 100644 --- a/targets/lenovo-x1-carbon.nix +++ b/targets/lenovo-x1-carbon.nix @@ -69,17 +69,17 @@ ({pkgs, ...}: { ghaf.graphics.weston.launchers = [ { - path = "${pkgs.openssh}/bin/ssh -i ${pkgs.waypipe-ssh}/keys/waypipe-ssh -o StrictHostKeyChecking=no 192.168.101.5 ${pkgs.waypipe}/bin/waypipe --vsock -s ${toString guivmConfig.waypipePort} server chromium --enable-features=UseOzonePlatform --ozone-platform=wayland"; + path = "${pkgs.openssh}/bin/ssh -i ${pkgs.waypipe-ssh}/keys/waypipe-ssh -o StrictHostKeyChecking=no chromium-vm.ghaf ${pkgs.waypipe}/bin/waypipe --vsock -s ${toString guivmConfig.waypipePort} server chromium --enable-features=UseOzonePlatform --ozone-platform=wayland"; icon = "${../assets/icons/png/browser.png}"; } { - path = "${pkgs.openssh}/bin/ssh -i ${pkgs.waypipe-ssh}/keys/waypipe-ssh -o StrictHostKeyChecking=no 192.168.101.6 ${pkgs.waypipe}/bin/waypipe --vsock -s ${toString guivmConfig.waypipePort} server gala --enable-features=UseOzonePlatform --ozone-platform=wayland"; + path = "${pkgs.openssh}/bin/ssh -i ${pkgs.waypipe-ssh}/keys/waypipe-ssh -o StrictHostKeyChecking=no gala-vm.ghaf ${pkgs.waypipe}/bin/waypipe --vsock -s ${toString guivmConfig.waypipePort} server gala --enable-features=UseOzonePlatform --ozone-platform=wayland"; icon = "${../assets/icons/png/app.png}"; } { - path = "${pkgs.openssh}/bin/ssh -i ${pkgs.waypipe-ssh}/keys/waypipe-ssh -o StrictHostKeyChecking=no 192.168.101.7 ${pkgs.waypipe}/bin/waypipe --vsock -s ${toString guivmConfig.waypipePort} server zathura"; + path = "${pkgs.openssh}/bin/ssh -i ${pkgs.waypipe-ssh}/keys/waypipe-ssh -o StrictHostKeyChecking=no zathura-vm.ghaf ${pkgs.waypipe}/bin/waypipe --vsock -s ${toString guivmConfig.waypipePort} server zathura"; icon = "${../assets/icons/png/pdf.png}"; } ]; @@ -145,7 +145,6 @@ { name = "chromium"; packages = [pkgs.chromium pkgs.pamixer]; - ipAddress = "192.168.101.5/24"; macAddress = "02:00:00:03:05:01"; ramMb = 3072; cores = 4; @@ -179,7 +178,6 @@ { name = "gala"; packages = [pkgs.gala-app]; - ipAddress = "192.168.101.6/24"; macAddress = "02:00:00:03:06:01"; ramMb = 1536; cores = 2; @@ -187,7 +185,6 @@ { name = "zathura"; packages = [pkgs.zathura]; - ipAddress = "192.168.101.7/24"; macAddress = "02:00:00:03:07:01"; ramMb = 512; cores = 1;