Skip to content

Commit

Permalink
Refactor VM networking
Browse files Browse the repository at this point in the history 
*Introduce new naming pattern for VMs
All VMs' names/hostnames are created using '<vmname>-vm'
pattern. All VMs' network interfaces named using
'tap-<hostname>' pattern. NetVM's bridge is configured to
include all 'tap-*' interfaces.
This approach makes no distinction between service VMs and
application VMs, but it allows to easily guess the hostname
without using a network toolkit.

*Introduce .ghaf local domain.
All VMs except of NetVM now get their IPs from NetVM's DHCP
server. The VMs are accessible by their hostnames with .ghaf
postfix added, for instance, 'ssh gui-vm.ghaf'.

*All VMs use separate subnet from now on, which is different
from the debug-subnet.

*The network configuration which is common for every VM is now
extracted into the separate 'vm-networking.nix' file.

Signed-off-by: Ivan Nikolaenko <[email protected]>
  • Loading branch information
@unbel13ver
unbel13ver committed Sep 29, 2023
1 parent aaeafbc commit c435710
Show file tree
Hide file tree
Showing 7 changed files with 95 additions and 143 deletions.
6 changes: 1 addition & 5 deletions docs/src/ref_impl/creating_appvm.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,23 +28,20 @@ vms = with pkgs; [
{
name = "chromium";
packages = [chromium];
ipAddress = "192.168.101.5/24";
macAddress = "02:00:00:03:03:05";
ramMb = 3072;
cores = 4;
}
{
name = "gala";
packages = [(pkgs.callPackage ../user-apps/gala {})];
ipAddress = "192.168.101.6/24";
macAddress = "02:00:00:03:03:06";
ramMb = 1536;
cores = 2;
}
{
name = "zathura";
packages = [zathura];
ipAddress = "192.168.101.7/24";
macAddress = "02:00:00:03:03:07";
ramMb = 512;
cores = 1;
Expand All @@ -57,9 +54,8 @@ Each VM has the following properties:

| **Property** | **Type** | **Unique** | **Description** | **Example** |
| -------------- | --------------------------- | ------------ | --------------------------------------------------------------------------------------------------------------- | --------------------- |
| name | str | yes | This name is prefixed with `vm-` and will be shown in microvm list. The prefixed name - e.g. `vm-chromium` will be also the VM hostname. | “chromium” |
| name | str | yes | This name is postfixed with `-vm` and will be shown in microvm list. The name - e.g. `chromium-vm` will be also the VM hostname. | “chromium” |
| packages | list of types.package | no | Packages to include in a VM. It is possible to make it empty or add several packages. | [chromium top] |
| ipAddress | str | yes | This IP will be used to access a VM from the host. Should has the same subnetwork, as other VMs: Net, GUI VMs. | "192.168.101.5/24" |
| macAddress | str | yes | Needed for network configuration. | "02:00:00:03:03:05" |
| ramMb | int, [1, …, host memory] | no | Memory in MB. | 3072 |
| cores | int, [1, …, host cores] | no | Virtual CPU cores. | 4 |
Expand Down
2 changes: 1 addition & 1 deletion modules/host/networking.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ in
# Connect VM tun/tap device to the bridge
# TODO configure this based on IF the netvm is enabled
networks."11-netvm" = {
matchConfig.Name = "vm-*";
matchConfig.Name = "tap-*";
networkConfig.Bridge = "virbr0";
};
};
Expand Down
64 changes: 7 additions & 57 deletions modules/virtualization/microvm/appvm.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,19 +8,21 @@
}: let
configHost = config;
cfg = config.ghaf.virtualization.microvm.appvm;
waypipe-ssh = pkgs.callPackage ../../../user-apps/waypipe-ssh {};

makeVm = {
vm,
index,
}: let
hostname = "vm-" + vm.name;
vmName = "${vm.name}-vm";
cid =
if vm.cid > 0
then vm.cid
else cfg.vsockBaseCID + index;
appvmConfiguration = {
imports = [
(import ./common/vm-networking.nix {
inherit vmName;
macAddress = vm.macAddress;
})
({
lib,
config,
Expand All @@ -39,22 +41,13 @@
};
};

users.users.${configHost.ghaf.users.accounts.user}.openssh.authorizedKeys.keyFiles = ["${waypipe-ssh}/keys/waypipe-ssh.pub"];
users.users.${configHost.ghaf.users.accounts.user}.openssh.authorizedKeys.keyFiles = ["${pkgs.waypipe-ssh}/keys/waypipe-ssh.pub"];

networking.hostName = hostname;
system.stateVersion = lib.trivial.release;

nixpkgs.buildPlatform.system = configHost.nixpkgs.buildPlatform.system;
nixpkgs.hostPlatform.system = configHost.nixpkgs.hostPlatform.system;

networking = {
enableIPv6 = false;
interfaces.ethint0.useDHCP = false;
firewall.allowedTCPPorts = [22];
firewall.allowedUDPPorts = [67];
useNetworkd = true;
};

environment.systemPackages = [
pkgs.waypipe
];
Expand All @@ -63,7 +56,6 @@
mem = vm.ramMb;
vcpu = vm.cores;
hypervisor = "qemu";

shares = [
{
tag = "ro-store";
Expand All @@ -73,13 +65,6 @@
];
writableStoreOverlay = lib.mkIf config.ghaf.development.debug.tools.enable "/nix/.rw-store";

interfaces = [
{
type = "tap";
id = hostname;
mac = vm.macAddress;
}
];
qemu.extraArgs = [
"-M"
"q35,accel=kvm:tcg,mem-merge=on,sata=off"
Expand All @@ -88,35 +73,6 @@
];
};

networking.nat = {
enable = true;
internalInterfaces = ["ethint0"];
};

# Set internal network's interface name to ethint0
systemd.network.links."10-ethint0" = {
matchConfig.PermanentMACAddress = vm.macAddress;
linkConfig.Name = "ethint0";
};

systemd.network = {
enable = true;
networks."10-ethint0" = {
matchConfig.MACAddress = vm.macAddress;
addresses = [
{
# IP-address for debugging subnet
addressConfig.Address = vm.ipAddress;
}
];
routes = [
{routeConfig.Gateway = "192.168.101.1";}
];
linkConfig.RequiredForOnline = "routable";
linkConfig.ActivationPolicy = "always-up";
};
};

imports = import ../../module-list.nix;
})
];
Expand Down Expand Up @@ -149,12 +105,6 @@ in {
type = types.listOf package;
default = [];
};
ipAddress = mkOption {
description = ''
AppVM's IP address in the inter-vm network
'';
type = str;
};
macAddress = mkOption {
description = ''
AppVM's network interface MAC address
Expand Down Expand Up @@ -216,7 +166,7 @@ in {
config = lib.mkIf cfg.enable {
microvm.vms = (
let
vms = lib.imap0 (index: vm: {"appvm-${vm.name}" = makeVm {inherit index vm;};}) cfg.vms;
vms = lib.imap0 (index: vm: {"${vm.name}-vm" = makeVm {inherit index vm;};}) cfg.vms;
in
lib.foldr lib.recursiveUpdate {} vms
);
Expand Down
51 changes: 51 additions & 0 deletions modules/virtualization/microvm/common/vm-networking.nix
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
# Copyright 2022-2023 TII (SSRC) and the Ghaf contributors
# SPDX-License-Identifier: Apache-2.0
{
vmName,
macAddress,
...
}: let
networkName = "ethint0";
in {
networking = {
hostName = vmName;
enableIPv6 = false;
firewall.allowedTCPPorts = [22];
firewall.allowedUDPPorts = [67];
useNetworkd = true;
nat = {
enable = true;
internalInterfaces = [networkName];
};
};

microvm.interfaces = [
{
type = "tap";
id = "tap-${vmName}";
mac = macAddress;
}
];

systemd.network = {
enable = true;
# Set internal network's interface name to networkName
links."10-${networkName}" = {
matchConfig.PermanentMACAddress = macAddress;
linkConfig.Name = networkName;
};
networks."10-${networkName}" = {
matchConfig.MACAddress = macAddress;
DHCP = "yes";
linkConfig.RequiredForOnline = "routable";
linkConfig.ActivationPolicy = "always-up";
};
};

# systemd-resolved does not support local names resolution
# without configuring a local domain. With the local domain,
# one would need also to disable DNSSEC for the clients.
# Disabling DNSSEC for other VM then NetVM is
# completely safe since they use NetVM as DNS proxy.
services.resolved.dnssec = "false";
}
52 changes: 4 additions & 48 deletions modules/virtualization/microvm/guivm.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,12 @@
...
}: let
configHost = config;
vmName = "gui-vm";
macAddress = "02:00:00:02:02:02";
waypipe-ssh = pkgs.callPackage ../../../user-apps/waypipe-ssh {};
guivmBaseConfiguration = {
imports = [
(import ./common/vm-networking.nix {inherit vmName macAddress;})
({
lib,
pkgs,
Expand Down Expand Up @@ -37,24 +40,14 @@
];
};

networking.hostName = "guivm";
system.stateVersion = lib.trivial.release;

nixpkgs.buildPlatform.system = configHost.nixpkgs.buildPlatform.system;
nixpkgs.hostPlatform.system = configHost.nixpkgs.hostPlatform.system;

networking = {
enableIPv6 = false;
interfaces.ethint0.useDHCP = false;
firewall.allowedTCPPorts = [22];
firewall.allowedUDPPorts = [67];
useNetworkd = true;
};

microvm = {
mem = 2048;
hypervisor = "qemu";

shares = [
{
tag = "ro-store";
Expand All @@ -64,49 +57,12 @@
];
writableStoreOverlay = lib.mkIf config.ghaf.development.debug.tools.enable "/nix/.rw-store";

interfaces = [
{
type = "tap";
id = "vm-guivm";
mac = "02:00:00:02:02:02";
}
];

qemu.extraArgs = [
"-device"
"vhost-vsock-pci,guest-cid=${toString cfg.vsockCID}"
];
};

networking.nat = {
enable = true;
internalInterfaces = ["ethint0"];
};

# Set internal network's interface name to ethint0
systemd.network.links."10-ethint0" = {
matchConfig.PermanentMACAddress = "02:00:00:02:02:02";
linkConfig.Name = "ethint0";
};

systemd.network = {
enable = true;
networks."10-ethint0" = {
matchConfig.MACAddress = "02:00:00:02:02:02";
addresses = [
{
# IP-address for debugging subnet
addressConfig.Address = "192.168.101.3/24";
}
];
routes = [
{routeConfig.Gateway = "192.168.101.1";}
];
linkConfig.RequiredForOnline = "routable";
linkConfig.ActivationPolicy = "always-up";
};
};

imports = import ../../module-list.nix;

# Waypipe service runs in the GUIVM and listens for incoming connections from AppVMs
Expand Down Expand Up @@ -173,7 +129,7 @@ in {
};

config = lib.mkIf cfg.enable {
microvm.vms."guivm" = {
microvm.vms."${vmName}" = {
autostart = true;
config =
guivmBaseConfiguration
Expand Down
Loading

0 comments on commit c435710

Please sign in to comment.