Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor VM networking #290

Merged
merged 1 commit into from
Oct 5, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 1 addition & 5 deletions docs/src/ref_impl/creating_appvm.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,23 +28,20 @@ vms = with pkgs; [
{
name = "chromium";
packages = [chromium];
ipAddress = "192.168.101.5/24";
macAddress = "02:00:00:03:03:05";
ramMb = 3072;
cores = 4;
}
{
name = "gala";
packages = [(pkgs.callPackage ../user-apps/gala {})];
ipAddress = "192.168.101.6/24";
macAddress = "02:00:00:03:03:06";
ramMb = 1536;
cores = 2;
}
{
name = "zathura";
packages = [zathura];
ipAddress = "192.168.101.7/24";
macAddress = "02:00:00:03:03:07";
ramMb = 512;
cores = 1;
Expand All @@ -57,9 +54,8 @@ Each VM has the following properties:

| **Property** | **Type** | **Unique** | **Description** | **Example** |
| -------------- | --------------------------- | ------------ | --------------------------------------------------------------------------------------------------------------- | --------------------- |
| name | str | yes | This name is prefixed with `vm-` and will be shown in microvm list. The prefixed name - e.g. `vm-chromium` will be also the VM hostname. | “chromium” |
| name | str | yes | This name is postfixed with `-vm` and will be shown in microvm list. The name - e.g. `chromium-vm` will be also the VM hostname. The lenght of the name must be 8 characters or less. | “chromium” |
| packages | list of types.package | no | Packages to include in a VM. It is possible to make it empty or add several packages. | [chromium top] |
| ipAddress | str | yes | This IP will be used to access a VM from the host. Should has the same subnetwork, as other VMs: Net, GUI VMs. | "192.168.101.5/24" |
| macAddress | str | yes | Needed for network configuration. | "02:00:00:03:03:05" |
| ramMb | int, [1, …, host memory] | no | Memory in MB. | 3072 |
| cores | int, [1, …, host cores] | no | Virtual CPU cores. | 4 |
Expand Down
2 changes: 1 addition & 1 deletion modules/host/networking.nix
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ in
# Connect VM tun/tap device to the bridge
# TODO configure this based on IF the netvm is enabled
networks."11-netvm" = {
matchConfig.Name = "vm-*";
matchConfig.Name = "tap-*";
networkConfig.Bridge = "virbr0";
};
};
Expand Down
64 changes: 7 additions & 57 deletions modules/virtualization/microvm/appvm.nix
Original file line number Diff line number Diff line change
Expand Up @@ -8,19 +8,21 @@
}: let
configHost = config;
cfg = config.ghaf.virtualization.microvm.appvm;
waypipe-ssh = pkgs.callPackage ../../../user-apps/waypipe-ssh {};

makeVm = {
vm,
index,
}: let
hostname = "vm-" + vm.name;
vmName = "${vm.name}-vm";
cid =
if vm.cid > 0
then vm.cid
else cfg.vsockBaseCID + index;
appvmConfiguration = {
imports = [
(import ./common/vm-networking.nix {
inherit vmName;
macAddress = vm.macAddress;
})
({
lib,
config,
Expand All @@ -39,22 +41,13 @@
};
};

users.users.${configHost.ghaf.users.accounts.user}.openssh.authorizedKeys.keyFiles = ["${waypipe-ssh}/keys/waypipe-ssh.pub"];
users.users.${configHost.ghaf.users.accounts.user}.openssh.authorizedKeys.keyFiles = ["${pkgs.waypipe-ssh}/keys/waypipe-ssh.pub"];

networking.hostName = hostname;
system.stateVersion = lib.trivial.release;

nixpkgs.buildPlatform.system = configHost.nixpkgs.buildPlatform.system;
nixpkgs.hostPlatform.system = configHost.nixpkgs.hostPlatform.system;

networking = {
enableIPv6 = false;
interfaces.ethint0.useDHCP = false;
firewall.allowedTCPPorts = [22];
firewall.allowedUDPPorts = [67];
useNetworkd = true;
};

environment.systemPackages = [
pkgs.waypipe
];
Expand All @@ -63,7 +56,6 @@
mem = vm.ramMb;
vcpu = vm.cores;
hypervisor = "qemu";

shares = [
{
tag = "ro-store";
Expand All @@ -73,13 +65,6 @@
];
writableStoreOverlay = lib.mkIf config.ghaf.development.debug.tools.enable "/nix/.rw-store";

interfaces = [
{
type = "tap";
id = hostname;
mac = vm.macAddress;
}
];
qemu.extraArgs = [
"-M"
"q35,accel=kvm:tcg,mem-merge=on,sata=off"
Expand All @@ -88,35 +73,6 @@
];
};

networking.nat = {
enable = true;
internalInterfaces = ["ethint0"];
};

# Set internal network's interface name to ethint0
systemd.network.links."10-ethint0" = {
matchConfig.PermanentMACAddress = vm.macAddress;
linkConfig.Name = "ethint0";
};

systemd.network = {
enable = true;
networks."10-ethint0" = {
matchConfig.MACAddress = vm.macAddress;
addresses = [
{
# IP-address for debugging subnet
addressConfig.Address = vm.ipAddress;
}
];
routes = [
{routeConfig.Gateway = "192.168.101.1";}
];
linkConfig.RequiredForOnline = "routable";
linkConfig.ActivationPolicy = "always-up";
};
};

imports = import ../../module-list.nix;
})
];
Expand Down Expand Up @@ -149,12 +105,6 @@ in {
type = types.listOf package;
default = [];
};
ipAddress = mkOption {
description = ''
AppVM's IP address in the inter-vm network
'';
type = str;
};
macAddress = mkOption {
description = ''
AppVM's network interface MAC address
Expand Down Expand Up @@ -216,7 +166,7 @@ in {
config = lib.mkIf cfg.enable {
microvm.vms = (
let
vms = lib.imap0 (index: vm: {"appvm-${vm.name}" = makeVm {inherit index vm;};}) cfg.vms;
vms = lib.imap0 (index: vm: {"${vm.name}-vm" = makeVm {inherit index vm;};}) cfg.vms;
in
lib.foldr lib.recursiveUpdate {} vms
);
Expand Down
52 changes: 52 additions & 0 deletions modules/virtualization/microvm/common/vm-networking.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
# Copyright 2022-2023 TII (SSRC) and the Ghaf contributors
# SPDX-License-Identifier: Apache-2.0
{
vmName,
macAddress,
...
}: let
networkName = "ethint0";
in {
networking = {
hostName = vmName;
enableIPv6 = false;
firewall.allowedTCPPorts = [22];
firewall.allowedUDPPorts = [67];
useNetworkd = true;
nat = {
enable = true;
internalInterfaces = [networkName];
};
};

microvm.interfaces = [
{
type = "tap";
# The interface names must have maximum length of 15 characters
id = "tap-${vmName}";
mikatammi marked this conversation as resolved.
Show resolved Hide resolved
mac = macAddress;
}
];

systemd.network = {
enable = true;
# Set internal network's interface name to networkName
links."10-${networkName}" = {
matchConfig.PermanentMACAddress = macAddress;
linkConfig.Name = networkName;
};
networks."10-${networkName}" = {
matchConfig.MACAddress = macAddress;
DHCP = "yes";
linkConfig.RequiredForOnline = "routable";
linkConfig.ActivationPolicy = "always-up";
};
};

# systemd-resolved does not support local names resolution
# without configuring a local domain. With the local domain,
# one would need also to disable DNSSEC for the clients.
# Disabling DNSSEC for other VM then NetVM is
# completely safe since they use NetVM as DNS proxy.
services.resolved.dnssec = "false";
}
52 changes: 4 additions & 48 deletions modules/virtualization/microvm/guivm.nix
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,12 @@
...
}: let
configHost = config;
vmName = "gui-vm";
macAddress = "02:00:00:02:02:02";
waypipe-ssh = pkgs.callPackage ../../../user-apps/waypipe-ssh {};
guivmBaseConfiguration = {
imports = [
(import ./common/vm-networking.nix {inherit vmName macAddress;})
({
lib,
pkgs,
Expand Down Expand Up @@ -37,24 +40,14 @@
];
};

networking.hostName = "guivm";
system.stateVersion = lib.trivial.release;

nixpkgs.buildPlatform.system = configHost.nixpkgs.buildPlatform.system;
nixpkgs.hostPlatform.system = configHost.nixpkgs.hostPlatform.system;

networking = {
enableIPv6 = false;
interfaces.ethint0.useDHCP = false;
firewall.allowedTCPPorts = [22];
firewall.allowedUDPPorts = [67];
useNetworkd = true;
};

microvm = {
mem = 2048;
hypervisor = "qemu";

shares = [
{
tag = "ro-store";
Expand All @@ -64,49 +57,12 @@
];
writableStoreOverlay = lib.mkIf config.ghaf.development.debug.tools.enable "/nix/.rw-store";

interfaces = [
{
type = "tap";
id = "vm-guivm";
mac = "02:00:00:02:02:02";
}
];

qemu.extraArgs = [
"-device"
"vhost-vsock-pci,guest-cid=${toString cfg.vsockCID}"
];
};

networking.nat = {
enable = true;
internalInterfaces = ["ethint0"];
};

# Set internal network's interface name to ethint0
systemd.network.links."10-ethint0" = {
matchConfig.PermanentMACAddress = "02:00:00:02:02:02";
linkConfig.Name = "ethint0";
};

systemd.network = {
enable = true;
networks."10-ethint0" = {
matchConfig.MACAddress = "02:00:00:02:02:02";
addresses = [
{
# IP-address for debugging subnet
addressConfig.Address = "192.168.101.3/24";
}
];
routes = [
{routeConfig.Gateway = "192.168.101.1";}
];
linkConfig.RequiredForOnline = "routable";
linkConfig.ActivationPolicy = "always-up";
};
};

imports = import ../../module-list.nix;

# Waypipe service runs in the GUIVM and listens for incoming connections from AppVMs
Expand Down Expand Up @@ -173,7 +129,7 @@ in {
};

config = lib.mkIf cfg.enable {
microvm.vms."guivm" = {
microvm.vms."${vmName}" = {
autostart = true;
config =
guivmBaseConfiguration
Expand Down
Loading