Solve duplicated network issues for ephemeral VMs (#680)

* Fix: Solve failing test removing it because is not used.

* Problem: If the service restarts, the diagnostic VM fails for network issues.

Solution: Loading already loaded VMs filtering by only persistent ones.

* Fix: Replaced interface check by interface remove and re-creation.

* Fix: Ensure to delete the IPv6 address first before trying to delete the interface to prevent if the deletion fails.

* Fix: Also delete the IPv4 ip to prevent 2 interfaces with the same IPv4.

---------

Co-authored-by: Andres D. Molins <[email protected]>

src/aleph/vm/network/interfaces.py

@@ -61,6 +61,19 @@ def add_ip_address(ipr: IPRoute, device_name: str, ip: Union[IPv4Interface, IPv6
         logger.error(f"Unknown exception while adding address {ip} to interface {device_name}: {e}")
 
 
+def delete_ip_address(ipr: IPRoute, device_name: str, ip: Union[IPv4Interface, IPv6Interface]):
+    """Delete an IP address to the given interface."""
+    interface_index: list[int] = ipr.link_lookup(ifname=device_name)
+    if not interface_index:
+        raise MissingInterfaceError(f"Interface {device_name} does not exist, can't delete address {ip} to it.")
+    try:
+        ipr.addr("del", index=interface_index[0], address=str(ip.ip), mask=ip.network.prefixlen)
+    except NetlinkError as e:
+        logger.error(f"Unknown exception while deleting address {ip} to interface {device_name}: {e}")
+    except OSError as e:
+        logger.error(f"Unknown exception while deleting address {ip} to interface {device_name}: {e}")
+
+
 def set_link_up(ipr: IPRoute, device_name: str):
     """Set the given interface up."""
     interface_index: list[int] = ipr.link_lookup(ifname=device_name)
@@ -154,4 +167,6 @@ async def delete(self) -> None:
         if self.ndp_proxy:
             await self.ndp_proxy.delete_range(self.device_name)
         with IPRoute() as ipr:
+            delete_ip_address(ipr, self.device_name, self.host_ip)
+            delete_ip_address(ipr, self.device_name, self.host_ipv6)
             delete_tap_interface(ipr, self.device_name)

src/aleph/vm/pool.py

@@ -115,6 +115,9 @@ async def create_a_vm(
                 if self.network:
                     vm_type = VmType.from_message_content(message)
                     tap_interface = await self.network.prepare_tap(vm_id, vm_hash, vm_type)
+                    # If the network interface already exists, remove it and then re-create it.
+                    if self.network.interface_exists(vm_id):
+                        await tap_interface.delete()
                     await self.network.create_tap(vm_id, tap_interface)
                 else:
                     tap_interface = None
@@ -163,12 +166,6 @@ def get_unique_vm_id(self) -> int:
             # anymore.
             currently_used_vm_ids = {execution.vm_id for execution in self.executions.values()}
             for i in range(settings.START_ID_INDEX, 255**2):
-
-                if self.network:
-                    # Check the network interface don't already exists, otherwise it will cause a crash
-                    if self.network.interface_exists(i):
-                        continue
-
                 if i not in currently_used_vm_ids:
                     return i
             else:
@@ -229,8 +226,8 @@ async def load_persistent_executions(self):
         for saved_execution in saved_executions:
             vm_hash = ItemHash(saved_execution.vm_hash)
 
-            if vm_hash in self.executions:
-                # The execution is already loaded, skip it
+            if vm_hash in self.executions or not saved_execution.persistent:
+                # The execution is already loaded or isn't persistent, skip it
                 continue
 
             vm_id = saved_execution.vm_id