From 4aaf12c28ac65c942429575a7b31344bf03c59c4 Mon Sep 17 00:00:00 2001 From: "Andres D. Molins" Date: Thu, 16 May 2024 18:39:09 +0200 Subject: [PATCH 01/10] Problem: A user cannot create a Confidential VM because we don't support that kind of messages, that have a different schema from Instances and Programs. Solution: Create a Confidential message schema with concrete fields. --- .github/workflows/tests.yml | 2 +- aleph_message/models/__init__.py | 16 ++- aleph_message/models/base.py | 1 + aleph_message/models/execution/__init__.py | 8 +- .../models/execution/confidential.py | 28 +++++ aleph_message/models/execution/instance.py | 18 +--- aleph_message/models/execution/volume.py | 14 +++ .../tests/messages/confidential_machine.json | 100 ++++++++++++++++++ aleph_message/tests/test_models.py | 11 ++ 9 files changed, 177 insertions(+), 21 deletions(-) create mode 100644 aleph_message/models/execution/confidential.py create mode 100644 aleph_message/tests/messages/confidential_machine.json diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1665a5a..b0412e0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -31,7 +31,7 @@ jobs: - name: Test with ruff in the Docker image run: | - docker run aleph-message:${GITHUB_REF##*/} ruff aleph_message + docker run aleph-message:${GITHUB_REF##*/} ruff check aleph_message - name: Pytest in the Docker image run: | diff --git a/aleph_message/models/__init__.py b/aleph_message/models/__init__.py index e84adbe..58c72d8 100644 --- a/aleph_message/models/__init__.py +++ b/aleph_message/models/__init__.py @@ -12,6 +12,7 @@ from .abstract import BaseContent from .base import Chain, HashType, MessageType from .execution.base import MachineType, Payment, PaymentType # noqa +from .execution.confidential import ConfidentialContent from .execution.instance import InstanceContent from .execution.program import ProgramContent from .item_hash import ItemHash, ItemType @@ -297,12 +298,18 @@ class InstanceMessage(BaseMessage): content: InstanceContent +class ConfidentialMessage(BaseMessage): + type: Literal[MessageType.confidential] + content: ConfidentialContent + + AlephMessage: TypeAlias = Union[ PostMessage, AggregateMessage, StoreMessage, ProgramMessage, InstanceMessage, + ConfidentialMessage, ForgetMessage, ] @@ -317,11 +324,16 @@ class InstanceMessage(BaseMessage): StoreMessage, ProgramMessage, InstanceMessage, + ConfidentialMessage, ForgetMessage, ] -ExecutableContent: TypeAlias = Union[InstanceContent, ProgramContent] -ExecutableMessage: TypeAlias = Union[InstanceMessage, ProgramMessage] +ExecutableContent: TypeAlias = Union[ + InstanceContent, ProgramContent, ConfidentialContent +] +ExecutableMessage: TypeAlias = Union[ + InstanceMessage, ProgramMessage, ConfidentialMessage +] def parse_message(message_dict: Dict) -> AlephMessage: diff --git a/aleph_message/models/base.py b/aleph_message/models/base.py index 5ed4111..259ac21 100644 --- a/aleph_message/models/base.py +++ b/aleph_message/models/base.py @@ -30,4 +30,5 @@ class MessageType(str, Enum): store = "STORE" program = "PROGRAM" instance = "INSTANCE" + confidential = "CONFIDENTIAL" forget = "FORGET" diff --git a/aleph_message/models/execution/__init__.py b/aleph_message/models/execution/__init__.py index c8fd7a7..591eb02 100644 --- a/aleph_message/models/execution/__init__.py +++ b/aleph_message/models/execution/__init__.py @@ -1,5 +1,11 @@ from .abstract import BaseExecutableContent +from .confidential import ConfidentialContent from .instance import InstanceContent from .program import ProgramContent -__all__ = ["BaseExecutableContent", "InstanceContent", "ProgramContent"] +__all__ = [ + "BaseExecutableContent", + "ConfidentialContent", + "InstanceContent", + "ProgramContent", +] diff --git a/aleph_message/models/execution/confidential.py b/aleph_message/models/execution/confidential.py new file mode 100644 index 0000000..14a8d8e --- /dev/null +++ b/aleph_message/models/execution/confidential.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from pydantic import Field + +from .abstract import BaseExecutableContent +from .base import Payment +from .volume import RootfsVolume +from ..item_hash import ItemHash + + +class ConfidentialPayment(Payment): + """Payment information for a confidential instance execution.""" + + # Added node item hash required on the payment field + node_hash: ItemHash + """Node item hash that execute the message""" + + +class ConfidentialContent(BaseExecutableContent): + """Message content for scheduling a VM confidential instance on the network.""" + + # Make payment field required for confidential messages + payment: ConfidentialPayment = Field( + description="Payment details for the confidential execution" + ) + rootfs: RootfsVolume = Field( + description="Root filesystem of the system, will be booted by the kernel" + ) diff --git a/aleph_message/models/execution/instance.py b/aleph_message/models/execution/instance.py index d4ce223..1d01ddd 100644 --- a/aleph_message/models/execution/instance.py +++ b/aleph_message/models/execution/instance.py @@ -2,24 +2,8 @@ from pydantic import Field -from aleph_message.models.abstract import HashableModel - from .abstract import BaseExecutableContent -from .volume import ParentVolume, PersistentVolumeSizeMib, VolumePersistence - - -class RootfsVolume(HashableModel): - """ - Root file system of a VM instance. - - The root file system of an instance is built as a copy of a reference image, named parent - image. The user determines a custom size and persistence model. - """ - - parent: ParentVolume - persistence: VolumePersistence - # Use the same size constraint as persistent volumes for now - size_mib: PersistentVolumeSizeMib +from .volume import RootfsVolume class InstanceContent(BaseExecutableContent): diff --git a/aleph_message/models/execution/volume.py b/aleph_message/models/execution/volume.py index 6102fba..1507201 100644 --- a/aleph_message/models/execution/volume.py +++ b/aleph_message/models/execution/volume.py @@ -74,4 +74,18 @@ def is_read_only(self): return False +class RootfsVolume(HashableModel): + """ + Root file system of a VM instance. + + The root file system of an instance is built as a copy of a reference image, named parent + image. The user determines a custom size and persistence model. + """ + + parent: ParentVolume + persistence: VolumePersistence + # Use the same size constraint as persistent volumes for now + size_mib: PersistentVolumeSizeMib + + MachineVolume = Union[ImmutableVolume, EphemeralVolume, PersistentVolume] diff --git a/aleph_message/tests/messages/confidential_machine.json b/aleph_message/tests/messages/confidential_machine.json new file mode 100644 index 0000000..9953479 --- /dev/null +++ b/aleph_message/tests/messages/confidential_machine.json @@ -0,0 +1,100 @@ +{ + "_id": { + "$oid": "6080402d7f44efefd611dc1e" + }, + "chain": "ETH", + "sender": "0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba", + "type": "CONFIDENTIAL", + "channel": "Fun-dApps", + "confirmed": true, + "content": { + "address": "0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba", + "allow_amend": false, + "variables": { + "VM_CUSTOM_VARIABLE": "SOMETHING", + "VM_CUSTOM_VARIABLE_2": "32" + }, + "environment": { + "reproducible": true, + "internet": false, + "aleph_api": false, + "shared_cache": false + }, + "resources": { + "vcpus": 1, + "memory": 2048, + "seconds": 30 + }, + "requirements": { + "cpu": { + "architecture": "x86_64" + } + }, + "rootfs": { + "parent": { + "ref": "549ec451d9b099cad112d4aaa2c00ac40fb6729a92ff252ff22eef0b5c3cb613", + "use_latest": true + }, + "persistence": "host", + "size_mib": 20480 + }, + "payment": { + "chain": "AVAX", + "receiver": "0x4145f182EF2F06b45E50468519C1B92C60FBd4A0", + "type": "superfluid", + "node_hash": "589d81de8531bc8f9281ca43610d6640571228eae07cc879c675ffdb28ffaf3c" + }, + "authorized_keys": [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGULT6A41Msmw2KEu0R9MvUjhuWNAsbdeZ0DOwYbt4Qt user@example", + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIH0jqdc5dmt75QhTrWqeHDV9xN8vxbgFyOYs2fuQl7CI", + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDRsrQV1HVrcnskNhyH0may8TG9fHCPawpAi3ZgAWU6V/R7ezvZOHnZdaFeIsOpFbPbt/l67Fur3qniSXllI2kvuh2D4BBJ9PwwlB2sgWzFDF34ADsfLQf+C/vpwrWemEEE91Tpj0dWbnf219i3mZLxy/+5Sv6kUy9YJlzWnDEUbaMAZK2CXrlK90b9Ns7mT82h6h3x9dLF/oCjBAKOSxbH2X+KgsDEZT0soxyluDqKNgKflkav+pvKFyD4J9IWM4j36r80yW+OPGsHqWoWleEhprfNb60RJPwKAYCDiBiSg6wCq5P+kS15O79Ko45wPaYDUwhRoNTcrWeadvTaCZgz9X3KDHgrX6wzdKqzQwtQeabhCaIGLFRMNl1Oy/BR8VozPbIe/mY28IN84An50UYkbve7nOGJucKc4hKxZKEVPpnVpRtIoWGwBJY2fi6C6wy2pBa8UX4C4t9NLJjNQSwFBzYOrphLu3ZW9A+267nogQHGnsJ5xnQ/MXximP3BlwM= user@example" + ], + "volumes": [ + { + "comment": "Python libraries. Read-only since a 'ref' is specified.", + "mount": "/opt/venv", + "ref": "5f31b0706f59404fad3d0bff97ef89ddf24da4761608ea0646329362c662ba51", + "use_latest": false + }, + { + "comment": "Ephemeral storage, read-write but will not persist after the VM stops", + "mount": "/var/cache", + "ephemeral": true, + "size_mib": 5 + }, + { + "comment": "Working data persisted on the VM supervisor, not available on other nodes", + "mount": "/var/lib/sqlite", + "name": "sqlite-data", + "persistence": "host", + "size_mib": 10 + }, + { + "comment": "Working data persisted on the Aleph network. New VMs will try to use the latest version of this volume, with no guarantee against conflicts", + "mount": "/var/lib/statistics", + "name": "statistics", + "persistence": "store", + "size_mib": 10 + }, + { + "comment": "Raw drive to use by a process, do not mount it", + "name": "raw-data", + "persistence": "host", + "size_mib": 10 + } + ], + "replaces": "0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba", + "time": 1619017773.8950517 + }, + "item_type": "inline", + "signature": "0x372da8230552b8c3e65c05b31a0ff3a24666d66c575f8e11019f62579bf48c2b7fe2f0bbe907a2a5bf8050989cdaf8a59ff8a1cbcafcdef0656c54279b4aa0c71b", + "size": 749, + "time": 1619017773.8950577, + "confirmations": [ + { + "chain": "ETH", + "height": 12284734, + "hash": "0x67f2f3cde5e94e70615c92629c70d22dc959a118f46e9411b29659c2fce87cdc" + } + ] +} diff --git a/aleph_message/tests/test_models.py b/aleph_message/tests/test_models.py index 5f07bcd..7b49db4 100644 --- a/aleph_message/tests/test_models.py +++ b/aleph_message/tests/test_models.py @@ -12,6 +12,7 @@ from aleph_message.exceptions import UnknownHashError from aleph_message.models import ( AggregateMessage, + ConfidentialMessage, ForgetMessage, InstanceMessage, ItemType, @@ -149,6 +150,16 @@ def test_instance_message_machine(): assert hash(message.content) +def test_confidential_message_machine(): + path = Path( + os.path.abspath(os.path.join(__file__, "../messages/confidential_machine.json")) + ) + message = create_message_from_file(path, factory=ConfidentialMessage) + + assert isinstance(message, ConfidentialMessage) + assert hash(message.content) + + def test_message_machine_port_mapping(): message_dict = { "chain": "ETH", From 3ebfc049f841e30c0035b4e0e3a0c20679227fe5 Mon Sep 17 00:00:00 2001 From: "Andres D. Molins" Date: Tue, 18 Jun 2024 11:59:49 +0200 Subject: [PATCH 02/10] Fix: Removed old implementation and Added new one using a field instead of a new message type. --- aleph_message/exceptions.py | 3 +- aleph_message/models/__init__.py | 16 ++--------- aleph_message/models/base.py | 1 - aleph_message/models/execution/__init__.py | 2 -- aleph_message/models/execution/abstract.py | 11 ++++++-- .../models/execution/confidential.py | 28 ------------------- aleph_message/models/execution/environment.py | 21 +++++++++++++- aleph_message/models/execution/instance.py | 22 ++++++++++++++- aleph_message/models/execution/volume.py | 17 ++--------- ...son => instance_confidential_machine.json} | 17 +++++------ aleph_message/tests/test_models.py | 11 ++++---- 11 files changed, 68 insertions(+), 81 deletions(-) delete mode 100644 aleph_message/models/execution/confidential.py rename aleph_message/tests/messages/{confidential_machine.json => instance_confidential_machine.json} (90%) diff --git a/aleph_message/exceptions.py b/aleph_message/exceptions.py index de043c3..c8f7873 100644 --- a/aleph_message/exceptions.py +++ b/aleph_message/exceptions.py @@ -1 +1,2 @@ -class UnknownHashError(ValueError): ... +class UnknownHashError(ValueError): + ... diff --git a/aleph_message/models/__init__.py b/aleph_message/models/__init__.py index 58c72d8..e84adbe 100644 --- a/aleph_message/models/__init__.py +++ b/aleph_message/models/__init__.py @@ -12,7 +12,6 @@ from .abstract import BaseContent from .base import Chain, HashType, MessageType from .execution.base import MachineType, Payment, PaymentType # noqa -from .execution.confidential import ConfidentialContent from .execution.instance import InstanceContent from .execution.program import ProgramContent from .item_hash import ItemHash, ItemType @@ -298,18 +297,12 @@ class InstanceMessage(BaseMessage): content: InstanceContent -class ConfidentialMessage(BaseMessage): - type: Literal[MessageType.confidential] - content: ConfidentialContent - - AlephMessage: TypeAlias = Union[ PostMessage, AggregateMessage, StoreMessage, ProgramMessage, InstanceMessage, - ConfidentialMessage, ForgetMessage, ] @@ -324,16 +317,11 @@ class ConfidentialMessage(BaseMessage): StoreMessage, ProgramMessage, InstanceMessage, - ConfidentialMessage, ForgetMessage, ] -ExecutableContent: TypeAlias = Union[ - InstanceContent, ProgramContent, ConfidentialContent -] -ExecutableMessage: TypeAlias = Union[ - InstanceMessage, ProgramMessage, ConfidentialMessage -] +ExecutableContent: TypeAlias = Union[InstanceContent, ProgramContent] +ExecutableMessage: TypeAlias = Union[InstanceMessage, ProgramMessage] def parse_message(message_dict: Dict) -> AlephMessage: diff --git a/aleph_message/models/base.py b/aleph_message/models/base.py index 259ac21..5ed4111 100644 --- a/aleph_message/models/base.py +++ b/aleph_message/models/base.py @@ -30,5 +30,4 @@ class MessageType(str, Enum): store = "STORE" program = "PROGRAM" instance = "INSTANCE" - confidential = "CONFIDENTIAL" forget = "FORGET" diff --git a/aleph_message/models/execution/__init__.py b/aleph_message/models/execution/__init__.py index 591eb02..9659ab2 100644 --- a/aleph_message/models/execution/__init__.py +++ b/aleph_message/models/execution/__init__.py @@ -1,11 +1,9 @@ from .abstract import BaseExecutableContent -from .confidential import ConfidentialContent from .instance import InstanceContent from .program import ProgramContent __all__ = [ "BaseExecutableContent", - "ConfidentialContent", "InstanceContent", "ProgramContent", ] diff --git a/aleph_message/models/execution/abstract.py b/aleph_message/models/execution/abstract.py index c6271b5..a94f254 100644 --- a/aleph_message/models/execution/abstract.py +++ b/aleph_message/models/execution/abstract.py @@ -1,13 +1,18 @@ from __future__ import annotations from abc import ABC -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Union from pydantic import Field from ..abstract import BaseContent, HashableModel from .base import Payment -from .environment import FunctionEnvironment, HostRequirements, MachineResources +from .environment import ( + FunctionEnvironment, + HostRequirements, + InstanceEnvironment, + MachineResources, +) from .volume import MachineVolume @@ -22,7 +27,7 @@ class BaseExecutableContent(HashableModel, BaseContent, ABC): variables: Optional[Dict[str, str]] = Field( default=None, description="Environment variables available in the VM" ) - environment: FunctionEnvironment = Field( + environment: Union[FunctionEnvironment, InstanceEnvironment] = Field( description="Properties of the execution environment" ) resources: MachineResources = Field(description="System resources required") diff --git a/aleph_message/models/execution/confidential.py b/aleph_message/models/execution/confidential.py deleted file mode 100644 index 14a8d8e..0000000 --- a/aleph_message/models/execution/confidential.py +++ /dev/null @@ -1,28 +0,0 @@ -from __future__ import annotations - -from pydantic import Field - -from .abstract import BaseExecutableContent -from .base import Payment -from .volume import RootfsVolume -from ..item_hash import ItemHash - - -class ConfidentialPayment(Payment): - """Payment information for a confidential instance execution.""" - - # Added node item hash required on the payment field - node_hash: ItemHash - """Node item hash that execute the message""" - - -class ConfidentialContent(BaseExecutableContent): - """Message content for scheduling a VM confidential instance on the network.""" - - # Make payment field required for confidential messages - payment: ConfidentialPayment = Field( - description="Payment details for the confidential execution" - ) - rootfs: RootfsVolume = Field( - description="Root filesystem of the system, will be booted by the kernel" - ) diff --git a/aleph_message/models/execution/environment.py b/aleph_message/models/execution/environment.py index 1e97073..9bc007e 100644 --- a/aleph_message/models/execution/environment.py +++ b/aleph_message/models/execution/environment.py @@ -7,6 +7,7 @@ from ...utils import Mebibytes from ..abstract import HashableModel +from ..item_hash import ItemHash class Subscription(HashableModel): @@ -92,7 +93,22 @@ class FunctionEnvironment(HashableModel): internet: bool = False aleph_api: bool = False shared_cache: bool = False - hypervisor: Optional[HypervisorType] + + +class InstanceEnvironment(HashableModel): + reproducible: bool = False + internet: bool = False + aleph_api: bool = False + shared_cache: bool = False + hypervisor: Optional[HypervisorType] = Field( + default=None, description="Hypervisor application to use. Default value is QEmu" + ) + confidential: Optional[bool] = Field( + default=False, description="Confidential Execution Environment" + ) + confidential_policy: Optional[str] = Field( + default=None, description="Confidential Policy specification" + ) class NodeRequirements(HashableModel): @@ -100,6 +116,9 @@ class NodeRequirements(HashableModel): address_regex: Optional[str] = Field( default=None, description="Node address must match this regular expression" ) + node_hash: Optional[ItemHash] = Field( + default=None, description="Hash of the compute resource node that must be used" + ) class Config: extra = Extra.forbid diff --git a/aleph_message/models/execution/instance.py b/aleph_message/models/execution/instance.py index 1d01ddd..ebb8d48 100644 --- a/aleph_message/models/execution/instance.py +++ b/aleph_message/models/execution/instance.py @@ -2,13 +2,33 @@ from pydantic import Field +from aleph_message.models.abstract import HashableModel + from .abstract import BaseExecutableContent -from .volume import RootfsVolume +from .environment import InstanceEnvironment +from .volume import ParentVolume, PersistentVolumeSizeMib, VolumePersistence + + +class RootfsVolume(HashableModel): + """ + Root file system of a VM instance. + + The root file system of an instance is built as a copy of a reference image, named parent + image. The user determines a custom size and persistence model. + """ + + parent: ParentVolume + persistence: VolumePersistence + # Use the same size constraint as persistent volumes for now + size_mib: PersistentVolumeSizeMib class InstanceContent(BaseExecutableContent): """Message content for scheduling a VM instance on the network.""" + environment: InstanceEnvironment = Field( + description="Properties of the instance execution environment" + ) rootfs: RootfsVolume = Field( description="Root filesystem of the system, will be booted by the kernel" ) diff --git a/aleph_message/models/execution/volume.py b/aleph_message/models/execution/volume.py index 1507201..0fd0c9e 100644 --- a/aleph_message/models/execution/volume.py +++ b/aleph_message/models/execution/volume.py @@ -16,7 +16,8 @@ class AbstractVolume(HashableModel, ABC): mount: Optional[str] = None @abstractmethod - def is_read_only(self): ... + def is_read_only(self): + ... class Config: extra = Extra.forbid @@ -74,18 +75,4 @@ def is_read_only(self): return False -class RootfsVolume(HashableModel): - """ - Root file system of a VM instance. - - The root file system of an instance is built as a copy of a reference image, named parent - image. The user determines a custom size and persistence model. - """ - - parent: ParentVolume - persistence: VolumePersistence - # Use the same size constraint as persistent volumes for now - size_mib: PersistentVolumeSizeMib - - MachineVolume = Union[ImmutableVolume, EphemeralVolume, PersistentVolume] diff --git a/aleph_message/tests/messages/confidential_machine.json b/aleph_message/tests/messages/instance_confidential_machine.json similarity index 90% rename from aleph_message/tests/messages/confidential_machine.json rename to aleph_message/tests/messages/instance_confidential_machine.json index 9953479..723ab5e 100644 --- a/aleph_message/tests/messages/confidential_machine.json +++ b/aleph_message/tests/messages/instance_confidential_machine.json @@ -4,7 +4,7 @@ }, "chain": "ETH", "sender": "0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba", - "type": "CONFIDENTIAL", + "type": "INSTANCE", "channel": "Fun-dApps", "confirmed": true, "content": { @@ -18,11 +18,14 @@ "reproducible": true, "internet": false, "aleph_api": false, - "shared_cache": false + "shared_cache": false, + "hypervisor": "qemu", + "confidential": true, + "confidential_policy": "0x1" }, "resources": { "vcpus": 1, - "memory": 2048, + "memory": 128, "seconds": 30 }, "requirements": { @@ -36,13 +39,7 @@ "use_latest": true }, "persistence": "host", - "size_mib": 20480 - }, - "payment": { - "chain": "AVAX", - "receiver": "0x4145f182EF2F06b45E50468519C1B92C60FBd4A0", - "type": "superfluid", - "node_hash": "589d81de8531bc8f9281ca43610d6640571228eae07cc879c675ffdb28ffaf3c" + "size_mib": 20000 }, "authorized_keys": [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGULT6A41Msmw2KEu0R9MvUjhuWNAsbdeZ0DOwYbt4Qt user@example", diff --git a/aleph_message/tests/test_models.py b/aleph_message/tests/test_models.py index 7b49db4..98e6d1d 100644 --- a/aleph_message/tests/test_models.py +++ b/aleph_message/tests/test_models.py @@ -12,7 +12,6 @@ from aleph_message.exceptions import UnknownHashError from aleph_message.models import ( AggregateMessage, - ConfidentialMessage, ForgetMessage, InstanceMessage, ItemType, @@ -150,14 +149,16 @@ def test_instance_message_machine(): assert hash(message.content) -def test_confidential_message_machine(): +def test_instance_message_machine_with_confidential_options(): path = Path( - os.path.abspath(os.path.join(__file__, "../messages/confidential_machine.json")) + os.path.abspath(os.path.join(__file__, "../messages/instance_machine.json")) ) - message = create_message_from_file(path, factory=ConfidentialMessage) + message = create_message_from_file(path, factory=InstanceMessage) - assert isinstance(message, ConfidentialMessage) + assert isinstance(message, InstanceMessage) assert hash(message.content) + assert message.content.environment.confidential is True + assert message.content.environment.confidential_policy == "0x1" def test_message_machine_port_mapping(): From 20a6e092a54c22d4ce2ed6a2f3cca7a1b628af03 Mon Sep 17 00:00:00 2001 From: "Andres D. Molins" Date: Tue, 18 Jun 2024 12:10:30 +0200 Subject: [PATCH 03/10] Fix: Solve code quality issues. --- aleph_message/exceptions.py | 3 +-- aleph_message/models/execution/volume.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/aleph_message/exceptions.py b/aleph_message/exceptions.py index c8f7873..de043c3 100644 --- a/aleph_message/exceptions.py +++ b/aleph_message/exceptions.py @@ -1,2 +1 @@ -class UnknownHashError(ValueError): - ... +class UnknownHashError(ValueError): ... diff --git a/aleph_message/models/execution/volume.py b/aleph_message/models/execution/volume.py index 0fd0c9e..6102fba 100644 --- a/aleph_message/models/execution/volume.py +++ b/aleph_message/models/execution/volume.py @@ -16,8 +16,7 @@ class AbstractVolume(HashableModel, ABC): mount: Optional[str] = None @abstractmethod - def is_read_only(self): - ... + def is_read_only(self): ... class Config: extra = Extra.forbid From e5c1aaa0e2d56e486c0bb14c19fa4509594d44bd Mon Sep 17 00:00:00 2001 From: "Andres D. Molins" Date: Tue, 18 Jun 2024 13:03:14 +0200 Subject: [PATCH 04/10] Fix: Solved test issues after last changes. --- .../tests/messages/instance_confidential_machine.json | 3 +++ aleph_message/tests/test_models.py | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/aleph_message/tests/messages/instance_confidential_machine.json b/aleph_message/tests/messages/instance_confidential_machine.json index 723ab5e..655d35d 100644 --- a/aleph_message/tests/messages/instance_confidential_machine.json +++ b/aleph_message/tests/messages/instance_confidential_machine.json @@ -31,6 +31,9 @@ "requirements": { "cpu": { "architecture": "x86_64" + }, + "node": { + "node_hash": "4d4db19afca380fdf06ba7f916153d0f740db9de9eee23ad26ba96a90d8a2920" } }, "rootfs": { diff --git a/aleph_message/tests/test_models.py b/aleph_message/tests/test_models.py index 98e6d1d..50d337b 100644 --- a/aleph_message/tests/test_models.py +++ b/aleph_message/tests/test_models.py @@ -151,7 +151,7 @@ def test_instance_message_machine(): def test_instance_message_machine_with_confidential_options(): path = Path( - os.path.abspath(os.path.join(__file__, "../messages/instance_machine.json")) + os.path.abspath(os.path.join(__file__, "../messages/instance_confidential_machine.json")) ) message = create_message_from_file(path, factory=InstanceMessage) @@ -159,6 +159,7 @@ def test_instance_message_machine_with_confidential_options(): assert hash(message.content) assert message.content.environment.confidential is True assert message.content.environment.confidential_policy == "0x1" + assert message.content.requirements.node.node_hash == "4d4db19afca380fdf06ba7f916153d0f740db9de9eee23ad26ba96a90d8a2920" def test_message_machine_port_mapping(): From 44bea47f14128ed9ddcfd3c6883d8c0049a92c79 Mon Sep 17 00:00:00 2001 From: "Andres D. Molins" Date: Tue, 18 Jun 2024 13:10:39 +0200 Subject: [PATCH 05/10] Fix: Solved code quality issues on tests caused by different black versions. --- aleph_message/tests/test_models.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/aleph_message/tests/test_models.py b/aleph_message/tests/test_models.py index 50d337b..6b13653 100644 --- a/aleph_message/tests/test_models.py +++ b/aleph_message/tests/test_models.py @@ -151,7 +151,9 @@ def test_instance_message_machine(): def test_instance_message_machine_with_confidential_options(): path = Path( - os.path.abspath(os.path.join(__file__, "../messages/instance_confidential_machine.json")) + os.path.abspath( + os.path.join(__file__, "../messages/instance_confidential_machine.json") + ) ) message = create_message_from_file(path, factory=InstanceMessage) @@ -159,7 +161,10 @@ def test_instance_message_machine_with_confidential_options(): assert hash(message.content) assert message.content.environment.confidential is True assert message.content.environment.confidential_policy == "0x1" - assert message.content.requirements.node.node_hash == "4d4db19afca380fdf06ba7f916153d0f740db9de9eee23ad26ba96a90d8a2920" + assert ( + message.content.requirements.node.node_hash + == "4d4db19afca380fdf06ba7f916153d0f740db9de9eee23ad26ba96a90d8a2920" + ) def test_message_machine_port_mapping(): From 417d070fd62c961df58fc2c2132fd281e0b1a2b2 Mon Sep 17 00:00:00 2001 From: "Andres D. Molins" Date: Tue, 18 Jun 2024 13:40:04 +0200 Subject: [PATCH 06/10] Fix: Solve mypy issue on CI. --- aleph_message/tests/test_models.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/aleph_message/tests/test_models.py b/aleph_message/tests/test_models.py index 6b13653..51157e5 100644 --- a/aleph_message/tests/test_models.py +++ b/aleph_message/tests/test_models.py @@ -161,10 +161,12 @@ def test_instance_message_machine_with_confidential_options(): assert hash(message.content) assert message.content.environment.confidential is True assert message.content.environment.confidential_policy == "0x1" - assert ( - message.content.requirements.node.node_hash - == "4d4db19afca380fdf06ba7f916153d0f740db9de9eee23ad26ba96a90d8a2920" - ) + if message.content.requirements: + if message.content.requirements.node: + assert ( + message.content.requirements.node.node_hash + == "4d4db19afca380fdf06ba7f916153d0f740db9de9eee23ad26ba96a90d8a2920" + ) def test_message_machine_port_mapping(): From 32a3209a5b95b9c08bbd0b0a47ac7a204bb164a1 Mon Sep 17 00:00:00 2001 From: "Andres D. Molins" Date: Tue, 18 Jun 2024 14:27:03 +0200 Subject: [PATCH 07/10] Fix: Added missing firmware field for confidentials. --- aleph_message/models/execution/environment.py | 3 +++ .../tests/messages/instance_confidential_machine.json | 3 ++- aleph_message/tests/test_models.py | 4 ++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/aleph_message/models/execution/environment.py b/aleph_message/models/execution/environment.py index 9bc007e..9b095d0 100644 --- a/aleph_message/models/execution/environment.py +++ b/aleph_message/models/execution/environment.py @@ -109,6 +109,9 @@ class InstanceEnvironment(HashableModel): confidential_policy: Optional[str] = Field( default=None, description="Confidential Policy specification" ) + firmware: Optional[ItemHash] = Field( + default=None, description="Confidential OVMF firmware to use" + ) class NodeRequirements(HashableModel): diff --git a/aleph_message/tests/messages/instance_confidential_machine.json b/aleph_message/tests/messages/instance_confidential_machine.json index 655d35d..8ec4899 100644 --- a/aleph_message/tests/messages/instance_confidential_machine.json +++ b/aleph_message/tests/messages/instance_confidential_machine.json @@ -21,7 +21,8 @@ "shared_cache": false, "hypervisor": "qemu", "confidential": true, - "confidential_policy": "0x1" + "confidential_policy": "0x1", + "firmware": "e258d248fda94c63753607f7c4494ee0fcbe92f1a76bfdac795c9d84101eb317" }, "resources": { "vcpus": 1, diff --git a/aleph_message/tests/test_models.py b/aleph_message/tests/test_models.py index 51157e5..7ab75c9 100644 --- a/aleph_message/tests/test_models.py +++ b/aleph_message/tests/test_models.py @@ -161,6 +161,10 @@ def test_instance_message_machine_with_confidential_options(): assert hash(message.content) assert message.content.environment.confidential is True assert message.content.environment.confidential_policy == "0x1" + assert ( + message.content.environment.firmware + == "e258d248fda94c63753607f7c4494ee0fcbe92f1a76bfdac795c9d84101eb317" + ) if message.content.requirements: if message.content.requirements.node: assert ( From adccff707f2d4cf8e77ed41ab7e6e6c093b27f77 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 18 Jun 2024 15:58:06 +0200 Subject: [PATCH 08/10] Add class TrustedExecutionEnvironment --- aleph_message/models/execution/environment.py | 46 +++++++++++++++---- .../instance_confidential_machine.json | 7 +-- aleph_message/tests/test_models.py | 30 +++++------- 3 files changed, 54 insertions(+), 29 deletions(-) diff --git a/aleph_message/models/execution/environment.py b/aleph_message/models/execution/environment.py index 9b095d0..355f20d 100644 --- a/aleph_message/models/execution/environment.py +++ b/aleph_message/models/execution/environment.py @@ -95,6 +95,41 @@ class FunctionEnvironment(HashableModel): shared_cache: bool = False +class AMDSEVPolicy(int, Enum): + """AMD Guest Policy for SEV-ES and SEV. + + The firmware maintains a guest policy provided by the guest owner. This policy is enforced by the + firmware and restricts what configuration and operational commands can be performed on this + guest by the hypervisor. The policy also requires a minimum firmware level. + + The policy comprises a set of flags that can be combined with bitwise OR. + + See https://github.com/virtee/sev/blob/fbfed998930a0d1e6126462b371890b9f8d77148/src/launch/sev.rs#L245 for reference. + """ + + NO_DBG = 0b1 # Debugging of the guest is disallowed + NO_KS = 0b10 # Sharing keys with other guests is disallowed + SEV_ES = 0b100 # SEV-ES is required + NO_SEND = 0b1000 # Sending the guest to another platform is disallowed + DOMAIN = 0b10000 # The guest must not be transmitted to another platform that is not in the domain + SEV = 0b100000 # The guest must not be transmitted to another platform that is not SEV capable + + +class TrustedExecutionEnvironment(HashableModel): + """Trusted Execution Environment properties.""" + + firmware: Optional[ItemHash] = Field( + default=None, description="Confidential OVMF firmware to use" + ) + policy: int = Field( + default=AMDSEVPolicy.NO_DBG, + description="Policy of the TEE. Default value is 0x01 for SEV without debugging.", + ) + + class Config: + extra = Extra.allow + + class InstanceEnvironment(HashableModel): reproducible: bool = False internet: bool = False @@ -103,14 +138,9 @@ class InstanceEnvironment(HashableModel): hypervisor: Optional[HypervisorType] = Field( default=None, description="Hypervisor application to use. Default value is QEmu" ) - confidential: Optional[bool] = Field( - default=False, description="Confidential Execution Environment" - ) - confidential_policy: Optional[str] = Field( - default=None, description="Confidential Policy specification" - ) - firmware: Optional[ItemHash] = Field( - default=None, description="Confidential OVMF firmware to use" + trusted_execution: Optional[TrustedExecutionEnvironment] = Field( + default=None, + description="Trusted Execution Environment properties. Defaults to no TEE.", ) diff --git a/aleph_message/tests/messages/instance_confidential_machine.json b/aleph_message/tests/messages/instance_confidential_machine.json index 8ec4899..791a4a9 100644 --- a/aleph_message/tests/messages/instance_confidential_machine.json +++ b/aleph_message/tests/messages/instance_confidential_machine.json @@ -20,9 +20,10 @@ "aleph_api": false, "shared_cache": false, "hypervisor": "qemu", - "confidential": true, - "confidential_policy": "0x1", - "firmware": "e258d248fda94c63753607f7c4494ee0fcbe92f1a76bfdac795c9d84101eb317" + "trusted_execution": { + "policy": 1, + "firmware": "e258d248fda94c63753607f7c4494ee0fcbe92f1a76bfdac795c9d84101eb317" + } }, "resources": { "vcpus": 1, diff --git a/aleph_message/tests/test_models.py b/aleph_message/tests/test_models.py index 7ab75c9..f80228d 100644 --- a/aleph_message/tests/test_models.py +++ b/aleph_message/tests/test_models.py @@ -26,6 +26,7 @@ create_new_message, parse_message, ) +from aleph_message.models.execution.environment import AMDSEVPolicy from aleph_message.tests.download_messages import MESSAGES_STORAGE_PATH console = Console(color_system="windows") @@ -130,7 +131,7 @@ def test_post_content(): def test_message_machine(): - path = Path(os.path.abspath(os.path.join(__file__, "../messages/machine.json"))) + path = Path(__file__).parent / "messages/machine.json" message = create_message_from_file(path, factory=ProgramMessage) assert isinstance(message, ProgramMessage) @@ -140,9 +141,7 @@ def test_message_machine(): def test_instance_message_machine(): - path = Path( - os.path.abspath(os.path.join(__file__, "../messages/instance_machine.json")) - ) + path = Path(__file__).parent / "messages/instance_machine.json" message = create_message_from_file(path, factory=InstanceMessage) assert isinstance(message, InstanceMessage) @@ -150,27 +149,22 @@ def test_instance_message_machine(): def test_instance_message_machine_with_confidential_options(): - path = Path( - os.path.abspath( - os.path.join(__file__, "../messages/instance_confidential_machine.json") - ) - ) + path = Path(__file__).parent / "messages/instance_confidential_machine.json" message = create_message_from_file(path, factory=InstanceMessage) assert isinstance(message, InstanceMessage) assert hash(message.content) - assert message.content.environment.confidential is True - assert message.content.environment.confidential_policy == "0x1" + assert message.content.environment.trusted_execution + assert message.content.environment.trusted_execution.policy == AMDSEVPolicy.NO_DBG assert ( - message.content.environment.firmware + message.content.environment.trusted_execution.firmware == "e258d248fda94c63753607f7c4494ee0fcbe92f1a76bfdac795c9d84101eb317" ) - if message.content.requirements: - if message.content.requirements.node: - assert ( - message.content.requirements.node.node_hash - == "4d4db19afca380fdf06ba7f916153d0f740db9de9eee23ad26ba96a90d8a2920" - ) + assert message.content.requirements and message.content.requirements.node + assert ( + message.content.requirements.node.node_hash + == "4d4db19afca380fdf06ba7f916153d0f740db9de9eee23ad26ba96a90d8a2920" + ) def test_message_machine_port_mapping(): From c1c183e7d5273fb3c8d8a79fc62c851b1f9620bd Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 18 Jun 2024 15:59:56 +0200 Subject: [PATCH 09/10] Add comment on retro-compatibility --- aleph_message/models/execution/environment.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/aleph_message/models/execution/environment.py b/aleph_message/models/execution/environment.py index 355f20d..6816a43 100644 --- a/aleph_message/models/execution/environment.py +++ b/aleph_message/models/execution/environment.py @@ -131,10 +131,8 @@ class Config: class InstanceEnvironment(HashableModel): - reproducible: bool = False internet: bool = False aleph_api: bool = False - shared_cache: bool = False hypervisor: Optional[HypervisorType] = Field( default=None, description="Hypervisor application to use. Default value is QEmu" ) @@ -142,6 +140,9 @@ class InstanceEnvironment(HashableModel): default=None, description="Trusted Execution Environment properties. Defaults to no TEE.", ) + # The following fields are kept for retro-compatibility. + reproducible: bool = False + shared_cache: bool = False class NodeRequirements(HashableModel): From 522025732f51b99e13e52646584148c3ecb36d14 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Tue, 18 Jun 2024 16:21:37 +0200 Subject: [PATCH 10/10] Improve tests --- aleph_message/models/execution/environment.py | 8 +++++++- aleph_message/tests/test_models.py | 17 +++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/aleph_message/models/execution/environment.py b/aleph_message/models/execution/environment.py index 6816a43..d320fe9 100644 --- a/aleph_message/models/execution/environment.py +++ b/aleph_message/models/execution/environment.py @@ -3,7 +3,7 @@ from enum import Enum from typing import List, Literal, Optional, Union -from pydantic import Extra, Field +from pydantic import Extra, Field, validator from ...utils import Mebibytes from ..abstract import HashableModel @@ -144,6 +144,12 @@ class InstanceEnvironment(HashableModel): reproducible: bool = False shared_cache: bool = False + @validator("trusted_execution", pre=True) + def check_hypervisor(cls, v, values): + if v and values.get("hypervisor") != HypervisorType.qemu: + raise ValueError("Trusted Execution Environment is only supported for QEmu") + return v + class NodeRequirements(HashableModel): owner: Optional[str] = Field(default=None, description="Address of the node owner") diff --git a/aleph_message/tests/test_models.py b/aleph_message/tests/test_models.py index f80228d..78cdf04 100644 --- a/aleph_message/tests/test_models.py +++ b/aleph_message/tests/test_models.py @@ -167,6 +167,23 @@ def test_instance_message_machine_with_confidential_options(): ) +def test_validation_on_confidential_options(): + """Ensure that a trusted environment is only allowed for QEmu.""" + path = Path(__file__).parent / "messages/instance_confidential_machine.json" + message_dict = json.loads(path.read_text()) + # Patch the hypervisor to be something other than QEmu + message_dict["content"]["environment"]["hypervisor"] = "firecracker" + try: + _ = create_new_message(message_dict, factory=InstanceMessage) + raise AssertionError("An exception should have been raised before this point.") + except ValidationError as e: + assert e.errors()[0]["loc"] == ("content", "environment", "trusted_execution") + assert ( + e.errors()[0]["msg"] + == "Trusted Execution Environment is only supported for QEmu" + ) + + def test_message_machine_port_mapping(): message_dict = { "chain": "ETH",