716 updating pydantic version

examples/example_fastapi/main.py

@@ -215,9 +215,9 @@ async def check_url(internet_host: HttpUrl, timeout_seconds: int = 5):
 async def read_internet():
     """Check Internet connectivity of the system, requiring IP connectivity, domain resolution and HTTPS/TLS."""
     internet_hosts: list[HttpUrl] = [
-        HttpUrl(url="https://aleph.im/", scheme="https"),
-        HttpUrl(url="https://ethereum.org", scheme="https"),
-        HttpUrl(url="https://ipfs.io/", scheme="https"),
+        HttpUrl(url="https://aleph.im/"),
+        HttpUrl(url="https://ethereum.org/"),
+        HttpUrl(url="https://ipfs.io/"),
     ]
     timeout_seconds = 5
 

packaging/Makefile

@@ -15,8 +15,8 @@ debian-package-code:
 	cp ../examples/instance_message_from_aleph.json ./aleph-vm/opt/aleph-vm/examples/instance_message_from_aleph.json
 	cp -r ../examples/data ./aleph-vm/opt/aleph-vm/examples/data
 	mkdir -p ./aleph-vm/opt/aleph-vm/examples/volumes
-	# Fixing this protobuf dependency version to avoid getting CI errors as version 5.29.0 have this compilation issue
-	pip3 install --progress-bar off --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.5.0' 'eth-account==0.10' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'aleph-superfluid~=0.2.1' 'sqlalchemy[asyncio]>=2.0' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0' 'pyroute2==0.7.12' 'python-cpuid==0.1.0' 'solathon==1.0.2' 'protobuf==5.28.3'
+  # Fixing this protobuf dependency version to avoid getting CI errors as version 5.29.0 have this compilation issue
+	pip3 install --progress-bar off --target ./aleph-vm/opt/aleph-vm/ 'git+https://github.com/aleph-im/aleph-message@108-upgrade-pydantic-version#egg=aleph-message' 'eth-account==0.10' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'aleph-superfluid~=0.2.1' 'sqlalchemy[asyncio]>=2.0' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0' 'pydantic-settings==2.6.1' 'pyroute2==0.7.12' 'python-cpuid==0.1.0' 'solathon==1.0.2' 'protobuf==5.28.3'
 	python3 -m compileall ./aleph-vm/opt/aleph-vm/
 
 debian-package-resources: firecracker-bins vmlinux download-ipfs-kubo  target/bin/sevctl

pyproject.toml

@@ -36,7 +36,7 @@ dependencies = [
   "aioredis==1.3.1",
   "aiosqlite==0.19",
   "alembic==1.13.1",
-  "aleph-message==0.5",
+  "aleph-message @ git+https://github.com/aleph-im/aleph-message@108-upgrade-pydantic-version#egg=main",
   "aleph-superfluid~=0.2.1",
   "dbus-python==1.3.2",
   "eth-account~=0.10",
@@ -49,9 +49,11 @@ dependencies = [
   "protobuf==5.28.3",
   "psutil==5.9.5",
   "py-cpuinfo==9",
-  "pydantic[dotenv]~=1.10.13",
+  "pydantic>=2",
+  "pydantic-settings==2.6.1",
   "pyroute2==0.7.12",
   "python-cpuid==0.1",
+  "python-dotenv",
   "pyyaml==6.0.1",
   "qmp==1.1",
   "schedule==1.2.1",
@@ -121,8 +123,9 @@ dependencies = [
   "mypy==1.8.0",
   "ruff==0.4.6",
   "isort==5.13.2",
-  "yamlfix==1.16.1",
+  "yamlfix==1.17.0",
   "pyproject-fmt==2.2.1",
+  "pydantic>=2",
 ]
 [tool.hatch.envs.linting.scripts]
 typing = "mypy {args:src/aleph/vm/ tests/ examples/example_fastapi runtimes/aleph-debian-12-python}"

src/aleph/vm/conf.py

@@ -9,13 +9,13 @@
 from os.path import abspath, exists, isdir, isfile, join
 from pathlib import Path
 from subprocess import CalledProcessError, check_output
-from typing import Any, Literal, NewType
+from typing import Any, Literal, NewType, Optional
 
 from aleph_message.models import Chain
 from aleph_message.models.execution.environment import HypervisorType
-from pydantic import BaseSettings, Field, HttpUrl
-from pydantic.env_settings import DotenvType, env_file_sentinel
-from pydantic.typing import StrPath
+from dotenv import load_dotenv
+from pydantic import Field, HttpUrl
+from pydantic_settings import BaseSettings, SettingsConfigDict
 
 from aleph.vm.orchestrator.chain import STREAM_CHAINS
 from aleph.vm.utils import (
@@ -25,6 +25,8 @@
     is_command_available,
 )
 
+load_dotenv()
+
 logger = logging.getLogger(__name__)
 
 Url = NewType("Url", str)
@@ -111,7 +113,7 @@
 
 
 class Settings(BaseSettings):
-    SUPERVISOR_HOST = "127.0.0.1"
+    SUPERVISOR_HOST: str = "127.0.0.1"
     SUPERVISOR_PORT: int = 4020
 
     # Public domain name
@@ -123,32 +125,32 @@
     START_ID_INDEX: int = 4
     PREALLOC_VM_COUNT: int = 0
     REUSE_TIMEOUT: float = 60 * 60.0
-    WATCH_FOR_MESSAGES = True
-    WATCH_FOR_UPDATES = True
+    WATCH_FOR_MESSAGES: bool = True
+    WATCH_FOR_UPDATES: bool = True
 
-    API_SERVER = "https://official.aleph.cloud"
+    API_SERVER: str = "https://official.aleph.cloud"
     # Connect to the Quad9 VPN provider using their IPv4 and IPv6 addresses.
-    CONNECTIVITY_IPV4_URL = "https://9.9.9.9/"
-    CONNECTIVITY_IPV6_URL = "https://[2620:fe::fe]/"
-    CONNECTIVITY_DNS_HOSTNAME = "example.org"
+    CONNECTIVITY_IPV4_URL: str = "https://9.9.9.9/"
+    CONNECTIVITY_IPV6_URL: str = "https://[2620:fe::fe]/"
+    CONNECTIVITY_DNS_HOSTNAME: str = "example.org"
 
-    USE_JAILER = True
+    USE_JAILER: bool = True
     # Changelog: PRINT_SYSTEM_LOGS use to print the MicroVM logs with the supervisor output.
     # They are now in separate journald entries, disabling the settings disable the logs output of Firecracker VM (only)
     # via the serial console. This break the logs endpoint for program, as such disabling it in prod is not recommended.
-    PRINT_SYSTEM_LOGS = True
-    IGNORE_TRACEBACK_FROM_DIAGNOSTICS = True
-    LOG_LEVEL = "WARNING"
-    DEBUG_ASYNCIO = False
+    PRINT_SYSTEM_LOGS: bool = True
+    IGNORE_TRACEBACK_FROM_DIAGNOSTICS: bool = True
+    LOG_LEVEL: str = "WARNING"
+    DEBUG_ASYNCIO: bool = False
 
     # Networking does not work inside Docker/Podman
-    ALLOW_VM_NETWORKING = True
+    ALLOW_VM_NETWORKING: bool = True
     NETWORK_INTERFACE: str | None = None
-    IPV4_ADDRESS_POOL = Field(
+    IPV4_ADDRESS_POOL: str = Field(
         default="172.16.0.0/12",
         description="IPv4 address range used to provide networks to VMs.",
     )
-    IPV4_NETWORK_PREFIX_LENGTH = Field(
+    IPV4_NETWORK_PREFIX_LENGTH: int = Field(
         default=24,
         description="Individual VM network prefix length in bits",
     )
@@ -166,7 +168,7 @@
         default=True,
         description="Enable IPv6 forwarding on the host. Required for IPv6 connectivity in VMs.",
     )
-    NFTABLES_CHAIN_PREFIX = "aleph"
+    NFTABLES_CHAIN_PREFIX: str = "aleph"
     USE_NDP_PROXY: bool = Field(
         default=True,
         description="Use the Neighbor Discovery Protocol Proxy to respond to Router Solicitation for instances on IPv6",
@@ -177,43 +179,43 @@
         description="Method used to resolve the dns server if DNS_NAMESERVERS is not present.",
     )
     DNS_NAMESERVERS: list[str] | None = None
-    DNS_NAMESERVERS_IPV4: list[str] | None
-    DNS_NAMESERVERS_IPV6: list[str] | None
+    DNS_NAMESERVERS_IPV4: list[str] | None = None
+    DNS_NAMESERVERS_IPV6: list[str] | None = None
 
-    FIRECRACKER_PATH = Path("/opt/firecracker/firecracker")
-    JAILER_PATH = Path("/opt/firecracker/jailer")
-    SEV_CTL_PATH = Path("/opt/sevctl")
-    LINUX_PATH = Path("/opt/firecracker/vmlinux.bin")
+    FIRECRACKER_PATH: Path = Path("/opt/firecracker/firecracker")
+    JAILER_PATH: Path = Path("/opt/firecracker/jailer")
+    SEV_CTL_PATH: Path = Path("/opt/sevctl")
+    LINUX_PATH: Path = Path("/opt/firecracker/vmlinux.bin")
     INIT_TIMEOUT: float = 20.0
 
-    CONNECTOR_URL = Url("http://localhost:4021")
+    CONNECTOR_URL: HttpUrl = HttpUrl("http://localhost:4021")
 
-    CACHE_ROOT = Path("/var/cache/aleph/vm")
-    MESSAGE_CACHE: Path = Field(
+    CACHE_ROOT: Path = Path("/var/cache/aleph/vm")
+    MESSAGE_CACHE: Optional[Path] = Field(
         None,
         description="Default to CACHE_ROOT/message",
     )
-    CODE_CACHE: Path = Field(None, description="Default to CACHE_ROOT/code")
-    RUNTIME_CACHE: Path = Field(None, description="Default to CACHE_ROOT/runtime")
-    DATA_CACHE: Path = Field(None, description="Default to CACHE_ROOT/data")
+    CODE_CACHE: Optional[Path] = Field(None, description="Default to CACHE_ROOT/code")
+    RUNTIME_CACHE: Optional[Path] = Field(None, description="Default to CACHE_ROOT/runtime")
+    DATA_CACHE: Optional[Path] = Field(None, description="Default to CACHE_ROOT/data")
 
-    EXECUTION_ROOT = Path("/var/lib/aleph/vm")
-    JAILER_BASE_DIRECTORY: Path = Field(None, description="Default to EXECUTION_ROOT/jailer")
-    EXECUTION_DATABASE: Path = Field(
+    EXECUTION_ROOT: Path = Path("/var/lib/aleph/vm")
+    JAILER_BASE_DIRECTORY: Optional[Path] = Field(None, description="Default to EXECUTION_ROOT/jailer")
+    EXECUTION_DATABASE: Optional[Path] = Field(
         None, description="Location of database file. Default to EXECUTION_ROOT/executions.sqlite3"
     )
-    EXECUTION_LOG_ENABLED = False
-    EXECUTION_LOG_DIRECTORY: Path = Field(
+    EXECUTION_LOG_ENABLED: bool = False
+    EXECUTION_LOG_DIRECTORY: Optional[Path] = Field(
         None, description="Location of executions log. Default to EXECUTION_ROOT/executions/"
     )
 
-    PERSISTENT_VOLUMES_DIR: Path = Field(
+    PERSISTENT_VOLUMES_DIR: Optional[Path] = Field(
         None, description="Persistent volumes location. Default to EXECUTION_ROOT/volumes/persistent/"
     )
-    JAILER_BASE_DIR: Path = Field(None)
+    JAILER_BASE_DIR: Optional[Path] = Field(None)
 
-    MAX_PROGRAM_ARCHIVE_SIZE = 10_000_000  # 10 MB
-    MAX_DATA_ARCHIVE_SIZE = 10_000_000  # 10 MB
+    MAX_PROGRAM_ARCHIVE_SIZE: int = 10_000_000  # 10 MB
+    MAX_DATA_ARCHIVE_SIZE: int = 10_000_000  # 10 MB
 
     PAYMENT_MONITOR_INTERVAL: float = Field(
         default=60.0,
@@ -254,7 +256,7 @@
     )
 
     # hashlib.sha256(b"secret-token").hexdigest()
-    ALLOCATION_TOKEN_HASH = "151ba92f2eb90bce67e912af2f7a5c17d8654b3d29895b042107ea312a7eebda"
+    ALLOCATION_TOKEN_HASH: str = "151ba92f2eb90bce67e912af2f7a5c17d8654b3d29895b042107ea312a7eebda"
 
     ENABLE_QEMU_SUPPORT: bool = Field(default=True)
     INSTANCE_DEFAULT_HYPERVISOR: HypervisorType | None = Field(
@@ -268,21 +270,23 @@
         "with SEV and SEV-ES",
     )
 
-    CONFIDENTIAL_DIRECTORY: Path = Field(
+    CONFIDENTIAL_DIRECTORY: Optional[Path] = Field(
         None,
         description="Confidential Computing default directory. Default to EXECUTION_ROOT/confidential",
     )
 
-    CONFIDENTIAL_SESSION_DIRECTORY: Path = Field(None, description="Default to EXECUTION_ROOT/sessions")
+    CONFIDENTIAL_SESSION_DIRECTORY: Optional[Path] = Field(None, description="Default to EXECUTION_ROOT/sessions")
 
     # Tests on programs
 
     FAKE_DATA_PROGRAM: Path | None = None
-    BENCHMARK_FAKE_DATA_PROGRAM = Path(abspath(join(__file__, "../../../../examples/example_fastapi")))
+    BENCHMARK_FAKE_DATA_PROGRAM: Path = Path(abspath(join(__file__, "../../../../examples/example_fastapi")))
 
-    FAKE_DATA_MESSAGE = Path(abspath(join(__file__, "../../../../examples/program_message_from_aleph.json")))
+    FAKE_DATA_MESSAGE: Path = Path(abspath(join(__file__, "../../../../examples/program_message_from_aleph.json")))
     FAKE_DATA_DATA: Path | None = Path(abspath(join(__file__, "../../../../examples/data/")))
-    FAKE_DATA_RUNTIME = Path(abspath(join(__file__, "../../../../runtimes/aleph-debian-12-python/rootfs.squashfs")))
+    FAKE_DATA_RUNTIME: Path = Path(
+        abspath(join(__file__, "../../../../runtimes/aleph-debian-12-python/rootfs.squashfs"))
+    )
     FAKE_DATA_VOLUME: Path | None = Path(abspath(join(__file__, "../../../../examples/volumes/volume-venv.squashfs")))
 
     # Tests on instances
@@ -292,19 +296,21 @@
         description="Identifier of the instance message used when testing the launch of an instance from the network",
     )
 
-    USE_FAKE_INSTANCE_BASE = False
-    FAKE_INSTANCE_BASE = Path(abspath(join(__file__, "../../../../runtimes/instance-rootfs/debian-12.btrfs")))
-    FAKE_QEMU_INSTANCE_BASE = Path(abspath(join(__file__, "../../../../runtimes/instance-rootfs/rootfs.img")))
+    USE_FAKE_INSTANCE_BASE: bool = False
+    FAKE_INSTANCE_BASE: Path = Path(abspath(join(__file__, "../../../../runtimes/instance-rootfs/debian-12.btrfs")))
+    FAKE_QEMU_INSTANCE_BASE: Path = Path(abspath(join(__file__, "../../../../runtimes/instance-rootfs/rootfs.img")))
     FAKE_INSTANCE_ID: str = Field(
         default="decadecadecadecadecadecadecadecadecadecadecadecadecadecadecadeca",
         description="Identifier used for the 'fake instance' message defined in "
         "examples/instance_message_from_aleph.json",
     )
-    FAKE_INSTANCE_MESSAGE = Path(abspath(join(__file__, "../../../../examples/instance_message_from_aleph.json")))
-    FAKE_INSTANCE_QEMU_MESSAGE = Path(abspath(join(__file__, "../../../../examples/qemu_message_from_aleph.json")))
+    FAKE_INSTANCE_MESSAGE: Path = Path(abspath(join(__file__, "../../../../examples/instance_message_from_aleph.json")))
+    FAKE_INSTANCE_QEMU_MESSAGE: Path = Path(
+        abspath(join(__file__, "../../../../examples/qemu_message_from_aleph.json"))
+    )
 
-    CHECK_FASTAPI_VM_ID = "63faf8b5db1cf8d965e6a464a0cb8062af8e7df131729e48738342d956f29ace"
-    LEGACY_CHECK_FASTAPI_VM_ID = "67705389842a0a1b95eaa408b009741027964edc805997475e95c505d642edd8"
+    CHECK_FASTAPI_VM_ID: str = "63faf8b5db1cf8d965e6a464a0cb8062af8e7df131729e48738342d956f29ace"
+    LEGACY_CHECK_FASTAPI_VM_ID: str = "67705389842a0a1b95eaa408b009741027964edc805997475e95c505d642edd8"
 
     # Developer options
 
@@ -337,7 +343,7 @@
         assert isfile(self.JAILER_PATH), f"File not found {self.JAILER_PATH}"
         assert isfile(self.LINUX_PATH), f"File not found {self.LINUX_PATH}"
         assert self.NETWORK_INTERFACE, "Network interface is not specified"
-        assert self.CONNECTOR_URL.startswith("http://") or self.CONNECTOR_URL.startswith("https://")
+        assert str(self.CONNECTOR_URL).startswith("http://") or str(self.CONNECTOR_URL).startswith("https://")
         if self.ALLOW_VM_NETWORKING:
             assert exists(
                 f"/sys/class/net/{self.NETWORK_INTERFACE}"
@@ -399,10 +405,14 @@
         STREAM_CHAINS[Chain.AVAX].rpc = str(self.RPC_AVAX)
         STREAM_CHAINS[Chain.BASE].rpc = str(self.RPC_BASE)
 
-        os.makedirs(self.MESSAGE_CACHE, exist_ok=True)
-        os.makedirs(self.CODE_CACHE, exist_ok=True)
-        os.makedirs(self.RUNTIME_CACHE, exist_ok=True)
-        os.makedirs(self.DATA_CACHE, exist_ok=True)
+        if self.MESSAGE_CACHE:
+            os.makedirs(self.MESSAGE_CACHE, exist_ok=True)
+        if self.CODE_CACHE:
+            os.makedirs(self.CODE_CACHE, exist_ok=True)
+        if self.RUNTIME_CACHE:
+            os.makedirs(self.RUNTIME_CACHE, exist_ok=True)
+        if self.DATA_CACHE:
+            os.makedirs(self.DATA_CACHE, exist_ok=True)
 
         os.makedirs(self.EXECUTION_ROOT, exist_ok=True)
 
@@ -418,10 +428,14 @@
 
             self.LINUX_PATH = linux_path_on_device
 
-        os.makedirs(self.EXECUTION_LOG_DIRECTORY, exist_ok=True)
-        os.makedirs(self.PERSISTENT_VOLUMES_DIR, exist_ok=True)
-        os.makedirs(self.CONFIDENTIAL_DIRECTORY, exist_ok=True)
-        os.makedirs(self.CONFIDENTIAL_SESSION_DIRECTORY, exist_ok=True)
+        if self.EXECUTION_LOG_DIRECTORY:
+            os.makedirs(self.EXECUTION_LOG_DIRECTORY, exist_ok=True)
+        if self.PERSISTENT_VOLUMES_DIR:
+            os.makedirs(self.PERSISTENT_VOLUMES_DIR, exist_ok=True)
+        if self.CONFIDENTIAL_DIRECTORY:
+            os.makedirs(self.CONFIDENTIAL_DIRECTORY, exist_ok=True)
+        if self.CONFIDENTIAL_SESSION_DIRECTORY:
+            os.makedirs(self.CONFIDENTIAL_SESSION_DIRECTORY, exist_ok=True)
 
         self.API_SERVER = self.API_SERVER.rstrip("/")
 
@@ -462,18 +476,25 @@
                 attributes[attr] = "<REDACTED>"
             else:
                 attributes[attr] = getattr(self, attr)
-
-        return "\n".join(f"{self.Config.env_prefix}{attribute} = {value}" for attribute, value in attributes.items())
+        return "\n".join(
+            f"{self.model_config.get('env_prefix', '')}{attribute} = {value}" for attribute, value in attributes.items()
+        )
 
     def __init__(
         self,
-        _env_file: DotenvType | None = env_file_sentinel,
+        _env_file: str | Path | None = None,
         _env_file_encoding: str | None = None,
         _env_nested_delimiter: str | None = None,
-        _secrets_dir: StrPath | None = None,
+        _secrets_dir: Path | None = None,
         **values: Any,
     ) -> None:
-        super().__init__(_env_file, _env_file_encoding, _env_nested_delimiter, _secrets_dir, **values)
+        super().__init__(
+            _env_file,
+            _env_file_encoding,
+            _env_nested_delimiter,
+            _secrets_dir,
+            **values,
+        )
         if not self.MESSAGE_CACHE:
             self.MESSAGE_CACHE = self.CACHE_ROOT / "message"
         if not self.CODE_CACHE:
@@ -497,10 +518,9 @@
         if not self.CONFIDENTIAL_SESSION_DIRECTORY:
             self.CONFIDENTIAL_SESSION_DIRECTORY = self.EXECUTION_ROOT / "sessions"
 
-    class Config:
-        env_prefix = "ALEPH_VM_"
-        case_sensitive = False
-        env_file = ".env"
+    model_config = SettingsConfigDict(
+        env_prefix="ALEPH_VM_", case_sensitive=False, env_file=".env", validate_default=False
+    )
 
 
 def make_db_url():

src/aleph/vm/controllers/__main__.py

@@ -26,7 +26,7 @@
 def configuration_from_file(path: Path):
     with open(path) as f:
         data = json.load(f)
-        return Configuration.parse_obj(data)
+        return Configuration.model_validate(data)
 
 
 def parse_args(args):