Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

🐛 Fixed issue with serialisation of docker specs #6819

Merged
Merged
Show file tree
Hide file tree
Changes from 8 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from pydantic import ConfigDict, Field, field_validator
from pydantic import Field, field_validator

from .generated_models.docker_rest_api import (
ContainerSpec,
Expand All @@ -7,7 +7,6 @@
ServiceSpec,
TaskSpec,
)
from .utils.change_case import camel_to_snake


class AioDockerContainerSpec(ContainerSpec):
Expand Down Expand Up @@ -38,8 +37,6 @@ class AioDockerResources1(Resources1):
None, description="Define resources reservation.", alias="Reservations"
)

model_config = ConfigDict(populate_by_name=True)


class AioDockerTaskSpec(TaskSpec):
container_spec: AioDockerContainerSpec | None = Field(
Expand All @@ -51,5 +48,3 @@ class AioDockerTaskSpec(TaskSpec):

class AioDockerServiceSpec(ServiceSpec):
task_template: AioDockerTaskSpec | None = Field(default=None, alias="TaskTemplate")

model_config = ConfigDict(populate_by_name=True, alias_generator=camel_to_snake)
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,7 @@ def validate_volume_limits(cls, v, info: ValidationInfo) -> str | None:
"outputs_path": "/tmp/outputs", # noqa: S108 nosec
"inputs_path": "/tmp/inputs", # noqa: S108 nosec
"state_paths": ["/tmp/save_1", "/tmp_save_2"], # noqa: S108 nosec
"state_exclude": ["/tmp/strip_me/*", "*.py"], # noqa: S108 nosec
"state_exclude": ["/tmp/strip_me/*"], # noqa: S108 nosec
GitHK marked this conversation as resolved.
Show resolved Hide resolved
},
{
"outputs_path": "/t_out",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -291,7 +291,9 @@ def test_container_outgoing_permit_list_and_container_allow_internet_without_com
)
},
):
assert TypeAdapter(DynamicSidecarServiceLabels).validate_json(json.dumps(dict_data))
assert TypeAdapter(DynamicSidecarServiceLabels).validate_json(
json.dumps(dict_data)
)


def test_container_allow_internet_no_compose_spec_not_ok():
Expand Down Expand Up @@ -414,7 +416,7 @@ def service_labels() -> dict[str, str]:
"inputs_path": "/tmp/inputs", # noqa: S108
"outputs_path": "/tmp/outputs", # noqa: S108
"state_paths": ["/tmp/save_1", "/tmp_save_2"], # noqa: S108
"state_exclude": ["/tmp/strip_me/*", "*.py"], # noqa: S108
"state_exclude": ["/tmp/strip_me/*"], # noqa: S108
GitHK marked this conversation as resolved.
Show resolved Hide resolved
}
),
"simcore.service.compose-spec": json.dumps(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,15 +58,34 @@

_DYNAMIC_SIDECAR_SERVICE_EXTENDABLE_SPECS: Final[tuple[list[str], ...]] = (
["labels"],
["task_template", "Resources", "Limits"],
["task_template", "Resources", "Reservation", "MemoryBytes"],
["task_template", "Resources", "Reservation", "NanoCPUs"],
["task_template", "Placement", "Constraints"],
["task_template", "ContainerSpec", "Env"],
["task_template", "Resources", "Reservation", "GenericResources"],
["task_template", "container_spec", "env"],
["task_template", "placement", "constraints"],
["task_template", "resources", "reservation", "generic_resources"],
["task_template", "resources", "limits"],
["task_template", "resources", "reservation", "memory_bytes"],
["task_template", "resources", "reservation", "nano_cp_us"],
)


def _get_dynamic_sidecar_service_final_spec(
GitHK marked this conversation as resolved.
Show resolved Hide resolved
dynamic_sidecar_service_spec_base: AioDockerServiceSpec,
user_specific_service_spec: AioDockerServiceSpec,
) -> AioDockerServiceSpec:
# NOTE: since user_specific_service_spec follows Docker Service Spec and not Aio
# we do not use aliases when exporting dynamic_sidecar_service_spec_base
return AioDockerServiceSpec.model_validate(
nested_update(
jsonable_encoder(
dynamic_sidecar_service_spec_base, exclude_unset=True, by_alias=False
),
jsonable_encoder(
user_specific_service_spec, exclude_unset=True, by_alias=False
),
include=_DYNAMIC_SIDECAR_SERVICE_EXTENDABLE_SPECS,
)
)


async def _create_proxy_service(
app,
*,
Expand Down Expand Up @@ -245,14 +264,8 @@ async def action(cls, app: FastAPI, scheduler_data: SchedulerData) -> None:
user_specific_service_spec = AioDockerServiceSpec.model_validate(
user_specific_service_spec
)
# NOTE: since user_specific_service_spec follows Docker Service Spec and not Aio
# we do not use aliases when exporting dynamic_sidecar_service_spec_base
dynamic_sidecar_service_final_spec = AioDockerServiceSpec.model_validate(
nested_update(
jsonable_encoder(dynamic_sidecar_service_spec_base, exclude_unset=True),
jsonable_encoder(user_specific_service_spec, exclude_unset=True),
include=_DYNAMIC_SIDECAR_SERVICE_EXTENDABLE_SPECS,
)
dynamic_sidecar_service_final_spec = _get_dynamic_sidecar_service_final_spec(
dynamic_sidecar_service_spec_base, user_specific_service_spec
)
rabbit_message = ProgressRabbitMessageNode.model_construct(
user_id=scheduler_data.user_id,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,10 @@
from simcore_service_director_v2.modules.dynamic_sidecar.docker_service_specs import (
get_dynamic_sidecar_spec,
)
from simcore_service_director_v2.modules.dynamic_sidecar.scheduler._core._event_create_sidecars import (
_DYNAMIC_SIDECAR_SERVICE_EXTENDABLE_SPECS,
_get_dynamic_sidecar_service_final_spec,
)
from simcore_service_director_v2.utils.dict_utils import nested_update


Expand Down Expand Up @@ -180,7 +184,7 @@ def expected_dynamic_sidecar_spec(
"paths_mapping": {
"inputs_path": "/tmp/inputs", # noqa: S108
"outputs_path": "/tmp/outputs", # noqa: S108
"state_exclude": ["/tmp/strip_me/*", "*.py"], # noqa: S108
GitHK marked this conversation as resolved.
Show resolved Hide resolved
"state_exclude": ["/tmp/strip_me/*"], # noqa: S108
"state_paths": ["/tmp/save_1", "/tmp_save_2"], # noqa: S108
},
"callbacks_mapping": CallbacksMapping.model_config[
Expand Down Expand Up @@ -239,7 +243,7 @@ def expected_dynamic_sidecar_spec(
"DY_SIDECAR_PATH_OUTPUTS": "/tmp/outputs", # noqa: S108
"DY_SIDECAR_PROJECT_ID": "dd1d04d9-d704-4f7e-8f0f-1ca60cc771fe",
"DY_SIDECAR_STATE_EXCLUDE": json_dumps(
["*.py", "/tmp/strip_me/*"] # noqa: S108
["/tmp/strip_me/*"] # noqa: S108
),
"DY_SIDECAR_STATE_PATHS": json_dumps(
["/tmp/save_1", "/tmp_save_2"] # noqa: S108
Expand Down Expand Up @@ -614,14 +618,66 @@ async def test_merge_dynamic_sidecar_specs_with_user_specific_specs(
another_merged_dict = nested_update(
orig_dict,
user_dict,
include=(
["labels"],
["task_template", "Resources", "Limits"],
["task_template", "Resources", "Reservation", "MemoryBytes"],
["task_template", "Resources", "Reservation", "NanoCPUs"],
["task_template", "Placement", "Constraints"],
["task_template", "ContainerSpec", "Env"],
["task_template", "Resources", "Reservation", "GenericResources"],
),
include=_DYNAMIC_SIDECAR_SERVICE_EXTENDABLE_SPECS,
)
assert another_merged_dict


def test_regression__get_dynamic_sidecar_service_final_spec():
mock_service_spec = AioDockerServiceSpec.model_validate(
{"Labels": {"l1": "false", "l0": "a"}}
)
mock_catalog_constraints = AioDockerServiceSpec.model_validate(
{
"Labels": {"l1": "true", "l2": "a"},
"TaskTemplate": {
"Placement": {
"Constraints": [
"c1==true",
"c2==true",
],
},
"Resources": {
"Limits": {"MemoryBytes": 1, "NanoCPUs": 1},
"Reservations": {
"GenericResources": [
{"DiscreteResourceSpec": {"Kind": "VRAM", "Value": 1}}
],
"MemoryBytes": 2,
"NanoCPUs": 2,
},
},
"ContainerSpec": {
"Env": [
"key-1=value-1",
"key2-value2=a",
]
},
},
}
)
result = _get_dynamic_sidecar_service_final_spec(
mock_service_spec, mock_catalog_constraints
)
assert result.model_dump(by_alias=True, exclude_unset=True) == {
"Labels": {"l1": "true", "l2": "a", "l0": "a"},
"TaskTemplate": {
"Placement": {
"Constraints": [
"c1==true",
"c2==true",
],
},
"Resources": {
"Limits": {"MemoryBytes": 1, "NanoCPUs": 1},
"Reservations": {
"GenericResources": [
{"DiscreteResourceSpec": {"Kind": "VRAM", "Value": 1}}
],
"MemoryBytes": 2,
"NanoCPUs": 2,
},
},
"ContainerSpec": {"Env": {"key-1": "value-1", "key2-value2": "a"}},
},
}
Loading