From 65b370c421155534d23e57ced148d1acf644b223 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Thu, 11 Apr 2024 09:47:47 -0400 Subject: [PATCH 1/4] enable img build spec for run sh --- .../startosis_engine/kurtosis_builtins.go | 2 +- .../add_service/add_service.go | 3 +- .../kurtosis_instruction/tasks/run_python.go | 10 ++- .../kurtosis_instruction/tasks/run_sh.go | 89 +++++++++++++------ .../tasks/tasks_shared.go | 29 ++++-- .../service_config/service_config.go | 4 +- 6 files changed, 95 insertions(+), 42 deletions(-) diff --git a/core/server/api_container/server/startosis_engine/kurtosis_builtins.go b/core/server/api_container/server/startosis_engine/kurtosis_builtins.go index 4224f5e5bc..e9ab0075f3 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_builtins.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_builtins.go @@ -80,7 +80,7 @@ func KurtosisPlanInstructions( request.NewRequest(serviceNetwork, runtimeValueStore), start_service.NewStartService(serviceNetwork), tasks.NewRunPythonService(serviceNetwork, runtimeValueStore, nonBlockingMode), - tasks.NewRunShService(serviceNetwork, runtimeValueStore, nonBlockingMode), + tasks.NewRunShService(serviceNetwork, runtimeValueStore, nonBlockingMode, packageId, packageContentProvider, packageReplaceOptions), stop_service.NewStopService(serviceNetwork), store_service_files.NewStoreServiceFiles(serviceNetwork), upload_files.NewUploadFiles(packageId, serviceNetwork, packageContentProvider, packageReplaceOptions), diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service.go index 7db6f39962..ec34e25a92 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service.go @@ -326,7 +326,8 @@ func validateAndConvertConfigAndReadyCondition( locatorOfModuleInWhichThisBuiltInIsBeingCalled, packageId, packageContentProvider, - packageReplaceOptions, imageDownloadMode) + packageReplaceOptions, + imageDownloadMode) if interpretationErr != nil { return nil, nil, interpretationErr } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go index 2971f953fa..03d2494243 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go @@ -4,6 +4,9 @@ import ( "context" "fmt" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/exec_result" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_build_spec" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_registry_spec" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/nix_build_spec" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service_directory" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/store_spec" @@ -219,7 +222,10 @@ func (builtin *RunPythonCapabilities) Interpret(_ string, arguments *builtin_arg } // build a service config from image and files artifacts expansion. - builtin.serviceConfig, err = getServiceConfig(image, filesArtifactExpansion, envVars) + var emptyImageBuildSpec *image_build_spec.ImageBuildSpec + var emptyImageRegistrySpec *image_registry_spec.ImageRegistrySpec + var emptyNixBuildSpec *nix_build_spec.NixBuildSpec + builtin.serviceConfig, err = getServiceConfig(image, emptyImageBuildSpec, emptyImageRegistrySpec, emptyNixBuildSpec, filesArtifactExpansion, envVars) if err != nil { return nil, startosis_errors.WrapWithInterpretationError(err, "An error occurred creating service config using image '%s'", image) } @@ -258,7 +264,7 @@ func (builtin *RunPythonCapabilities) Validate(_ *builtin_argument.ArgumentValue if builtin.serviceConfig.GetFilesArtifactsExpansion() != nil { serviceDirpathsToArtifactIdentifiers = builtin.serviceConfig.GetFilesArtifactsExpansion().ServiceDirpathsToArtifactIdentifiers } - return validateTasksCommon(validatorEnvironment, builtin.storeSpecList, serviceDirpathsToArtifactIdentifiers, builtin.serviceConfig.GetContainerImageName()) + return validateTasksCommon(validatorEnvironment, builtin.storeSpecList, serviceDirpathsToArtifactIdentifiers, builtin.serviceConfig) } // Execute This is just v0 for run_python task - we can later improve on it. diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go index 451b12cf1a..f8bda97d86 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go @@ -3,6 +3,9 @@ package tasks import ( "context" "fmt" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_build_spec" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_registry_spec" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/nix_build_spec" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service_directory" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/store_spec" @@ -18,6 +21,7 @@ import ( "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/plan_yaml" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/runtime_value_store" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_errors" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_packages" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_validator" "github.com/kurtosis-tech/stacktrace" "github.com/xtgo/uuid" @@ -33,7 +37,13 @@ const ( runningShScriptPrefix = "Running sh script" ) -func NewRunShService(serviceNetwork service_network.ServiceNetwork, runtimeValueStore *runtime_value_store.RuntimeValueStore, nonBlockingMode bool) *kurtosis_plan_instruction.KurtosisPlanInstruction { +func NewRunShService( + serviceNetwork service_network.ServiceNetwork, + runtimeValueStore *runtime_value_store.RuntimeValueStore, + nonBlockingMode bool, + packageId string, + packageContentProvider startosis_packages.PackageContentProvider, + packageReplaceOptions map[string]string) *kurtosis_plan_instruction.KurtosisPlanInstruction { return &kurtosis_plan_instruction.KurtosisPlanInstruction{ KurtosisBaseBuiltin: &kurtosis_starlark_framework.KurtosisBaseBuiltin{ Name: RunShBuiltinName, @@ -44,14 +54,14 @@ func NewRunShService(serviceNetwork service_network.ServiceNetwork, runtimeValue IsOptional: false, ZeroValueProvider: builtin_argument.ZeroValueProvider[starlark.String], }, - { - Name: ImageNameArgName, - IsOptional: true, - ZeroValueProvider: builtin_argument.ZeroValueProvider[starlark.String], - Validator: func(argumentValue starlark.Value) *startosis_errors.InterpretationError { - return builtin_argument.NonEmptyString(argumentValue, ImageNameArgName) - }, - }, + //{ + // Name: ImageNameArgName, + // IsOptional: true, + // ZeroValueProvider: builtin_argument.ZeroValueProvider[starlark.String], + // Validator: func(argumentValue starlark.Value) *startosis_errors.InterpretationError { + // return builtin_argument.NonEmptyString(argumentValue, ImageNameArgName) + // }, + //}, { Name: FilesArgName, IsOptional: true, @@ -83,17 +93,20 @@ func NewRunShService(serviceNetwork service_network.ServiceNetwork, runtimeValue Capabilities: func() kurtosis_plan_instruction.KurtosisPlanInstructionCapabilities { return &RunShCapabilities{ - serviceNetwork: serviceNetwork, - runtimeValueStore: runtimeValueStore, - name: "", - nonBlockingMode: nonBlockingMode, - serviceConfig: nil, // populated at interpretation time - run: "", // populated at interpretation time - resultUuid: "", // populated at interpretation time - storeSpecList: nil, - wait: DefaultWaitTimeoutDurationStr, - description: "", // populated at interpretation time - returnValue: nil, // populated at interpretation time + serviceNetwork: serviceNetwork, + runtimeValueStore: runtimeValueStore, + packageId: packageId, + packageReplaceOptions: packageReplaceOptions, + packageContentProvider: packageContentProvider, + name: "", + nonBlockingMode: nonBlockingMode, + serviceConfig: nil, // populated at interpretation time + run: "", // populated at interpretation time + resultUuid: "", // populated at interpretation time + storeSpecList: nil, + wait: DefaultWaitTimeoutDurationStr, + description: "", // populated at interpretation time + returnValue: nil, // populated at interpretation time } }, @@ -117,6 +130,11 @@ type RunShCapabilities struct { run string nonBlockingMode bool + // fields required for image building + packageId string + packageContentProvider startosis_packages.PackageContentProvider + packageReplaceOptions map[string]string + serviceConfig *service.ServiceConfig storeSpecList []*store_spec.StoreSpec returnValue *starlarkstruct.Struct @@ -124,22 +142,35 @@ type RunShCapabilities struct { description string } -func (builtin *RunShCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) { +func (builtin *RunShCapabilities) Interpret(locatorOfModuleInWhichThisBuiltinIsBeingCalled string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) { runCommand, err := builtin_argument.ExtractArgumentValue[starlark.String](arguments, RunArgName) if err != nil { return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", RunArgName) } builtin.run = runCommand.GoString() - var image string + var maybeImageName string + var maybeImageBuildSpec *image_build_spec.ImageBuildSpec + var maybeImageRegistrySpec *image_registry_spec.ImageRegistrySpec + var maybeNixBuildSpec *nix_build_spec.NixBuildSpec + var interpretationErr *startosis_errors.InterpretationError if arguments.IsSet(ImageNameArgName) { - imageStarlark, err := builtin_argument.ExtractArgumentValue[starlark.String](arguments, ImageNameArgName) + rawImageVal, err := builtin_argument.ExtractArgumentValue[starlark.Value](arguments, ImageNameArgName) if err != nil { - return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", ImageNameArgName) + return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract raw image attribute.") + } + // TODO: refactor image build spec into a common interface + maybeImageName, maybeImageBuildSpec, maybeImageRegistrySpec, maybeNixBuildSpec, interpretationErr = service_config.ConvertImage( + rawImageVal, + locatorOfModuleInWhichThisBuiltinIsBeingCalled, + builtin.packageId, + builtin.packageContentProvider, + builtin.packageReplaceOptions) + if interpretationErr != nil { + return nil, startosis_errors.WrapWithInterpretationError(interpretationErr, "An error occurred converting image for run sh.") } - image = imageStarlark.GoString() } else { - image = defaultRunShImageName + maybeImageName = defaultRunShImageName } var filesArtifactExpansion *service_directory.FilesArtifactsExpansion @@ -170,9 +201,9 @@ func (builtin *RunShCapabilities) Interpret(_ string, arguments *builtin_argumen } // build a service config from image and files artifacts expansion. - builtin.serviceConfig, err = getServiceConfig(image, filesArtifactExpansion, envVars) + builtin.serviceConfig, err = getServiceConfig(maybeImageName, maybeImageBuildSpec, maybeImageRegistrySpec, maybeNixBuildSpec, filesArtifactExpansion, envVars) if err != nil { - return nil, startosis_errors.WrapWithInterpretationError(err, "An error occurred creating service config using image '%s'", image) + return nil, startosis_errors.WrapWithInterpretationError(err, "An error occurred creating service config using for run sh task.") } if arguments.IsSet(StoreFilesArgName) { @@ -215,7 +246,7 @@ func (builtin *RunShCapabilities) Validate(_ *builtin_argument.ArgumentValuesSet if builtin.serviceConfig.GetFilesArtifactsExpansion() != nil { serviceDirpathsToArtifactIdentifiers = builtin.serviceConfig.GetFilesArtifactsExpansion().ServiceDirpathsToArtifactIdentifiers } - return validateTasksCommon(validatorEnvironment, builtin.storeSpecList, serviceDirpathsToArtifactIdentifiers, builtin.serviceConfig.GetContainerImageName()) + return validateTasksCommon(validatorEnvironment, builtin.storeSpecList, serviceDirpathsToArtifactIdentifiers, builtin.serviceConfig) } // Execute This is just v0 for run_sh task - we can later improve on it. diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/tasks_shared.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/tasks_shared.go index d5d728b7fd..7dc3822716 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/tasks_shared.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/tasks_shared.go @@ -3,7 +3,10 @@ package tasks import ( "context" "fmt" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_build_spec" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_download_mode" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_registry_spec" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/nix_build_spec" "reflect" "strings" "time" @@ -137,7 +140,7 @@ func createInterpretationResult(resultUuid string, storeSpecList []*store_spec.S return result } -func validateTasksCommon(validatorEnvironment *startosis_validator.ValidatorEnvironment, storeSpecList []*store_spec.StoreSpec, serviceDirpathsToArtifactIdentifiers map[string][]string, imageName string) *startosis_errors.ValidationError { +func validateTasksCommon(validatorEnvironment *startosis_validator.ValidatorEnvironment, storeSpecList []*store_spec.StoreSpec, serviceDirpathsToArtifactIdentifiers map[string][]string, serviceConfig *service.ServiceConfig) *startosis_errors.ValidationError { if storeSpecList != nil { if err := validatePathIsUniqueWhileCreatingFileArtifact(storeSpecList); err != nil { return startosis_errors.WrapWithValidationError(err, "error occurred while validating file paths to copy into file artifact") @@ -156,7 +159,16 @@ func validateTasksCommon(validatorEnvironment *startosis_validator.ValidatorEnvi } } - validatorEnvironment.AppendRequiredImagePull(imageName) + // add the images to be built here + if serviceConfig.GetImageBuildSpec() != nil { + validatorEnvironment.AppendRequiredImageBuild(serviceConfig.GetContainerImageName(), serviceConfig.GetImageBuildSpec()) + } else if serviceConfig.GetImageRegistrySpec() != nil { + validatorEnvironment.AppendImageToPullWithAuth(serviceConfig.GetContainerImageName(), serviceConfig.GetImageRegistrySpec()) + } else if serviceConfig.GetNixBuildSpec() != nil { + validatorEnvironment.AppendRequiredNixBuild(serviceConfig.GetContainerImageName(), serviceConfig.GetNixBuildSpec()) + } else { + validatorEnvironment.AppendRequiredImagePull(serviceConfig.GetContainerImageName()) + } return nil } @@ -252,15 +264,18 @@ func resultMapToString(resultMap map[string]starlark.Comparable, builtinNameForL } func getServiceConfig( - image string, + maybeImageName string, + maybeImageBuildSpec *image_build_spec.ImageBuildSpec, + maybeImageRegistrySpec *image_registry_spec.ImageRegistrySpec, + maybeNixBuildSpec *nix_build_spec.NixBuildSpec, filesArtifactExpansion *service_directory.FilesArtifactsExpansion, envVars *map[string]string, ) (*service.ServiceConfig, error) { serviceConfig, err := service.CreateServiceConfig( - image, - nil, - nil, - nil, + maybeImageName, + maybeImageBuildSpec, + maybeImageRegistrySpec, + maybeNixBuildSpec, nil, nil, // This make sure that the container does not stop as soon as it starts diff --git a/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/service_config.go b/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/service_config.go index 54145d6145..492bf248f7 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/service_config.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/service_config.go @@ -275,7 +275,7 @@ func (config *ServiceConfig) ToKurtosisType( return nil, startosis_errors.NewInterpretationError("Required attribute '%s' could not be found on type '%s'", ImageAttr, ServiceConfigTypeName) } // TODO: refactor image build spec into a common interface - imageName, maybeImageBuildSpec, maybeImageRegistrySpec, maybeNixBuildSpec, interpretationErr = convertImage( + imageName, maybeImageBuildSpec, maybeImageRegistrySpec, maybeNixBuildSpec, interpretationErr = ConvertImage( rawImageAttrValue, locatorOfModuleInWhichThisBuiltInIsBeingCalled, packageId, @@ -698,7 +698,7 @@ func convertFilesArguments(attrNameForLogging string, filesDict *starlark.Dict) // If [image] is an ImageBuildSpec type, returns name for the image to build and ImageBuildSpec converted to KurtosisType // If [image] is a string, returns the image name with no image build spec (image will be fetched from local cache or remote) // If [image] is an ImageSpec type, returns the name for the image and the ImageSpec converted to KurtosisType -func convertImage( +func ConvertImage( image starlark.Value, locatorOfModuleInWhichThisBuiltInIsBeingCalled string, packageId string, From c2354cd8022eeb50046df1d5ffd98a097be7a4c5 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Thu, 11 Apr 2024 10:07:34 -0400 Subject: [PATCH 2/4] add back image name arg --- .../kurtosis_instruction/tasks/run_sh.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go index f8bda97d86..a81874e436 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go @@ -54,14 +54,12 @@ func NewRunShService( IsOptional: false, ZeroValueProvider: builtin_argument.ZeroValueProvider[starlark.String], }, - //{ - // Name: ImageNameArgName, - // IsOptional: true, - // ZeroValueProvider: builtin_argument.ZeroValueProvider[starlark.String], - // Validator: func(argumentValue starlark.Value) *startosis_errors.InterpretationError { - // return builtin_argument.NonEmptyString(argumentValue, ImageNameArgName) - // }, - //}, + { + Name: ImageNameArgName, + IsOptional: true, + ZeroValueProvider: builtin_argument.ZeroValueProvider[starlark.Value], + Validator: nil, + }, { Name: FilesArgName, IsOptional: true, From b7ed1fe6e5f657c85b421304fd9c7de4b0cc0ed4 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Wed, 17 Apr 2024 09:55:45 -0400 Subject: [PATCH 3/4] enable for run python --- .../startosis_engine/kurtosis_builtins.go | 2 +- .../kurtosis_instruction/tasks/run_python.go | 75 ++++++++++++------- .../kurtosis_instruction/tasks/run_sh.go | 2 +- 3 files changed, 52 insertions(+), 27 deletions(-) diff --git a/core/server/api_container/server/startosis_engine/kurtosis_builtins.go b/core/server/api_container/server/startosis_engine/kurtosis_builtins.go index e9ab0075f3..c25ba58349 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_builtins.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_builtins.go @@ -79,7 +79,7 @@ func KurtosisPlanInstructions( render_templates.NewRenderTemplatesInstruction(serviceNetwork, runtimeValueStore), request.NewRequest(serviceNetwork, runtimeValueStore), start_service.NewStartService(serviceNetwork), - tasks.NewRunPythonService(serviceNetwork, runtimeValueStore, nonBlockingMode), + tasks.NewRunPythonService(serviceNetwork, runtimeValueStore, nonBlockingMode, packageId, packageContentProvider, packageReplaceOptions), tasks.NewRunShService(serviceNetwork, runtimeValueStore, nonBlockingMode, packageId, packageContentProvider, packageReplaceOptions), stop_service.NewStopService(serviceNetwork), store_service_files.NewStoreServiceFiles(serviceNetwork), diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go index 03d2494243..117b0fcc3b 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go @@ -22,6 +22,7 @@ import ( "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/plan_yaml" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/runtime_value_store" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_errors" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_packages" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_validator" "github.com/kurtosis-tech/stacktrace" "github.com/xtgo/uuid" @@ -47,7 +48,13 @@ const ( runPythonDefaultDescription = "Running Python script" ) -func NewRunPythonService(serviceNetwork service_network.ServiceNetwork, runtimeValueStore *runtime_value_store.RuntimeValueStore, nonBlockingMode bool) *kurtosis_plan_instruction.KurtosisPlanInstruction { +func NewRunPythonService( + serviceNetwork service_network.ServiceNetwork, + runtimeValueStore *runtime_value_store.RuntimeValueStore, + nonBlockingMode bool, + packageId string, + packageContentProvider startosis_packages.PackageContentProvider, + packageReplaceOptions map[string]string) *kurtosis_plan_instruction.KurtosisPlanInstruction { return &kurtosis_plan_instruction.KurtosisPlanInstruction{ KurtosisBaseBuiltin: &kurtosis_starlark_framework.KurtosisBaseBuiltin{ Name: RunPythonBuiltinName, @@ -102,19 +109,22 @@ func NewRunPythonService(serviceNetwork service_network.ServiceNetwork, runtimeV Capabilities: func() kurtosis_plan_instruction.KurtosisPlanInstructionCapabilities { return &RunPythonCapabilities{ - serviceNetwork: serviceNetwork, - runtimeValueStore: runtimeValueStore, - pythonArguments: nil, - packages: nil, - name: "", - nonBlockingMode: nonBlockingMode, - serviceConfig: nil, // populated at interpretation time - run: "", // populated at interpretation time - resultUuid: "", // populated at interpretation time - storeSpecList: nil, - wait: DefaultWaitTimeoutDurationStr, - description: "", // populated at interpretation time - returnValue: nil, // populated at interpretation time + serviceNetwork: serviceNetwork, + runtimeValueStore: runtimeValueStore, + pythonArguments: nil, + packages: nil, + name: "", + nonBlockingMode: nonBlockingMode, + packageId: packageId, + packageContentProvider: packageContentProvider, + packageReplaceOptions: packageReplaceOptions, + serviceConfig: nil, // populated at interpretation time + run: "", // populated at interpretation time + resultUuid: "", // populated at interpretation time + storeSpecList: nil, + wait: DefaultWaitTimeoutDurationStr, + description: "", // populated at interpretation time + returnValue: nil, // populated at interpretation time } }, @@ -142,6 +152,11 @@ type RunPythonCapabilities struct { pythonArguments []string packages []string + // fields required for image building + packageId string + packageContentProvider startosis_packages.PackageContentProvider + packageReplaceOptions map[string]string + returnValue *starlarkstruct.Struct serviceConfig *service.ServiceConfig storeSpecList []*store_spec.StoreSpec @@ -149,7 +164,7 @@ type RunPythonCapabilities struct { description string } -func (builtin *RunPythonCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) { +func (builtin *RunPythonCapabilities) Interpret(locatorOfModuleInWhichThisBuiltinIsBeingCalled string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) { randomUuid := uuid.NewRandom() builtin.name = fmt.Sprintf("task-%v", randomUuid.String()) @@ -183,15 +198,28 @@ func (builtin *RunPythonCapabilities) Interpret(_ string, arguments *builtin_arg builtin.packages = packagesList } - var image string + var maybeImageName string + var maybeImageBuildSpec *image_build_spec.ImageBuildSpec + var maybeImageRegistrySpec *image_registry_spec.ImageRegistrySpec + var maybeNixBuildSpec *nix_build_spec.NixBuildSpec + var interpretationErr *startosis_errors.InterpretationError if arguments.IsSet(ImageNameArgName) { - imageStarlark, err := builtin_argument.ExtractArgumentValue[starlark.String](arguments, ImageNameArgName) + rawImageVal, err := builtin_argument.ExtractArgumentValue[starlark.Value](arguments, ImageNameArgName) if err != nil { - return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", ImageNameArgName) + return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract raw image attribute.") + } + + maybeImageName, maybeImageBuildSpec, maybeImageRegistrySpec, maybeNixBuildSpec, interpretationErr = service_config.ConvertImage( + rawImageVal, + locatorOfModuleInWhichThisBuiltinIsBeingCalled, + builtin.packageId, + builtin.packageContentProvider, + builtin.packageReplaceOptions) + if interpretationErr != nil { + return nil, startosis_errors.WrapWithInterpretationError(interpretationErr, "An error occurred converting image for run sh.") } - image = imageStarlark.GoString() } else { - image = defaultRunPythonImageName + maybeImageName = defaultRunPythonImageName } var filesArtifactExpansion *service_directory.FilesArtifactsExpansion @@ -222,12 +250,9 @@ func (builtin *RunPythonCapabilities) Interpret(_ string, arguments *builtin_arg } // build a service config from image and files artifacts expansion. - var emptyImageBuildSpec *image_build_spec.ImageBuildSpec - var emptyImageRegistrySpec *image_registry_spec.ImageRegistrySpec - var emptyNixBuildSpec *nix_build_spec.NixBuildSpec - builtin.serviceConfig, err = getServiceConfig(image, emptyImageBuildSpec, emptyImageRegistrySpec, emptyNixBuildSpec, filesArtifactExpansion, envVars) + builtin.serviceConfig, err = getServiceConfig(maybeImageName, maybeImageBuildSpec, maybeImageRegistrySpec, maybeNixBuildSpec, filesArtifactExpansion, envVars) if err != nil { - return nil, startosis_errors.WrapWithInterpretationError(err, "An error occurred creating service config using image '%s'", image) + return nil, startosis_errors.WrapWithInterpretationError(err, "An error occurred creating service config for run python.") } if arguments.IsSet(StoreFilesArgName) { diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go index a81874e436..fdb7bf47b9 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go @@ -157,7 +157,7 @@ func (builtin *RunShCapabilities) Interpret(locatorOfModuleInWhichThisBuiltinIsB if err != nil { return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract raw image attribute.") } - // TODO: refactor image build spec into a common interface + maybeImageName, maybeImageBuildSpec, maybeImageRegistrySpec, maybeNixBuildSpec, interpretationErr = service_config.ConvertImage( rawImageVal, locatorOfModuleInWhichThisBuiltinIsBeingCalled, From 1d6c68a93a73b3eb8979646be45a89c3e747baca Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Wed, 17 Apr 2024 10:19:36 -0400 Subject: [PATCH 4/4] turn off image validator --- .../kurtosis_instruction/tasks/run_python.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go index 117b0fcc3b..82ca291240 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go @@ -79,10 +79,8 @@ func NewRunPythonService( { Name: ImageNameArgName, IsOptional: true, - ZeroValueProvider: builtin_argument.ZeroValueProvider[starlark.String], - Validator: func(argumentValue starlark.Value) *startosis_errors.InterpretationError { - return builtin_argument.NonEmptyString(argumentValue, ImageNameArgName) - }, + ZeroValueProvider: builtin_argument.ZeroValueProvider[starlark.Value], + Validator: nil, }, { Name: FilesArgName,