diff --git a/core/server/api_container/server/startosis_engine/kurtosis_builtins.go b/core/server/api_container/server/startosis_engine/kurtosis_builtins.go index 9a318b22c6..e86ee10df3 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_builtins.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_builtins.go @@ -81,8 +81,8 @@ func KurtosisPlanInstructions( render_templates.NewRenderTemplatesInstruction(serviceNetwork, runtimeValueStore), request.NewRequest(serviceNetwork, runtimeValueStore), start_service.NewStartService(serviceNetwork), - tasks.NewRunPythonService(serviceNetwork, runtimeValueStore, nonBlockingMode), - tasks.NewRunShService(serviceNetwork, runtimeValueStore, nonBlockingMode), + tasks.NewRunPythonService(serviceNetwork, runtimeValueStore, nonBlockingMode, packageId, packageContentProvider, packageReplaceOptions), + tasks.NewRunShService(serviceNetwork, runtimeValueStore, nonBlockingMode, packageId, packageContentProvider, packageReplaceOptions), stop_service.NewStopService(serviceNetwork), store_service_files.NewStoreServiceFiles(serviceNetwork), upload_files.NewUploadFiles(packageId, serviceNetwork, packageContentProvider, packageReplaceOptions), diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service.go index 7db6f39962..ec34e25a92 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service.go @@ -326,7 +326,8 @@ func validateAndConvertConfigAndReadyCondition( locatorOfModuleInWhichThisBuiltInIsBeingCalled, packageId, packageContentProvider, - packageReplaceOptions, imageDownloadMode) + packageReplaceOptions, + imageDownloadMode) if interpretationErr != nil { return nil, nil, interpretationErr } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go index 2971f953fa..82ca291240 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go @@ -4,6 +4,9 @@ import ( "context" "fmt" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/exec_result" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_build_spec" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_registry_spec" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/nix_build_spec" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service_directory" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/store_spec" @@ -19,6 +22,7 @@ import ( "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/plan_yaml" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/runtime_value_store" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_errors" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_packages" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_validator" "github.com/kurtosis-tech/stacktrace" "github.com/xtgo/uuid" @@ -44,7 +48,13 @@ const ( runPythonDefaultDescription = "Running Python script" ) -func NewRunPythonService(serviceNetwork service_network.ServiceNetwork, runtimeValueStore *runtime_value_store.RuntimeValueStore, nonBlockingMode bool) *kurtosis_plan_instruction.KurtosisPlanInstruction { +func NewRunPythonService( + serviceNetwork service_network.ServiceNetwork, + runtimeValueStore *runtime_value_store.RuntimeValueStore, + nonBlockingMode bool, + packageId string, + packageContentProvider startosis_packages.PackageContentProvider, + packageReplaceOptions map[string]string) *kurtosis_plan_instruction.KurtosisPlanInstruction { return &kurtosis_plan_instruction.KurtosisPlanInstruction{ KurtosisBaseBuiltin: &kurtosis_starlark_framework.KurtosisBaseBuiltin{ Name: RunPythonBuiltinName, @@ -69,10 +79,8 @@ func NewRunPythonService(serviceNetwork service_network.ServiceNetwork, runtimeV { Name: ImageNameArgName, IsOptional: true, - ZeroValueProvider: builtin_argument.ZeroValueProvider[starlark.String], - Validator: func(argumentValue starlark.Value) *startosis_errors.InterpretationError { - return builtin_argument.NonEmptyString(argumentValue, ImageNameArgName) - }, + ZeroValueProvider: builtin_argument.ZeroValueProvider[starlark.Value], + Validator: nil, }, { Name: FilesArgName, @@ -99,19 +107,22 @@ func NewRunPythonService(serviceNetwork service_network.ServiceNetwork, runtimeV Capabilities: func() kurtosis_plan_instruction.KurtosisPlanInstructionCapabilities { return &RunPythonCapabilities{ - serviceNetwork: serviceNetwork, - runtimeValueStore: runtimeValueStore, - pythonArguments: nil, - packages: nil, - name: "", - nonBlockingMode: nonBlockingMode, - serviceConfig: nil, // populated at interpretation time - run: "", // populated at interpretation time - resultUuid: "", // populated at interpretation time - storeSpecList: nil, - wait: DefaultWaitTimeoutDurationStr, - description: "", // populated at interpretation time - returnValue: nil, // populated at interpretation time + serviceNetwork: serviceNetwork, + runtimeValueStore: runtimeValueStore, + pythonArguments: nil, + packages: nil, + name: "", + nonBlockingMode: nonBlockingMode, + packageId: packageId, + packageContentProvider: packageContentProvider, + packageReplaceOptions: packageReplaceOptions, + serviceConfig: nil, // populated at interpretation time + run: "", // populated at interpretation time + resultUuid: "", // populated at interpretation time + storeSpecList: nil, + wait: DefaultWaitTimeoutDurationStr, + description: "", // populated at interpretation time + returnValue: nil, // populated at interpretation time } }, @@ -139,6 +150,11 @@ type RunPythonCapabilities struct { pythonArguments []string packages []string + // fields required for image building + packageId string + packageContentProvider startosis_packages.PackageContentProvider + packageReplaceOptions map[string]string + returnValue *starlarkstruct.Struct serviceConfig *service.ServiceConfig storeSpecList []*store_spec.StoreSpec @@ -146,7 +162,7 @@ type RunPythonCapabilities struct { description string } -func (builtin *RunPythonCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) { +func (builtin *RunPythonCapabilities) Interpret(locatorOfModuleInWhichThisBuiltinIsBeingCalled string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) { randomUuid := uuid.NewRandom() builtin.name = fmt.Sprintf("task-%v", randomUuid.String()) @@ -180,15 +196,28 @@ func (builtin *RunPythonCapabilities) Interpret(_ string, arguments *builtin_arg builtin.packages = packagesList } - var image string + var maybeImageName string + var maybeImageBuildSpec *image_build_spec.ImageBuildSpec + var maybeImageRegistrySpec *image_registry_spec.ImageRegistrySpec + var maybeNixBuildSpec *nix_build_spec.NixBuildSpec + var interpretationErr *startosis_errors.InterpretationError if arguments.IsSet(ImageNameArgName) { - imageStarlark, err := builtin_argument.ExtractArgumentValue[starlark.String](arguments, ImageNameArgName) + rawImageVal, err := builtin_argument.ExtractArgumentValue[starlark.Value](arguments, ImageNameArgName) if err != nil { - return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", ImageNameArgName) + return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract raw image attribute.") + } + + maybeImageName, maybeImageBuildSpec, maybeImageRegistrySpec, maybeNixBuildSpec, interpretationErr = service_config.ConvertImage( + rawImageVal, + locatorOfModuleInWhichThisBuiltinIsBeingCalled, + builtin.packageId, + builtin.packageContentProvider, + builtin.packageReplaceOptions) + if interpretationErr != nil { + return nil, startosis_errors.WrapWithInterpretationError(interpretationErr, "An error occurred converting image for run sh.") } - image = imageStarlark.GoString() } else { - image = defaultRunPythonImageName + maybeImageName = defaultRunPythonImageName } var filesArtifactExpansion *service_directory.FilesArtifactsExpansion @@ -219,9 +248,9 @@ func (builtin *RunPythonCapabilities) Interpret(_ string, arguments *builtin_arg } // build a service config from image and files artifacts expansion. - builtin.serviceConfig, err = getServiceConfig(image, filesArtifactExpansion, envVars) + builtin.serviceConfig, err = getServiceConfig(maybeImageName, maybeImageBuildSpec, maybeImageRegistrySpec, maybeNixBuildSpec, filesArtifactExpansion, envVars) if err != nil { - return nil, startosis_errors.WrapWithInterpretationError(err, "An error occurred creating service config using image '%s'", image) + return nil, startosis_errors.WrapWithInterpretationError(err, "An error occurred creating service config for run python.") } if arguments.IsSet(StoreFilesArgName) { @@ -258,7 +287,7 @@ func (builtin *RunPythonCapabilities) Validate(_ *builtin_argument.ArgumentValue if builtin.serviceConfig.GetFilesArtifactsExpansion() != nil { serviceDirpathsToArtifactIdentifiers = builtin.serviceConfig.GetFilesArtifactsExpansion().ServiceDirpathsToArtifactIdentifiers } - return validateTasksCommon(validatorEnvironment, builtin.storeSpecList, serviceDirpathsToArtifactIdentifiers, builtin.serviceConfig.GetContainerImageName()) + return validateTasksCommon(validatorEnvironment, builtin.storeSpecList, serviceDirpathsToArtifactIdentifiers, builtin.serviceConfig) } // Execute This is just v0 for run_python task - we can later improve on it. diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go index 451b12cf1a..fdb7bf47b9 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go @@ -3,6 +3,9 @@ package tasks import ( "context" "fmt" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_build_spec" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_registry_spec" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/nix_build_spec" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service_directory" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/store_spec" @@ -18,6 +21,7 @@ import ( "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/plan_yaml" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/runtime_value_store" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_errors" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_packages" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_validator" "github.com/kurtosis-tech/stacktrace" "github.com/xtgo/uuid" @@ -33,7 +37,13 @@ const ( runningShScriptPrefix = "Running sh script" ) -func NewRunShService(serviceNetwork service_network.ServiceNetwork, runtimeValueStore *runtime_value_store.RuntimeValueStore, nonBlockingMode bool) *kurtosis_plan_instruction.KurtosisPlanInstruction { +func NewRunShService( + serviceNetwork service_network.ServiceNetwork, + runtimeValueStore *runtime_value_store.RuntimeValueStore, + nonBlockingMode bool, + packageId string, + packageContentProvider startosis_packages.PackageContentProvider, + packageReplaceOptions map[string]string) *kurtosis_plan_instruction.KurtosisPlanInstruction { return &kurtosis_plan_instruction.KurtosisPlanInstruction{ KurtosisBaseBuiltin: &kurtosis_starlark_framework.KurtosisBaseBuiltin{ Name: RunShBuiltinName, @@ -47,10 +57,8 @@ func NewRunShService(serviceNetwork service_network.ServiceNetwork, runtimeValue { Name: ImageNameArgName, IsOptional: true, - ZeroValueProvider: builtin_argument.ZeroValueProvider[starlark.String], - Validator: func(argumentValue starlark.Value) *startosis_errors.InterpretationError { - return builtin_argument.NonEmptyString(argumentValue, ImageNameArgName) - }, + ZeroValueProvider: builtin_argument.ZeroValueProvider[starlark.Value], + Validator: nil, }, { Name: FilesArgName, @@ -83,17 +91,20 @@ func NewRunShService(serviceNetwork service_network.ServiceNetwork, runtimeValue Capabilities: func() kurtosis_plan_instruction.KurtosisPlanInstructionCapabilities { return &RunShCapabilities{ - serviceNetwork: serviceNetwork, - runtimeValueStore: runtimeValueStore, - name: "", - nonBlockingMode: nonBlockingMode, - serviceConfig: nil, // populated at interpretation time - run: "", // populated at interpretation time - resultUuid: "", // populated at interpretation time - storeSpecList: nil, - wait: DefaultWaitTimeoutDurationStr, - description: "", // populated at interpretation time - returnValue: nil, // populated at interpretation time + serviceNetwork: serviceNetwork, + runtimeValueStore: runtimeValueStore, + packageId: packageId, + packageReplaceOptions: packageReplaceOptions, + packageContentProvider: packageContentProvider, + name: "", + nonBlockingMode: nonBlockingMode, + serviceConfig: nil, // populated at interpretation time + run: "", // populated at interpretation time + resultUuid: "", // populated at interpretation time + storeSpecList: nil, + wait: DefaultWaitTimeoutDurationStr, + description: "", // populated at interpretation time + returnValue: nil, // populated at interpretation time } }, @@ -117,6 +128,11 @@ type RunShCapabilities struct { run string nonBlockingMode bool + // fields required for image building + packageId string + packageContentProvider startosis_packages.PackageContentProvider + packageReplaceOptions map[string]string + serviceConfig *service.ServiceConfig storeSpecList []*store_spec.StoreSpec returnValue *starlarkstruct.Struct @@ -124,22 +140,35 @@ type RunShCapabilities struct { description string } -func (builtin *RunShCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) { +func (builtin *RunShCapabilities) Interpret(locatorOfModuleInWhichThisBuiltinIsBeingCalled string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) { runCommand, err := builtin_argument.ExtractArgumentValue[starlark.String](arguments, RunArgName) if err != nil { return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", RunArgName) } builtin.run = runCommand.GoString() - var image string + var maybeImageName string + var maybeImageBuildSpec *image_build_spec.ImageBuildSpec + var maybeImageRegistrySpec *image_registry_spec.ImageRegistrySpec + var maybeNixBuildSpec *nix_build_spec.NixBuildSpec + var interpretationErr *startosis_errors.InterpretationError if arguments.IsSet(ImageNameArgName) { - imageStarlark, err := builtin_argument.ExtractArgumentValue[starlark.String](arguments, ImageNameArgName) + rawImageVal, err := builtin_argument.ExtractArgumentValue[starlark.Value](arguments, ImageNameArgName) if err != nil { - return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", ImageNameArgName) + return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract raw image attribute.") + } + + maybeImageName, maybeImageBuildSpec, maybeImageRegistrySpec, maybeNixBuildSpec, interpretationErr = service_config.ConvertImage( + rawImageVal, + locatorOfModuleInWhichThisBuiltinIsBeingCalled, + builtin.packageId, + builtin.packageContentProvider, + builtin.packageReplaceOptions) + if interpretationErr != nil { + return nil, startosis_errors.WrapWithInterpretationError(interpretationErr, "An error occurred converting image for run sh.") } - image = imageStarlark.GoString() } else { - image = defaultRunShImageName + maybeImageName = defaultRunShImageName } var filesArtifactExpansion *service_directory.FilesArtifactsExpansion @@ -170,9 +199,9 @@ func (builtin *RunShCapabilities) Interpret(_ string, arguments *builtin_argumen } // build a service config from image and files artifacts expansion. - builtin.serviceConfig, err = getServiceConfig(image, filesArtifactExpansion, envVars) + builtin.serviceConfig, err = getServiceConfig(maybeImageName, maybeImageBuildSpec, maybeImageRegistrySpec, maybeNixBuildSpec, filesArtifactExpansion, envVars) if err != nil { - return nil, startosis_errors.WrapWithInterpretationError(err, "An error occurred creating service config using image '%s'", image) + return nil, startosis_errors.WrapWithInterpretationError(err, "An error occurred creating service config using for run sh task.") } if arguments.IsSet(StoreFilesArgName) { @@ -215,7 +244,7 @@ func (builtin *RunShCapabilities) Validate(_ *builtin_argument.ArgumentValuesSet if builtin.serviceConfig.GetFilesArtifactsExpansion() != nil { serviceDirpathsToArtifactIdentifiers = builtin.serviceConfig.GetFilesArtifactsExpansion().ServiceDirpathsToArtifactIdentifiers } - return validateTasksCommon(validatorEnvironment, builtin.storeSpecList, serviceDirpathsToArtifactIdentifiers, builtin.serviceConfig.GetContainerImageName()) + return validateTasksCommon(validatorEnvironment, builtin.storeSpecList, serviceDirpathsToArtifactIdentifiers, builtin.serviceConfig) } // Execute This is just v0 for run_sh task - we can later improve on it. diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/tasks_shared.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/tasks_shared.go index d5d728b7fd..7dc3822716 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/tasks_shared.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/tasks_shared.go @@ -3,7 +3,10 @@ package tasks import ( "context" "fmt" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_build_spec" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_download_mode" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_registry_spec" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/nix_build_spec" "reflect" "strings" "time" @@ -137,7 +140,7 @@ func createInterpretationResult(resultUuid string, storeSpecList []*store_spec.S return result } -func validateTasksCommon(validatorEnvironment *startosis_validator.ValidatorEnvironment, storeSpecList []*store_spec.StoreSpec, serviceDirpathsToArtifactIdentifiers map[string][]string, imageName string) *startosis_errors.ValidationError { +func validateTasksCommon(validatorEnvironment *startosis_validator.ValidatorEnvironment, storeSpecList []*store_spec.StoreSpec, serviceDirpathsToArtifactIdentifiers map[string][]string, serviceConfig *service.ServiceConfig) *startosis_errors.ValidationError { if storeSpecList != nil { if err := validatePathIsUniqueWhileCreatingFileArtifact(storeSpecList); err != nil { return startosis_errors.WrapWithValidationError(err, "error occurred while validating file paths to copy into file artifact") @@ -156,7 +159,16 @@ func validateTasksCommon(validatorEnvironment *startosis_validator.ValidatorEnvi } } - validatorEnvironment.AppendRequiredImagePull(imageName) + // add the images to be built here + if serviceConfig.GetImageBuildSpec() != nil { + validatorEnvironment.AppendRequiredImageBuild(serviceConfig.GetContainerImageName(), serviceConfig.GetImageBuildSpec()) + } else if serviceConfig.GetImageRegistrySpec() != nil { + validatorEnvironment.AppendImageToPullWithAuth(serviceConfig.GetContainerImageName(), serviceConfig.GetImageRegistrySpec()) + } else if serviceConfig.GetNixBuildSpec() != nil { + validatorEnvironment.AppendRequiredNixBuild(serviceConfig.GetContainerImageName(), serviceConfig.GetNixBuildSpec()) + } else { + validatorEnvironment.AppendRequiredImagePull(serviceConfig.GetContainerImageName()) + } return nil } @@ -252,15 +264,18 @@ func resultMapToString(resultMap map[string]starlark.Comparable, builtinNameForL } func getServiceConfig( - image string, + maybeImageName string, + maybeImageBuildSpec *image_build_spec.ImageBuildSpec, + maybeImageRegistrySpec *image_registry_spec.ImageRegistrySpec, + maybeNixBuildSpec *nix_build_spec.NixBuildSpec, filesArtifactExpansion *service_directory.FilesArtifactsExpansion, envVars *map[string]string, ) (*service.ServiceConfig, error) { serviceConfig, err := service.CreateServiceConfig( - image, - nil, - nil, - nil, + maybeImageName, + maybeImageBuildSpec, + maybeImageRegistrySpec, + maybeNixBuildSpec, nil, nil, // This make sure that the container does not stop as soon as it starts diff --git a/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/service_config.go b/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/service_config.go index 54145d6145..492bf248f7 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/service_config.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/service_config.go @@ -275,7 +275,7 @@ func (config *ServiceConfig) ToKurtosisType( return nil, startosis_errors.NewInterpretationError("Required attribute '%s' could not be found on type '%s'", ImageAttr, ServiceConfigTypeName) } // TODO: refactor image build spec into a common interface - imageName, maybeImageBuildSpec, maybeImageRegistrySpec, maybeNixBuildSpec, interpretationErr = convertImage( + imageName, maybeImageBuildSpec, maybeImageRegistrySpec, maybeNixBuildSpec, interpretationErr = ConvertImage( rawImageAttrValue, locatorOfModuleInWhichThisBuiltInIsBeingCalled, packageId, @@ -698,7 +698,7 @@ func convertFilesArguments(attrNameForLogging string, filesDict *starlark.Dict) // If [image] is an ImageBuildSpec type, returns name for the image to build and ImageBuildSpec converted to KurtosisType // If [image] is a string, returns the image name with no image build spec (image will be fetched from local cache or remote) // If [image] is an ImageSpec type, returns the name for the image and the ImageSpec converted to KurtosisType -func convertImage( +func ConvertImage( image starlark.Value, locatorOfModuleInWhichThisBuiltInIsBeingCalled string, packageId string,