Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: enable image build spec for tasks #2374

Merged
merged 5 commits into from
Apr 17, 2024
Merged

feat: enable image build spec for tasks #2374

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
65b370c
enable img build spec for run sh
tedim52 Apr 11, 2024
c2354cd
add back image name arg
tedim52 Apr 11, 2024
b7ed1fe
enable for run python
tedim52 Apr 17, 2024
cdf254d
Merge branch 'main' into tedi/runshimg
tedim52 Apr 17, 2024
1d6c68a
turn off image validator
tedim52 Apr 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions core/server/api_container/server/startosis_engine/kurtosis_builtins.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,8 +81,8 @@ func KurtosisPlanInstructions(
render_templates.NewRenderTemplatesInstruction(serviceNetwork, runtimeValueStore),
request.NewRequest(serviceNetwork, runtimeValueStore),
start_service.NewStartService(serviceNetwork),
tasks.NewRunPythonService(serviceNetwork, runtimeValueStore, nonBlockingMode),
tasks.NewRunShService(serviceNetwork, runtimeValueStore, nonBlockingMode),
tasks.NewRunPythonService(serviceNetwork, runtimeValueStore, nonBlockingMode, packageId, packageContentProvider, packageReplaceOptions),
tasks.NewRunShService(serviceNetwork, runtimeValueStore, nonBlockingMode, packageId, packageContentProvider, packageReplaceOptions),
stop_service.NewStopService(serviceNetwork),
store_service_files.NewStoreServiceFiles(serviceNetwork),
upload_files.NewUploadFiles(packageId, serviceNetwork, packageContentProvider, packageReplaceOptions),
Expand Down
3 changes: 2 additions & 1 deletion ...ver/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -326,7 +326,8 @@ func validateAndConvertConfigAndReadyCondition(
locatorOfModuleInWhichThisBuiltInIsBeingCalled,
packageId,
packageContentProvider,
packageReplaceOptions, imageDownloadMode)
packageReplaceOptions,
imageDownloadMode)
if interpretationErr != nil {
return nil, nil, interpretationErr
}
Expand Down
83 changes: 56 additions & 27 deletions core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@ import (
"context"
"fmt"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/exec_result"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_build_spec"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_registry_spec"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/nix_build_spec"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service_directory"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/store_spec"
Expand All @@ -19,6 +22,7 @@ import (
"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/plan_yaml"
"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/runtime_value_store"
"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_errors"
"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_packages"
"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_validator"
"github.com/kurtosis-tech/stacktrace"
"github.com/xtgo/uuid"
Expand All @@ -44,7 +48,13 @@ const (
runPythonDefaultDescription = "Running Python script"
)

func NewRunPythonService(serviceNetwork service_network.ServiceNetwork, runtimeValueStore *runtime_value_store.RuntimeValueStore, nonBlockingMode bool) *kurtosis_plan_instruction.KurtosisPlanInstruction {
func NewRunPythonService(
serviceNetwork service_network.ServiceNetwork,
runtimeValueStore *runtime_value_store.RuntimeValueStore,
nonBlockingMode bool,
packageId string,
packageContentProvider startosis_packages.PackageContentProvider,
packageReplaceOptions map[string]string) *kurtosis_plan_instruction.KurtosisPlanInstruction {
return &kurtosis_plan_instruction.KurtosisPlanInstruction{
KurtosisBaseBuiltin: &kurtosis_starlark_framework.KurtosisBaseBuiltin{
Name: RunPythonBuiltinName,
Expand All @@ -69,10 +79,8 @@ func NewRunPythonService(serviceNetwork service_network.ServiceNetwork, runtimeV
{
Name: ImageNameArgName,
IsOptional: true,
ZeroValueProvider: builtin_argument.ZeroValueProvider[starlark.String],
Validator: func(argumentValue starlark.Value) *startosis_errors.InterpretationError {
return builtin_argument.NonEmptyString(argumentValue, ImageNameArgName)
},
ZeroValueProvider: builtin_argument.ZeroValueProvider[starlark.Value],
Validator: nil,
},
{
Name: FilesArgName,
Expand All @@ -99,19 +107,22 @@ func NewRunPythonService(serviceNetwork service_network.ServiceNetwork, runtimeV

Capabilities: func() kurtosis_plan_instruction.KurtosisPlanInstructionCapabilities {
return &RunPythonCapabilities{
serviceNetwork: serviceNetwork,
runtimeValueStore: runtimeValueStore,
pythonArguments: nil,
packages: nil,
name: "",
nonBlockingMode: nonBlockingMode,
serviceConfig: nil, // populated at interpretation time
run: "", // populated at interpretation time
resultUuid: "", // populated at interpretation time
storeSpecList: nil,
wait: DefaultWaitTimeoutDurationStr,
description: "", // populated at interpretation time
returnValue: nil, // populated at interpretation time
serviceNetwork: serviceNetwork,
runtimeValueStore: runtimeValueStore,
pythonArguments: nil,
packages: nil,
name: "",
nonBlockingMode: nonBlockingMode,
packageId: packageId,
packageContentProvider: packageContentProvider,
packageReplaceOptions: packageReplaceOptions,
serviceConfig: nil, // populated at interpretation time
run: "", // populated at interpretation time
resultUuid: "", // populated at interpretation time
storeSpecList: nil,
wait: DefaultWaitTimeoutDurationStr,
description: "", // populated at interpretation time
returnValue: nil, // populated at interpretation time
}
},

Expand Down Expand Up @@ -139,14 +150,19 @@ type RunPythonCapabilities struct {
pythonArguments []string
packages []string

// fields required for image building
packageId string
packageContentProvider startosis_packages.PackageContentProvider
packageReplaceOptions map[string]string

returnValue *starlarkstruct.Struct
serviceConfig *service.ServiceConfig
storeSpecList []*store_spec.StoreSpec
wait string
description string
}

func (builtin *RunPythonCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *RunPythonCapabilities) Interpret(locatorOfModuleInWhichThisBuiltinIsBeingCalled string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
randomUuid := uuid.NewRandom()
builtin.name = fmt.Sprintf("task-%v", randomUuid.String())

Expand Down Expand Up @@ -180,15 +196,28 @@ func (builtin *RunPythonCapabilities) Interpret(_ string, arguments *builtin_arg
builtin.packages = packagesList
}

var image string
var maybeImageName string
var maybeImageBuildSpec *image_build_spec.ImageBuildSpec
var maybeImageRegistrySpec *image_registry_spec.ImageRegistrySpec
var maybeNixBuildSpec *nix_build_spec.NixBuildSpec
var interpretationErr *startosis_errors.InterpretationError
if arguments.IsSet(ImageNameArgName) {
imageStarlark, err := builtin_argument.ExtractArgumentValue[starlark.String](arguments, ImageNameArgName)
rawImageVal, err := builtin_argument.ExtractArgumentValue[starlark.Value](arguments, ImageNameArgName)
if err != nil {
return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", ImageNameArgName)
return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract raw image attribute.")
}

maybeImageName, maybeImageBuildSpec, maybeImageRegistrySpec, maybeNixBuildSpec, interpretationErr = service_config.ConvertImage(
rawImageVal,
locatorOfModuleInWhichThisBuiltinIsBeingCalled,
builtin.packageId,
builtin.packageContentProvider,
builtin.packageReplaceOptions)
if interpretationErr != nil {
return nil, startosis_errors.WrapWithInterpretationError(interpretationErr, "An error occurred converting image for run sh.")
}
image = imageStarlark.GoString()
} else {
image = defaultRunPythonImageName
maybeImageName = defaultRunPythonImageName
}

var filesArtifactExpansion *service_directory.FilesArtifactsExpansion
Expand Down Expand Up @@ -219,9 +248,9 @@ func (builtin *RunPythonCapabilities) Interpret(_ string, arguments *builtin_arg
}

// build a service config from image and files artifacts expansion.
builtin.serviceConfig, err = getServiceConfig(image, filesArtifactExpansion, envVars)
builtin.serviceConfig, err = getServiceConfig(maybeImageName, maybeImageBuildSpec, maybeImageRegistrySpec, maybeNixBuildSpec, filesArtifactExpansion, envVars)
if err != nil {
return nil, startosis_errors.WrapWithInterpretationError(err, "An error occurred creating service config using image '%s'", image)
return nil, startosis_errors.WrapWithInterpretationError(err, "An error occurred creating service config for run python.")
}

if arguments.IsSet(StoreFilesArgName) {
Expand Down Expand Up @@ -258,7 +287,7 @@ func (builtin *RunPythonCapabilities) Validate(_ *builtin_argument.ArgumentValue
if builtin.serviceConfig.GetFilesArtifactsExpansion() != nil {
serviceDirpathsToArtifactIdentifiers = builtin.serviceConfig.GetFilesArtifactsExpansion().ServiceDirpathsToArtifactIdentifiers
}
return validateTasksCommon(validatorEnvironment, builtin.storeSpecList, serviceDirpathsToArtifactIdentifiers, builtin.serviceConfig.GetContainerImageName())
return validateTasksCommon(validatorEnvironment, builtin.storeSpecList, serviceDirpathsToArtifactIdentifiers, builtin.serviceConfig)
}

// Execute This is just v0 for run_python task - we can later improve on it.
Expand Down
79 changes: 54 additions & 25 deletions core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@ package tasks
import (
"context"
"fmt"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_build_spec"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_registry_spec"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/nix_build_spec"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service_directory"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/store_spec"
Expand All @@ -18,6 +21,7 @@ import (
"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/plan_yaml"
"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/runtime_value_store"
"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_errors"
"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_packages"
"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_validator"
"github.com/kurtosis-tech/stacktrace"
"github.com/xtgo/uuid"
Expand All @@ -33,7 +37,13 @@ const (
runningShScriptPrefix = "Running sh script"
)

func NewRunShService(serviceNetwork service_network.ServiceNetwork, runtimeValueStore *runtime_value_store.RuntimeValueStore, nonBlockingMode bool) *kurtosis_plan_instruction.KurtosisPlanInstruction {
func NewRunShService(
serviceNetwork service_network.ServiceNetwork,
runtimeValueStore *runtime_value_store.RuntimeValueStore,
nonBlockingMode bool,
packageId string,
packageContentProvider startosis_packages.PackageContentProvider,
packageReplaceOptions map[string]string) *kurtosis_plan_instruction.KurtosisPlanInstruction {
return &kurtosis_plan_instruction.KurtosisPlanInstruction{
KurtosisBaseBuiltin: &kurtosis_starlark_framework.KurtosisBaseBuiltin{
Name: RunShBuiltinName,
Expand All @@ -47,10 +57,8 @@ func NewRunShService(serviceNetwork service_network.ServiceNetwork, runtimeValue
{
Name: ImageNameArgName,
IsOptional: true,
ZeroValueProvider: builtin_argument.ZeroValueProvider[starlark.String],
Validator: func(argumentValue starlark.Value) *startosis_errors.InterpretationError {
return builtin_argument.NonEmptyString(argumentValue, ImageNameArgName)
},
ZeroValueProvider: builtin_argument.ZeroValueProvider[starlark.Value],
Validator: nil,
},
{
Name: FilesArgName,
Expand Down Expand Up @@ -83,17 +91,20 @@ func NewRunShService(serviceNetwork service_network.ServiceNetwork, runtimeValue

Capabilities: func() kurtosis_plan_instruction.KurtosisPlanInstructionCapabilities {
return &RunShCapabilities{
serviceNetwork: serviceNetwork,
runtimeValueStore: runtimeValueStore,
name: "",
nonBlockingMode: nonBlockingMode,
serviceConfig: nil, // populated at interpretation time
run: "", // populated at interpretation time
resultUuid: "", // populated at interpretation time
storeSpecList: nil,
wait: DefaultWaitTimeoutDurationStr,
description: "", // populated at interpretation time
returnValue: nil, // populated at interpretation time
serviceNetwork: serviceNetwork,
runtimeValueStore: runtimeValueStore,
packageId: packageId,
packageReplaceOptions: packageReplaceOptions,
packageContentProvider: packageContentProvider,
name: "",
nonBlockingMode: nonBlockingMode,
serviceConfig: nil, // populated at interpretation time
run: "", // populated at interpretation time
resultUuid: "", // populated at interpretation time
storeSpecList: nil,
wait: DefaultWaitTimeoutDurationStr,
description: "", // populated at interpretation time
returnValue: nil, // populated at interpretation time
}
},

Expand All @@ -117,29 +128,47 @@ type RunShCapabilities struct {
run string
nonBlockingMode bool

// fields required for image building
packageId string
packageContentProvider startosis_packages.PackageContentProvider
packageReplaceOptions map[string]string

serviceConfig *service.ServiceConfig
storeSpecList []*store_spec.StoreSpec
returnValue *starlarkstruct.Struct
wait string
description string
}

func (builtin *RunShCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *RunShCapabilities) Interpret(locatorOfModuleInWhichThisBuiltinIsBeingCalled string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
runCommand, err := builtin_argument.ExtractArgumentValue[starlark.String](arguments, RunArgName)
if err != nil {
return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", RunArgName)
}
builtin.run = runCommand.GoString()

var image string
var maybeImageName string
var maybeImageBuildSpec *image_build_spec.ImageBuildSpec
var maybeImageRegistrySpec *image_registry_spec.ImageRegistrySpec
var maybeNixBuildSpec *nix_build_spec.NixBuildSpec
var interpretationErr *startosis_errors.InterpretationError
if arguments.IsSet(ImageNameArgName) {
imageStarlark, err := builtin_argument.ExtractArgumentValue[starlark.String](arguments, ImageNameArgName)
rawImageVal, err := builtin_argument.ExtractArgumentValue[starlark.Value](arguments, ImageNameArgName)
if err != nil {
return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", ImageNameArgName)
return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract raw image attribute.")
}

maybeImageName, maybeImageBuildSpec, maybeImageRegistrySpec, maybeNixBuildSpec, interpretationErr = service_config.ConvertImage(
rawImageVal,
locatorOfModuleInWhichThisBuiltinIsBeingCalled,
builtin.packageId,
builtin.packageContentProvider,
builtin.packageReplaceOptions)
if interpretationErr != nil {
return nil, startosis_errors.WrapWithInterpretationError(interpretationErr, "An error occurred converting image for run sh.")
}
image = imageStarlark.GoString()
} else {
image = defaultRunShImageName
maybeImageName = defaultRunShImageName
}

var filesArtifactExpansion *service_directory.FilesArtifactsExpansion
Expand Down Expand Up @@ -170,9 +199,9 @@ func (builtin *RunShCapabilities) Interpret(_ string, arguments *builtin_argumen
}

// build a service config from image and files artifacts expansion.
builtin.serviceConfig, err = getServiceConfig(image, filesArtifactExpansion, envVars)
builtin.serviceConfig, err = getServiceConfig(maybeImageName, maybeImageBuildSpec, maybeImageRegistrySpec, maybeNixBuildSpec, filesArtifactExpansion, envVars)
if err != nil {
return nil, startosis_errors.WrapWithInterpretationError(err, "An error occurred creating service config using image '%s'", image)
return nil, startosis_errors.WrapWithInterpretationError(err, "An error occurred creating service config using for run sh task.")
}

if arguments.IsSet(StoreFilesArgName) {
Expand Down Expand Up @@ -215,7 +244,7 @@ func (builtin *RunShCapabilities) Validate(_ *builtin_argument.ArgumentValuesSet
if builtin.serviceConfig.GetFilesArtifactsExpansion() != nil {
serviceDirpathsToArtifactIdentifiers = builtin.serviceConfig.GetFilesArtifactsExpansion().ServiceDirpathsToArtifactIdentifiers
}
return validateTasksCommon(validatorEnvironment, builtin.storeSpecList, serviceDirpathsToArtifactIdentifiers, builtin.serviceConfig.GetContainerImageName())
return validateTasksCommon(validatorEnvironment, builtin.storeSpecList, serviceDirpathsToArtifactIdentifiers, builtin.serviceConfig)
}

// Execute This is just v0 for run_sh task - we can later improve on it.
Expand Down
Loading
Loading