feat!: allow to mount multiple artifacts to the same folder in a serv…

…ice. Users will need to replace the `Directory.artifac_name` field key with `Directory.artifac_names` (#2025)

## Description:
allow to mount multiple artifacts to the same folder in a service

## Is this change user facing?
YES

## References (if applicable):
Fix #1519

container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go

@@ -1738,7 +1738,7 @@ func (manager *DockerManager) getContainerHostConfig(
 	}
 
 	// NOTE: Do NOT use PublishAllPorts here!!!! This will work if a Dockerfile doesn't have an EXPOSE directive, but
-	//  if the Dockerfile *does* have an EXPOSE directive then _only_ the ports with EXPOSE will be published
+	//  if the Dockerfile *does* have and EXPOSE directive then _only_ the ports with EXPOSE will be published
 	// See also: https://www.ctl.io/developers/blog/post/docker-networking-rules/
 
 	containerHostConfigPtr := &container.HostConfig{

container-engine-lib/lib/backend_interface/objects/service/service_config_test.go

@@ -97,9 +97,9 @@ func testFilesArtifactExpansion() *service_directory.FilesArtifactsExpansion {
 			"ENV_VAR1": "env_var1_value",
 			"ENV_VAR2": "env_var2_value",
 		},
-		ServiceDirpathsToArtifactIdentifiers: map[string]string{
-			"/pahth/number1": "first_identifier",
-			"/path/number2":  "second_idenfifier",
+		ServiceDirpathsToArtifactIdentifiers: map[string][]string{
+			"/path/number1": {"first_identifier"},
+			"/path/number2": {"second_identifier"},
 		},
 		ExpanderDirpathsToServiceDirpaths: map[string]string{
 			"/expander/dir1": "/service/dir1",

container-engine-lib/lib/backend_interface/objects/service_directory/files_artifacts_expansion.go

@@ -9,8 +9,8 @@ type FilesArtifactsExpansion struct {
 	// its operation
 	ExpanderEnvVars map[string]string
 
-	// Map of dirpaths on the target service to the artificat identifier that will be mounted at this location
-	ServiceDirpathsToArtifactIdentifiers map[string]string
+	// Map of dirpaths on the target service to the artifact identifier that will be mounted at this location
+	ServiceDirpathsToArtifactIdentifiers map[string][]string
 
 	// Map of dirpaths that the expander container expects (which the expander will expand into), mapped to
 	// dirpaths on the user service container where those same directories should be made available

container-engine-lib/lib/database_accessors/enclave_db/service_registration/repository_test.go

@@ -342,9 +342,9 @@ func testFilesArtifactExpansion() *service_directory.FilesArtifactsExpansion {
 			"ENV_VAR1": "env_var1_value",
 			"ENV_VAR2": "env_var2_value",
 		},
-		ServiceDirpathsToArtifactIdentifiers: map[string]string{
-			"/pahth/number1": "first_identifier",
-			"/path/number2":  "second_idenfifier",
+		ServiceDirpathsToArtifactIdentifiers: map[string][]string{
+			"/path/number1": {"first_identifier"},
+			"/path/number2": {"second_identifier"},
 		},
 		ExpanderDirpathsToServiceDirpaths: map[string]string{
 			"/expander/dir1": "/service/dir1",

core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service.go

@@ -209,10 +209,12 @@ func (builtin *AddServiceCapabilities) TryResolveWith(instructionsAreEqual bool,
 	// We check if there has been some updates to the files it's mounting. If that's the case, it should be rerun
 	filesArtifactsExpansion := builtin.serviceConfig.GetFilesArtifactsExpansion()
 	if filesArtifactsExpansion != nil {
-		for _, filesArtifactName := range filesArtifactsExpansion.ServiceDirpathsToArtifactIdentifiers {
-			if enclaveComponents.HasFilesArtifactBeenUpdated(filesArtifactName) {
-				enclaveComponents.AddService(builtin.serviceName, enclave_structure.ComponentIsUpdated)
-				return enclave_structure.InstructionIsUpdate
+		for _, filesArtifactNames := range filesArtifactsExpansion.ServiceDirpathsToArtifactIdentifiers {
+			for _, filesArtifactName := range filesArtifactNames {
+				if enclaveComponents.HasFilesArtifactBeenUpdated(filesArtifactName) {
+					enclaveComponents.AddService(builtin.serviceName, enclave_structure.ComponentIsUpdated)
+					return enclave_structure.InstructionIsUpdate
+				}
 			}
 		}
 	}

core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service_shared.go

@@ -87,9 +87,11 @@ func validateSingleService(validatorEnvironment *startosis_validator.ValidatorEn
 		return startosis_errors.NewValidationError("There was an error validating '%s' as service with the name '%s' already exists inside the package. Adding two different services with the same name isn't allowed; we recommend prefixing/suffixing the two service names or using two different names entirely.", AddServiceBuiltinName, serviceName)
 	}
 	if serviceConfig.GetFilesArtifactsExpansion() != nil {
-		for _, artifactName := range serviceConfig.GetFilesArtifactsExpansion().ServiceDirpathsToArtifactIdentifiers {
-			if validatorEnvironment.DoesArtifactNameExist(artifactName) == startosis_validator.ComponentNotFound {
-				return startosis_errors.NewValidationError("There was an error validating '%s' as artifact name '%s' does not exist", AddServiceBuiltinName, artifactName)
+		for _, artifactNames := range serviceConfig.GetFilesArtifactsExpansion().ServiceDirpathsToArtifactIdentifiers {
+			for _, artifactName := range artifactNames {
+				if validatorEnvironment.DoesArtifactNameExist(artifactName) == startosis_validator.ComponentNotFound {
+					return startosis_errors.NewValidationError("There was an error validating '%s' as artifact name '%s' does not exist", AddServiceBuiltinName, artifactName)
+				}
 			}
 		}
 	}

core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_services.go

@@ -259,9 +259,11 @@ func (builtin *AddServicesCapabilities) TryResolveWith(instructionsAreEqual bool
 		// Check whether one file as been updated - if yes the instruction will need to be rerun
 		filesArtifactsExpansion := serviceConfig.GetFilesArtifactsExpansion()
 		if filesArtifactsExpansion != nil {
-			for _, filesArtifactName := range filesArtifactsExpansion.ServiceDirpathsToArtifactIdentifiers {
-				if enclaveComponents.HasFilesArtifactBeenUpdated(filesArtifactName) {
-					atLeastOneFileHasBeenUpdated = true
+			for _, filesArtifactNames := range filesArtifactsExpansion.ServiceDirpathsToArtifactIdentifiers {
+				for _, filesArtifactName := range filesArtifactNames {
+					if enclaveComponents.HasFilesArtifactBeenUpdated(filesArtifactName) {
+						atLeastOneFileHasBeenUpdated = true
+					}
 				}
 			}
 		}

core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go

@@ -192,7 +192,11 @@ func (builtin *RunPythonCapabilities) Interpret(_ string, arguments *builtin_arg
 			if interpretationErr != nil {
 				return nil, interpretationErr
 			}
-			filesArtifactExpansion, interpretationErr = service_config.ConvertFilesArtifactsMounts(filesArtifactMountDirPaths, builtin.serviceNetwork)
+			multipleFilesArtifactsMountDirPaths := map[string][]string{}
+			for pathToFile, fileArtifactName := range filesArtifactMountDirPaths {
+				multipleFilesArtifactsMountDirPaths[pathToFile] = []string{fileArtifactName}
+			}
+			filesArtifactExpansion, interpretationErr = service_config.ConvertFilesArtifactsMounts(multipleFilesArtifactsMountDirPaths, builtin.serviceNetwork)
 			if interpretationErr != nil {
 				return nil, interpretationErr
 			}
@@ -233,7 +237,7 @@ func (builtin *RunPythonCapabilities) Interpret(_ string, arguments *builtin_arg
 
 func (builtin *RunPythonCapabilities) Validate(_ *builtin_argument.ArgumentValuesSet, validatorEnvironment *startosis_validator.ValidatorEnvironment) *startosis_errors.ValidationError {
 	// TODO add validation for python script
-	var serviceDirpathsToArtifactIdentifiers map[string]string
+	var serviceDirpathsToArtifactIdentifiers map[string][]string
 	if builtin.serviceConfig.GetFilesArtifactsExpansion() != nil {
 		serviceDirpathsToArtifactIdentifiers = builtin.serviceConfig.GetFilesArtifactsExpansion().ServiceDirpathsToArtifactIdentifiers
 	}

core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go

@@ -136,7 +136,11 @@ func (builtin *RunShCapabilities) Interpret(_ string, arguments *builtin_argumen
 			if interpretationErr != nil {
 				return nil, interpretationErr
 			}
-			filesArtifactExpansion, interpretationErr = service_config.ConvertFilesArtifactsMounts(filesArtifactMountDirPaths, builtin.serviceNetwork)
+			multipleFilesArtifactsMountDirPaths := map[string][]string{}
+			for pathToFile, fileArtifactName := range filesArtifactMountDirPaths {
+				multipleFilesArtifactsMountDirPaths[pathToFile] = []string{fileArtifactName}
+			}
+			filesArtifactExpansion, interpretationErr = service_config.ConvertFilesArtifactsMounts(multipleFilesArtifactsMountDirPaths, builtin.serviceNetwork)
 			if interpretationErr != nil {
 				return nil, interpretationErr
 			}
@@ -179,7 +183,7 @@ func (builtin *RunShCapabilities) Interpret(_ string, arguments *builtin_argumen
 
 func (builtin *RunShCapabilities) Validate(_ *builtin_argument.ArgumentValuesSet, validatorEnvironment *startosis_validator.ValidatorEnvironment) *startosis_errors.ValidationError {
 	// TODO validate bash
-	var serviceDirpathsToArtifactIdentifiers map[string]string
+	var serviceDirpathsToArtifactIdentifiers map[string][]string
 	if builtin.serviceConfig.GetFilesArtifactsExpansion() != nil {
 		serviceDirpathsToArtifactIdentifiers = builtin.serviceConfig.GetFilesArtifactsExpansion().ServiceDirpathsToArtifactIdentifiers
 	}

core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/tasks_shared.go

@@ -134,7 +134,7 @@ func createInterpretationResult(resultUuid string, storeSpecList []*store_spec.S
 	return result
 }
 
-func validateTasksCommon(validatorEnvironment *startosis_validator.ValidatorEnvironment, storeSpecList []*store_spec.StoreSpec, serviceDirpathsToArtifactIdentifiers map[string]string, imageName string) *startosis_errors.ValidationError {
+func validateTasksCommon(validatorEnvironment *startosis_validator.ValidatorEnvironment, storeSpecList []*store_spec.StoreSpec, serviceDirpathsToArtifactIdentifiers map[string][]string, imageName string) *startosis_errors.ValidationError {
 	if storeSpecList != nil {
 		if err := validatePathIsUniqueWhileCreatingFileArtifact(storeSpecList); err != nil {
 			return startosis_errors.WrapWithValidationError(err, "error occurred while validating file paths to copy into file artifact")
@@ -145,9 +145,11 @@ func validateTasksCommon(validatorEnvironment *startosis_validator.ValidatorEnvi
 		}
 	}
 
-	for _, artifactName := range serviceDirpathsToArtifactIdentifiers {
-		if validatorEnvironment.DoesArtifactNameExist(artifactName) == startosis_validator.ComponentNotFound {
-			return startosis_errors.NewValidationError("There was an error validating '%s' as artifact name '%s' does not exist", RunPythonBuiltinName, artifactName)
+	for _, artifactNames := range serviceDirpathsToArtifactIdentifiers {
+		for _, artifactName := range artifactNames {
+			if validatorEnvironment.DoesArtifactNameExist(artifactName) == startosis_validator.ComponentNotFound {
+				return startosis_errors.NewValidationError("There was an error validating '%s' as artifact name '%s' does not exist", RunPythonBuiltinName, artifactName)
+			}
 		}
 	}
 

core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/builtin_argument/validators.go

@@ -1,6 +1,7 @@
 package builtin_argument
 
 import (
+	"fmt"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_errors"
 	"go.starlark.net/starlark"
@@ -21,6 +22,25 @@ func NonEmptyString(value starlark.Value, argNameForLogging string) *startosis_e
 	return nil
 }
 
+func StringListWithNotEmptyValues(value starlark.Value, argNameForLogging string) *startosis_errors.InterpretationError {
+	starlarkList, ok := value.(*starlark.List)
+	if !ok {
+		return startosis_errors.NewInterpretationError("Value for '%s' was expected to be a starlark.List but was '%s'", argNameForLogging, reflect.TypeOf(value))
+	}
+	iterator := starlarkList.Iterate()
+	defer iterator.Done()
+	var itemValue starlark.Value
+	var index = 0
+	for iterator.Next(&itemValue) {
+		argumentDescription := fmt.Sprintf("element %d in argument '%s'", index, argNameForLogging)
+		if interpretationErr := NonEmptyString(itemValue, argumentDescription); interpretationErr != nil {
+			return interpretationErr
+		}
+		index++
+	}
+	return nil
+}
+
 func Uint64InRange(value starlark.Value, argNameForLogging string, min uint64, max uint64) *startosis_errors.InterpretationError {
 	valueInt, ok := value.(starlark.Int)
 	if !ok {

core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/directory_file_artifact_framework_test.go

@@ -20,17 +20,17 @@ func (suite *KurtosisTypeConstructorTestSuite) TestDirectoryFileArtifact() {
 }
 
 func (t *directoryFileArtifactTestCase) GetStarlarkCode() string {
-	return fmt.Sprintf("%s(%s=%q)", directory.DirectoryTypeName, directory.ArtifactNameAttr, testFilesArtifactName1)
+	return fmt.Sprintf("%s(%s=[%q])", directory.DirectoryTypeName, directory.ArtifactNamesAttr, testFilesArtifactName1)
 }
 
 func (t *directoryFileArtifactTestCase) Assert(typeValue builtin_argument.KurtosisValueType) {
 	directoryStarlark, ok := typeValue.(*directory.Directory)
 	require.True(t, ok)
 
-	artifactName, found, err := directoryStarlark.GetArtifactNameIfSet()
+	artifactNames, found, err := directoryStarlark.GetArtifactNamesIfSet()
 	require.Nil(t, err)
 	require.True(t, found)
-	require.Equal(t, testFilesArtifactName1, artifactName)
+	require.Equal(t, []string{testFilesArtifactName1}, artifactNames)
 
 	persistentKey, found, err := directoryStarlark.GetPersistentKeyIfSet()
 	require.Nil(t, err)

core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/directory_multiple_file_artifacts_framework_test.go

@@ -0,0 +1,33 @@
+package test_engine
+
+import (
+	"fmt"
+	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/builtin_argument"
+	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_types/directory"
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+type directoryMultipleFileArtifactsTestCase struct {
+	*testing.T
+}
+
+func (suite *KurtosisTypeConstructorTestSuite) TestDirectoryMultipleFileArtifacts() {
+	suite.run(&directoryMultipleFileArtifactsTestCase{
+		T: suite.T(),
+	})
+}
+
+func (t *directoryMultipleFileArtifactsTestCase) GetStarlarkCode() string {
+	return fmt.Sprintf("%s(%s=[%q, %q])", directory.DirectoryTypeName, directory.ArtifactNamesAttr, testFilesArtifactName1, testFilesArtifactName2)
+}
+
+func (t *directoryMultipleFileArtifactsTestCase) Assert(typeValue builtin_argument.KurtosisValueType) {
+	directoryStarlark, ok := typeValue.(*directory.Directory)
+	require.True(t, ok)
+
+	artifactNames, found, err := directoryStarlark.GetArtifactNamesIfSet()
+	require.Nil(t, err)
+	require.True(t, found)
+	require.Equal(t, []string{testFilesArtifactName1, testFilesArtifactName2}, artifactNames)
+}

core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/directory_persistent_directory_framework_test.go

@@ -26,10 +26,10 @@ func (t *directoryPersistentDirectoryTestCase) Assert(typeValue builtin_argument
 	directoryStarlark, ok := typeValue.(*directory.Directory)
 	require.True(t, ok)
 
-	artifactName, found, err := directoryStarlark.GetArtifactNameIfSet()
+	artifactNames, found, err := directoryStarlark.GetArtifactNamesIfSet()
 	require.Nil(t, err)
 	require.False(t, found)
-	require.Empty(t, artifactName)
+	require.Empty(t, artifactNames)
 
 	persistentKey, found, err := directoryStarlark.GetPersistentKeyIfSet()
 	require.Nil(t, err)

core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/service_config_full_framework_backward_compatible_test.go

@@ -82,9 +82,9 @@ func (t *serviceConfigFullTestCaseBackwardCompatible) Assert(typeValue builtin_a
 	}
 	require.Equal(t, expectedPublicPorts, serviceConfig.GetPublicPorts())
 
-	expectedFilesArtifactMap := map[string]string{
-		testFilesArtifactPath1: testFilesArtifactName1,
-		testFilesArtifactPath2: testFilesArtifactName2,
+	expectedFilesArtifactMap := map[string][]string{
+		testFilesArtifactPath1: {testFilesArtifactName1},
+		testFilesArtifactPath2: {testFilesArtifactName2},
 	}
 	require.NotNil(t, serviceConfig.GetFilesArtifactsExpansion())
 	require.Equal(t, expectedFilesArtifactMap, serviceConfig.GetFilesArtifactsExpansion().ServiceDirpathsToArtifactIdentifiers)

core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/service_config_full_framework_test.go

@@ -35,8 +35,8 @@ func (suite *KurtosisTypeConstructorTestSuite) TestServiceConfigFull() {
 }
 
 func (t *serviceConfigFullTestCase) GetStarlarkCode() string {
-	fileArtifact1 := fmt.Sprintf("%s(%s=%q)", directory.DirectoryTypeName, directory.ArtifactNameAttr, testFilesArtifactName1)
-	fileArtifact2 := fmt.Sprintf("%s(%s=%q)", directory.DirectoryTypeName, directory.ArtifactNameAttr, testFilesArtifactName2)
+	fileArtifact1 := fmt.Sprintf("%s(%s=[%q])", directory.DirectoryTypeName, directory.ArtifactNamesAttr, testFilesArtifactName1)
+	fileArtifact2 := fmt.Sprintf("%s(%s=[%q])", directory.DirectoryTypeName, directory.ArtifactNamesAttr, testFilesArtifactName2)
 	persistentDirectory := fmt.Sprintf("%s(%s=%q)", directory.DirectoryTypeName, directory.PersistentKeyAttr, testPersistentDirectoryKey)
 	starlarkCode := fmt.Sprintf("%s(%s=%q, %s=%s, %s=%s, %s=%s, %s=%s, %s=%s, %s=%s, %s=%q, %s=%d, %s=%d, %s=%d, %s=%d, %s=%s, %s=%v)",
 		service_config.ServiceConfigTypeName,
@@ -90,9 +90,9 @@ func (t *serviceConfigFullTestCase) Assert(typeValue builtin_argument.KurtosisVa
 	}
 	require.Equal(t, expectedPublicPorts, serviceConfig.GetPublicPorts())
 
-	expectedFilesArtifactMap := map[string]string{
-		testFilesArtifactPath1: testFilesArtifactName1,
-		testFilesArtifactPath2: testFilesArtifactName2,
+	expectedFilesArtifactMap := map[string][]string{
+		testFilesArtifactPath1: {testFilesArtifactName1},
+		testFilesArtifactPath2: {testFilesArtifactName2},
 	}
 	require.NotNil(t, serviceConfig.GetFilesArtifactsExpansion())
 	require.Equal(t, expectedFilesArtifactMap, serviceConfig.GetFilesArtifactsExpansion().ServiceDirpathsToArtifactIdentifiers)

core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/service_config_multiple_files_in_same_folder_test.go

@@ -0,0 +1,64 @@
+package test_engine
+
+import (
+	"fmt"
+	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/service_network"
+	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/builtin_argument"
+	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_types/directory"
+	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_types/service_config"
+	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_packages"
+	"github.com/stretchr/testify/require"
+	"net"
+	"testing"
+)
+
+type serviceConfigMultipleFilesInSameFolderTestCase struct {
+	*testing.T
+	serviceNetwork         *service_network.MockServiceNetwork
+	packageContentProvider *startosis_packages.MockPackageContentProvider
+}
+
+func (suite *KurtosisTypeConstructorTestSuite) TestServiceConfigMultipleFilesInSameFolder() {
+
+	suite.serviceNetwork.EXPECT().GetApiContainerInfo().Times(1).Return(
+		service_network.NewApiContainerInfo(net.IPv4(0, 0, 0, 0), 0, "0.0.0"),
+	)
+
+	suite.run(&serviceConfigMultipleFilesInSameFolderTestCase{
+		T:                      suite.T(),
+		serviceNetwork:         suite.serviceNetwork,
+		packageContentProvider: suite.packageContentProvider,
+	})
+}
+
+func (t *serviceConfigMultipleFilesInSameFolderTestCase) GetStarlarkCode() string {
+	filesArtifactsDirectory := fmt.Sprintf("%s(%s=[%q, %q])", directory.DirectoryTypeName, directory.ArtifactNamesAttr, testFilesArtifactName1, testFilesArtifactName2)
+	starlarkCode := fmt.Sprintf("%s(%s=%q, %s=%s)",
+		service_config.ServiceConfigTypeName,
+		service_config.ImageAttr, testContainerImageName,
+		service_config.FilesAttr, fmt.Sprintf("{%q: %s}", testFilesArtifactPath1, filesArtifactsDirectory),
+	)
+
+	return starlarkCode
+}
+
+func (t *serviceConfigMultipleFilesInSameFolderTestCase) Assert(typeValue builtin_argument.KurtosisValueType) {
+	serviceConfigStarlark, ok := typeValue.(*service_config.ServiceConfig)
+	require.True(t, ok)
+
+	serviceConfig, err := serviceConfigStarlark.ToKurtosisType(
+		t.serviceNetwork,
+		testModulePackageId,
+		testModuleMainFileLocator,
+		t.packageContentProvider,
+		testNoPackageReplaceOptions)
+	require.Nil(t, err)
+
+	require.Equal(t, testContainerImageName, serviceConfig.GetContainerImageName())
+
+	expectedFilesArtifactMap := map[string][]string{
+		testFilesArtifactPath1: {testFilesArtifactName1, testFilesArtifactName2},
+	}
+	require.NotNil(t, serviceConfig.GetFilesArtifactsExpansion())
+	require.Equal(t, expectedFilesArtifactMap, serviceConfig.GetFilesArtifactsExpansion().ServiceDirpathsToArtifactIdentifiers)
+}