From 19c763d5b8d0bfdb7246284916a5e8e48db30837 Mon Sep 17 00:00:00 2001 From: Brandon Sprague Date: Wed, 14 Feb 2024 10:52:22 -0800 Subject: [PATCH] Add configuration for separate parser image (#182) --- cmd/runner/taskrunner/taskrunner.go | 56 ++++++++++++++++++----------- cmd/server/configs/local.conf | 3 +- cmd/server/main.go | 22 +++++++----- secrets/secrets.go | 40 +++++++++++++-------- 4 files changed, 76 insertions(+), 45 deletions(-) diff --git a/cmd/runner/taskrunner/taskrunner.go b/cmd/runner/taskrunner/taskrunner.go index fd3b197..06ec4c6 100644 --- a/cmd/runner/taskrunner/taskrunner.go +++ b/cmd/runner/taskrunner/taskrunner.go @@ -1,6 +1,9 @@ // Package taskrunner implements the logic for preparing a portfolio for // analysis, regardless of the underlying substrate we'll run the external // processing logic on (e.g Docker or locally). +// +// TODO: We use the tag "latest" throughout. For most cases, we'll want to +// version this and take in the image tag as part of the request. package taskrunner import ( @@ -20,8 +23,10 @@ type Config struct { // like: /configs/{local,dev}.conf ConfigPath string - // BaseImage is the runner image to execute, not specifying a tag. - BaseImage *task.BaseImage + // RunnerImage is the runner image to execute, not specifying a tag. + RunnerImage *task.BaseImage + // RunnerImage is the parser image to execute, not specifying a tag. + ParserImage *task.BaseImage Logger *zap.Logger @@ -33,8 +38,12 @@ func (c *Config) validate() error { return errors.New("no runner config path given") } - if err := validateImage(c.BaseImage); err != nil { - return fmt.Errorf("invalid base image: %w", err) + if err := validateImage(c.RunnerImage); err != nil { + return fmt.Errorf("invalid runner image: %w", err) + } + + if err := validateImage(c.ParserImage); err != nil { + return fmt.Errorf("invalid parser image: %w", err) } if c.Logger == nil { @@ -63,10 +72,11 @@ type Runner interface { } type TaskRunner struct { - logger *zap.Logger - runner Runner - baseImage *task.BaseImage - configPath string + logger *zap.Logger + runner Runner + runnerImage *task.BaseImage + parserImage *task.BaseImage + configPath string } func New(cfg *Config) (*TaskRunner, error) { @@ -75,10 +85,11 @@ func New(cfg *Config) (*TaskRunner, error) { } return &TaskRunner{ - logger: cfg.Logger, - runner: cfg.Runner, - baseImage: cfg.BaseImage, - configPath: cfg.ConfigPath, + logger: cfg.Logger, + runner: cfg.Runner, + runnerImage: cfg.RunnerImage, + parserImage: cfg.ParserImage, + configPath: cfg.ConfigPath, }, nil } @@ -112,7 +123,7 @@ func (tr *TaskRunner) ParsePortfolio(ctx context.Context, req *task.ParsePortfol Key: "PARSE_PORTFOLIO_REQUEST", Value: value, }, - }) + }, withTag(tr.parserImage, "latest")) } func (tr *TaskRunner) CreateAudit(ctx context.Context, req *task.CreateAuditRequest) (task.ID, task.RunnerID, error) { @@ -129,7 +140,7 @@ func (tr *TaskRunner) CreateAudit(ctx context.Context, req *task.CreateAuditRequ Key: "CREATE_AUDIT_REQUEST", Value: value, }, - }) + }, withTag(tr.runnerImage, "latest")) } func (tr *TaskRunner) CreateReport(ctx context.Context, req *task.CreateReportRequest) (task.ID, task.RunnerID, error) { @@ -146,10 +157,17 @@ func (tr *TaskRunner) CreateReport(ctx context.Context, req *task.CreateReportRe Key: "CREATE_REPORT_REQUEST", Value: value, }, - }) + }, withTag(tr.runnerImage, "latest")) } -func (tr *TaskRunner) run(ctx context.Context, env []task.EnvVar) (task.ID, task.RunnerID, error) { +func withTag(img *task.BaseImage, tag string) *task.Image { + return &task.Image{ + Base: *img, + Tag: tag, + } +} + +func (tr *TaskRunner) run(ctx context.Context, env []task.EnvVar, image *task.Image) (task.ID, task.RunnerID, error) { tr.logger.Info("triggering task run", zap.Any("env", env)) taskID := uuid.NewString() runnerID, err := tr.runner.Run(ctx, &task.Config{ @@ -159,11 +177,7 @@ func (tr *TaskRunner) run(ctx context.Context, env []task.EnvVar) (task.ID, task }), Flags: []string{"--config=" + tr.configPath}, Command: []string{"/runner"}, - Image: &task.Image{ - Base: *tr.baseImage, - // TODO: Take in the image digest as part of the task definition, as this can change per request. - Tag: "latest", - }, + Image: image, }) if err != nil { return "", "", fmt.Errorf("failed to run task %q, %q: %w", taskID, runnerID, err) diff --git a/cmd/server/configs/local.conf b/cmd/server/configs/local.conf index d7ac4ac..1bbef1b 100644 --- a/cmd/server/configs/local.conf +++ b/cmd/server/configs/local.conf @@ -25,4 +25,5 @@ secret_runner_config_resource_group rmi-pacta-local secret_runner_config_managed_identity_client_id c02b7346-6ba6-438a-8136-1ccb608e5449 secret_runner_config_job_name pacta-runner-local secret_runner_config_image_registry rmisa.azurecr.io -secret_runner_config_image_name runner +secret_runner_config_runner_image_name runner +secret_runner_config_parser_image_name parser diff --git a/cmd/server/main.go b/cmd/server/main.go index b1e9b46..92b9b7c 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -97,8 +97,9 @@ func run(args []string) error { runnerConfigManagedIdentityClientID = fs.String("secret_runner_config_managed_identity_client_id", "", "Client ID of the identity to run runner jobs with") runnerConfigJobName = fs.String("secret_runner_config_job_name", "", "Name of the Container Apps Job to start instances of.") - runnerConfigImageRegistry = fs.String("secret_runner_config_image_registry", "", "Registry where PACTA runner images live, like 'rmisa.azurecr.io'") - runnerConfigImageName = fs.String("secret_runner_config_image_name", "", "Name of the Docker image of the PACTA runner, like 'runner'") + runnerConfigImageRegistry = fs.String("secret_runner_config_image_registry", "", "Registry where PACTA runner images live, like 'rmisa.azurecr.io'") + runnerConfigRunnerImageName = fs.String("secret_runner_config_runner_image_name", "", "Name of the Docker image of the PACTA runner, like 'runner'") + runnerConfigParserImageName = fs.String("secret_runner_config_parser_image_name", "", "Name of the Docker image of the PACTA parser, like 'pactaparser'") ) // Allows for passing in configuration via a -config path/to/env-file.conf // flag, see https://pkg.go.dev/github.com/namsral/flag#readme-usage @@ -143,9 +144,10 @@ func run(args []string) error { ResourceGroup: *runnerConfigResourceGroup, ManagedIdentityClientID: *runnerConfigManagedIdentityClientID, JobName: *runnerConfigJobName, - Image: &secrets.RawRunnerImage{ - Registry: *runnerConfigImageRegistry, - Name: *runnerConfigImageName, + Images: &secrets.RawRunnerImages{ + Registry: *runnerConfigImageRegistry, + RunnerName: *runnerConfigRunnerImageName, + ParserName: *runnerConfigParserImageName, }, }, }) @@ -236,9 +238,13 @@ func run(args []string) error { tr, err := taskrunner.New(&taskrunner.Config{ ConfigPath: runCfg.ConfigPath, - BaseImage: &task.BaseImage{ - Registry: runCfg.Image.Registry, - Name: runCfg.Image.Name, + RunnerImage: &task.BaseImage{ + Registry: runCfg.RunnerImage.Registry, + Name: runCfg.RunnerImage.Name, + }, + ParserImage: &task.BaseImage{ + Registry: runCfg.ParserImage.Registry, + Name: runCfg.ParserImage.Name, }, Logger: logger, Runner: runner, diff --git a/secrets/secrets.go b/secrets/secrets.go index b5350d0..f626737 100644 --- a/secrets/secrets.go +++ b/secrets/secrets.go @@ -25,8 +25,9 @@ type AuthVerificationKey struct { } type RunnerConfig struct { - ConfigPath string - Image RunnerImage + ConfigPath string + RunnerImage Image + ParserImage Image SubscriptionID string ResourceGroup string @@ -34,7 +35,7 @@ type RunnerConfig struct { JobName string } -type RunnerImage struct { +type Image struct { Registry string Name string } @@ -52,7 +53,7 @@ type RawAuthVerificationKey struct { type RawRunnerConfig struct { ConfigPath string - Image *RawRunnerImage + Images *RawRunnerImages SubscriptionID string ResourceGroup string @@ -60,9 +61,11 @@ type RawRunnerConfig struct { JobName string } -type RawRunnerImage struct { +type RawRunnerImages struct { Registry string - Name string + + RunnerName string + ParserName string } func LoadPACTA(rawCfg *RawPACTAConfig) (*PACTAConfig, error) { @@ -198,14 +201,17 @@ func parseRunnerConfig(cfg *RawRunnerConfig) (RunnerConfig, error) { return RunnerConfig{}, errors.New("no runner config was provided") } - if cfg.Image == nil { - return RunnerConfig{}, errors.New("no runner_config.image was provided") + if cfg.Images == nil { + return RunnerConfig{}, errors.New("no runner_config.images was provided") } - if cfg.Image.Name == "" { - return RunnerConfig{}, errors.New("no runner_config.image.name was provided") + if cfg.Images.RunnerName == "" { + return RunnerConfig{}, errors.New("no runner_config.images.runner_name was provided") } - if cfg.Image.Registry == "" { - return RunnerConfig{}, errors.New("no runner_config.image.registry was provided") + if cfg.Images.ParserName == "" { + return RunnerConfig{}, errors.New("no runner_config.images.parser_name was provided") + } + if cfg.Images.Registry == "" { + return RunnerConfig{}, errors.New("no runner_config.images.registry was provided") } if cfg.ConfigPath == "" { @@ -229,9 +235,13 @@ func parseRunnerConfig(cfg *RawRunnerConfig) (RunnerConfig, error) { ResourceGroup: cfg.ResourceGroup, ManagedIdentityClientID: cfg.ManagedIdentityClientID, JobName: cfg.JobName, - Image: RunnerImage{ - Registry: cfg.Image.Registry, - Name: cfg.Image.Name, + RunnerImage: Image{ + Registry: cfg.Images.Registry, + Name: cfg.Images.RunnerName, + }, + ParserImage: Image{ + Registry: cfg.Images.Registry, + Name: cfg.Images.ParserName, }, }, nil }