From 8d07abb59466d889ef514c2e90386bd56515333c Mon Sep 17 00:00:00 2001 From: Brandon Sprague Date: Mon, 16 Oct 2023 19:28:34 -0700 Subject: [PATCH] Start working on runner blob integration --- WORKSPACE | 8 +- azure/azblob/BUILD.bazel | 16 + azure/azblob/azblob.go | 185 +++++++++++ azure/aztask/aztask.go | 109 +++---- blob/BUILD.bazel | 14 + blob/blob.go | 30 ++ blob/blob_test.go | 122 ++++++++ cmd/runner/BUILD.bazel | 6 +- cmd/runner/main.go | 288 +++++++++++++++++- cmd/runner/taskrunner/BUILD.bazel | 1 - cmd/runner/taskrunner/taskrunner.go | 112 +++++-- cmd/server/BUILD.bazel | 4 +- cmd/server/configs/local.conf | 3 + cmd/server/main.go | 87 +++--- cmd/server/pactasrv/BUILD.bazel | 3 + cmd/server/pactasrv/pactasrv.go | 16 +- cmd/server/pactasrv/portfolio.go | 22 ++ {executors/docker => dockertask}/BUILD.bazel | 9 +- dockertask/dockertask.go | 110 +++++++ executors/docker/docker.go | 110 ------- executors/local/BUILD.bazel | 9 - executors/local/local.go | 33 -- frontend/.nvmrc | 1 + frontend/openapi/generated/pacta/index.ts | 1 + .../pacta/models/NewPortfolioAsset.ts | 12 + .../pacta/services/DefaultService.ts | 17 ++ frontend/package-lock.json | 143 +++------ frontend/package.json | 1 + frontend/pages/admin/index.vue | 6 + frontend/pages/admin/portfolio_test.vue | 72 +++++ frontend/plugins/axios.ts | 9 + go.mod | 1 + go.sum | 2 + openapi/pacta.yaml | 24 ++ pacta/pacta.go | 6 - scripts/run_db.sh | 3 + task/task.go | 60 +++- 37 files changed, 1250 insertions(+), 405 deletions(-) create mode 100644 azure/azblob/BUILD.bazel create mode 100644 azure/azblob/azblob.go create mode 100644 blob/BUILD.bazel create mode 100644 blob/blob.go create mode 100644 blob/blob_test.go create mode 100644 cmd/server/pactasrv/portfolio.go rename {executors/docker => dockertask}/BUILD.bazel (68%) create mode 100644 dockertask/dockertask.go delete mode 100644 executors/docker/docker.go delete mode 100644 executors/local/BUILD.bazel delete mode 100644 executors/local/local.go create mode 100644 frontend/.nvmrc create mode 100644 frontend/openapi/generated/pacta/models/NewPortfolioAsset.ts create mode 100644 frontend/pages/admin/portfolio_test.vue create mode 100644 frontend/plugins/axios.ts diff --git a/WORKSPACE b/WORKSPACE index 827000a..542d39c 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -99,10 +99,10 @@ oci_pull( platforms = ["linux/amd64"], ) -# TODO: Replace this with the base image provided by RMI oci_pull( name = "runner_base", - digest = "sha256:46c5b9bd3e3efff512e28350766b54355fce6337a0b44ba3f822ab918eca4520", - image = "gcr.io/distroless/base", - platforms = ["linux/amd64"], + digest = "sha256:d0b2922dc48cb6acb7c767f89f0c92ccbe1a043166971bac0b585b3851a9b720", + # TODO: Replace this with a real one + image = "docker.io/curfewreplica/pactatest", + # platforms = ["linux/amd64"], ) \ No newline at end of file diff --git a/azure/azblob/BUILD.bazel b/azure/azblob/BUILD.bazel new file mode 100644 index 0000000..dbdc03b --- /dev/null +++ b/azure/azblob/BUILD.bazel @@ -0,0 +1,16 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "azblob", + srcs = ["azblob.go"], + importpath = "github.com/RMI/pacta/azure/azblob", + visibility = ["//visibility:public"], + deps = [ + "//blob", + "@com_github_azure_azure_sdk_for_go_sdk_azcore//:azcore", + "@com_github_azure_azure_sdk_for_go_sdk_azcore//to", + "@com_github_azure_azure_sdk_for_go_sdk_storage_azblob//:azblob", + "@com_github_azure_azure_sdk_for_go_sdk_storage_azblob//sas", + "@com_github_azure_azure_sdk_for_go_sdk_storage_azblob//service", + ], +) diff --git a/azure/azblob/azblob.go b/azure/azblob/azblob.go new file mode 100644 index 0000000..00b7835 --- /dev/null +++ b/azure/azblob/azblob.go @@ -0,0 +1,185 @@ +// Package azblob wraps the existing Azure blob library to provide basic upload, +// download, and URL signing functionality against a standardized interface. +package azblob + +import ( + "context" + "fmt" + "io" + "sync" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas" + azservice "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service" + "github.com/RMI/pacta/blob" +) + +const ( + Scheme = blob.Scheme("az") +) + +type Client struct { + storageAccount string + now func() time.Time + + client *azblob.Client + svcClient *azservice.Client + + cachedUDCMu *sync.Mutex + cachedUDC *azservice.UserDelegationCredential + cachedUDCExpiry time.Time +} + +func NewClient(creds azcore.TokenCredential, storageAccount string) (*Client, error) { + serviceURL := fmt.Sprintf("https://%s.blob.core.windows.net/", storageAccount) + + client, err := azblob.NewClient(serviceURL, creds, nil) + if err != nil { + return nil, fmt.Errorf("failed to init Azure blob client: %w", err) + } + + svcClient, err := azservice.NewClient(serviceURL, creds, nil) + if err != nil { + return nil, fmt.Errorf("failed to init Azure blob service client: %w", err) + } + + return &Client{ + storageAccount: storageAccount, + now: func() time.Time { return time.Now().UTC() }, + + client: client, + svcClient: svcClient, + + cachedUDCMu: &sync.Mutex{}, + }, nil +} + +func (c *Client) Scheme() blob.Scheme { + return Scheme +} + +func (c *Client) WriteBlob(ctx context.Context, uri string, r io.Reader) error { + ctr, blb, ok := blob.SplitURI(Scheme, uri) + if !ok { + return fmt.Errorf("malformed URI %q is not for Azure", uri) + } + + if _, err := c.client.UploadStream(ctx, ctr, blb, r, nil); err != nil { + return fmt.Errorf("failed to upload blob: %w", err) + } + return nil +} + +func (c *Client) ReadBlob(ctx context.Context, uri string) (io.ReadCloser, error) { + ctr, blb, ok := blob.SplitURI(Scheme, uri) + if !ok { + return nil, fmt.Errorf("malformed URI %q is not for Azure", uri) + } + + resp, err := c.client.DownloadStream(ctx, ctr, blb, nil) + if err != nil { + return nil, fmt.Errorf("failed to read blob: %w", err) + } + + return resp.Body, nil +} + +// SignedUploadURL returns a URL that is allowed to upload to the given URI. +// See https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/storage/azblob@v1.0.0/sas#example-package-UserDelegationSAS +func (c *Client) SignedUploadURL(ctx context.Context, uri string) (string, error) { + return c.signBlob(ctx, uri, &sas.BlobPermissions{Create: true, Write: true}) +} + +// SignedDownloadURL returns a URL that is allowed to download the file at the given URI. +// See https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/storage/azblob@v1.0.0/sas#example-package-UserDelegationSAS +func (c *Client) SignedDownloadURL(ctx context.Context, uri string) (string, error) { + return c.signBlob(ctx, uri, &sas.BlobPermissions{Read: true}) +} + +func (c *Client) signBlob(ctx context.Context, uri string, perms *sas.BlobPermissions) (string, error) { + ctr, blb, ok := blob.SplitURI(Scheme, uri) + if !ok { + return "", fmt.Errorf("malformed URI %q is not for Azure", uri) + } + + // The blob component is important, otherwise the signed URL is applicable to the whole container. + if blb == "" { + return "", fmt.Errorf("uri %q did not contain a blob component", uri) + } + + now := c.now().UTC().Add(-10 * time.Second) + udc, err := c.getUserDelegationCredential(ctx, now) + if err != nil { + return "", fmt.Errorf("failed to get udc: %w", err) + } + + // Create Blob Signature Values with desired permissions and sign with user delegation credential + sasQueryParams, err := sas.BlobSignatureValues{ + Protocol: sas.ProtocolHTTPS, + StartTime: now, + ExpiryTime: now.Add(15 * time.Minute), + Permissions: perms.String(), + ContainerName: ctr, + BlobName: blb, + }.SignWithUserDelegation(udc) + if err != nil { + return "", fmt.Errorf("failed to sign blob: %w", err) + } + + return fmt.Sprintf("https://%s.blob.core.windows.net/%s/%s?%s", c.storageAccount, ctr, blb, sasQueryParams.Encode()), nil +} + +func (c *Client) ListBlobs(ctx context.Context, uriPrefix string) ([]string, error) { + ctr, blobPrefix, ok := blob.SplitURI(Scheme, uriPrefix) + if !ok { + return nil, fmt.Errorf("malformed URI prefix %q is not for Azure", uriPrefix) + } + + if blobPrefix == "" { + return nil, fmt.Errorf("uri prefix %q did not contain a blob component", uriPrefix) + } + + pager := c.client.NewListBlobsFlatPager(ctr, &azblob.ListBlobsFlatOptions{ + Prefix: &blobPrefix, + }) + + var blobs []string + for pager.More() { + resp, err := pager.NextPage(ctx) + if err != nil { + return nil, fmt.Errorf("failed to load page of blobs: %w", err) + } + for _, bi := range resp.Segment.BlobItems { + blobs = append(blobs, blob.Join(Scheme, ctr, *bi.Name)) + } + } + + return blobs, nil +} + +func (c *Client) getUserDelegationCredential(ctx context.Context, now time.Time) (*azservice.UserDelegationCredential, error) { + c.cachedUDCMu.Lock() + defer c.cachedUDCMu.Unlock() + + expiry := now.Add(48 * time.Hour) + info := azservice.KeyInfo{ + Start: to.Ptr(now.UTC().Format(sas.TimeFormat)), + Expiry: to.Ptr(expiry.UTC().Format(sas.TimeFormat)), + } + + if !c.cachedUDCExpiry.IsZero() && c.cachedUDCExpiry.Sub(now) > 1*time.Minute { + return c.cachedUDC, nil + } + + udc, err := c.svcClient.GetUserDelegationCredential(ctx, info, nil) + if err != nil { + return nil, fmt.Errorf("failed to get delegated credentials: %w", err) + } + c.cachedUDC = udc + c.cachedUDCExpiry = expiry + + return udc, nil +} diff --git a/azure/aztask/aztask.go b/azure/aztask/aztask.go index a5e0fa7..feff010 100644 --- a/azure/aztask/aztask.go +++ b/azure/aztask/aztask.go @@ -3,7 +3,6 @@ package aztask import ( - "bytes" "context" "errors" "fmt" @@ -27,16 +26,9 @@ type Config struct { // Location is the location to run the runner, like centralus Location string - // ConfigPath should be a full path to a config file in the runner image, - // like: /configs/{local,dev}.conf - ConfigPath string - // Identity is the account the runner should act as. Identity *RunnerIdentity - // Image the runner image to execute - Image *RunnerImage - Rand *rand.Rand } @@ -45,16 +37,12 @@ func (c *Config) validate() error { return errors.New("no container location given") } - if c.ConfigPath == "" { - return errors.New("no runner config path given") - } - if err := c.Identity.validate(); err != nil { return fmt.Errorf("invalid identity config: %w", err) } - if err := c.Image.validate(); err != nil { - return fmt.Errorf("invalid image config: %w", err) + if c.Rand == nil { + return errors.New("no random number generator given") } return nil @@ -99,35 +87,7 @@ func (r *RunnerIdentity) EnvironmentID() string { return fmt.Sprintf(tmpl, r.SubscriptionID, r.ResourceGroup, r.ManagedEnvironment) } -type RunnerImage struct { - // Like rmipacta.azurecr.io - Registry string - // Like runner - Name string -} - -func (ri *RunnerImage) validate() error { - if ri.Registry == "" { - return errors.New("no runner image registry given") - } - if ri.Name == "" { - return errors.New("no runner image name given") - } - return nil -} - -func (r *RunnerImage) WithTag(tag string) string { - var buf bytes.Buffer - // /: - buf.WriteString(r.Registry) - buf.WriteRune('/') - buf.WriteString(r.Name) - buf.WriteRune(':') - buf.WriteString(tag) - return buf.String() -} - -func NewTaskRunner(creds azcore.TokenCredential, cfg *Config) (*Runner, error) { +func NewRunner(creds azcore.TokenCredential, cfg *Config) (*Runner, error) { if err := cfg.validate(); err != nil { return nil, fmt.Errorf("invalid task runner config: %w", err) } @@ -149,11 +109,29 @@ func NewTaskRunner(creds azcore.TokenCredential, cfg *Config) (*Runner, error) { }, nil } -func (r *Runner) StartRun(ctx context.Context, req *task.StartRunRequest) (task.ID, error) { +func (r *Runner) Run(ctx context.Context, cfg *task.Config) (task.ID, error) { + name := r.gen.NewID() identity := r.cfg.Identity.String() envID := r.cfg.Identity.EnvironmentID() + envVars := []*armappcontainers.EnvironmentVar{ + { + Name: to.Ptr("AZURE_CLIENT_ID"), + Value: to.Ptr(r.cfg.Identity.ClientID), + }, + { + Name: to.Ptr("MANAGED_IDENTITY_CLIENT_ID"), + Value: to.Ptr(r.cfg.Identity.ClientID), + }, + } + for _, v := range cfg.Env { + envVars = append(envVars, &armappcontainers.EnvironmentVar{ + Name: to.Ptr(v.Key), + Value: to.Ptr(v.Value), + }) + } + job := armappcontainers.Job{ Location: &r.cfg.Location, Identity: &armappcontainers.ManagedServiceIdentity{ @@ -176,7 +154,7 @@ func (r *Runner) StartRun(ctx context.Context, req *task.StartRunRequest) (task. ReplicaRetryLimit: to.Ptr(int32(0)), Registries: []*armappcontainers.RegistryCredentials{ { - Server: to.Ptr(r.cfg.Image.Registry), + Server: to.Ptr(cfg.Image.Base.Registry), Identity: to.Ptr(identity), }, }, @@ -188,30 +166,12 @@ func (r *Runner) StartRun(ctx context.Context, req *task.StartRunRequest) (task. Template: &armappcontainers.JobTemplate{ Containers: []*armappcontainers.Container{ { - Args: []*string{ - to.Ptr("--config=" + r.cfg.ConfigPath), - }, - Command: []*string{ - to.Ptr("/runner"), - }, - Env: []*armappcontainers.EnvironmentVar{ - { - Name: to.Ptr("AZURE_CLIENT_ID"), - Value: to.Ptr(r.cfg.Identity.ClientID), - }, - { - Name: to.Ptr("MANAGED_IDENTITY_CLIENT_ID"), - Value: to.Ptr(r.cfg.Identity.ClientID), - }, - { - Name: to.Ptr("PORTFOLIO_ID"), - Value: to.Ptr(string(req.PortfolioID)), - }, - }, - // TODO: Take in the image digest as part of the task definition, as this can change per request. - Image: to.Ptr(r.cfg.Image.WithTag("latest")), - Name: to.Ptr(name), - Probes: []*armappcontainers.ContainerAppProbe{}, + Args: toPtrs(cfg.Flags), + Command: toPtrs(cfg.Command), + Env: envVars, + Image: to.Ptr(cfg.Image.String()), + Name: to.Ptr(name), + Probes: []*armappcontainers.ContainerAppProbe{}, Resources: &armappcontainers.ContainerResources{ CPU: to.Ptr(1.0), Memory: to.Ptr("2Gi"), @@ -246,3 +206,14 @@ func (r *Runner) StartRun(ctx context.Context, req *task.StartRunRequest) (task. return task.ID(*res.ID), nil } + +func toPtrs[T any](in []T) []*T { + if in == nil { + return nil + } + out := make([]*T, len(in)) + for i, v := range in { + out[i] = &v + } + return out +} diff --git a/blob/BUILD.bazel b/blob/BUILD.bazel new file mode 100644 index 0000000..2d32550 --- /dev/null +++ b/blob/BUILD.bazel @@ -0,0 +1,14 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "blob", + srcs = ["blob.go"], + importpath = "github.com/RMI/pacta/blob", + visibility = ["//visibility:public"], +) + +go_test( + name = "blob_test", + srcs = ["blob_test.go"], + embed = [":blob"], +) diff --git a/blob/blob.go b/blob/blob.go new file mode 100644 index 0000000..33cec9c --- /dev/null +++ b/blob/blob.go @@ -0,0 +1,30 @@ +// Package blob provides basic primitives and helpers for working with blob +// storage systems. +package blob + +import "strings" + +type Scheme string + +func (s Scheme) String() string { + return string(s) + "://" +} + +func Join(s Scheme, parts ...string) string { + return s.String() + strings.Join(parts, "/") +} + +func HasScheme(s Scheme, uri string) bool { + return strings.HasPrefix(uri, s.String()) +} + +func SplitURI(s Scheme, uri string) (string, string, bool) { + if !HasScheme(s, uri) { + return "", "", false + } + ns, obj, ok := strings.Cut(strings.TrimPrefix(uri, s.String()), "/") + if !ok { + return ns, "", ns != "" + } + return ns, obj, true +} diff --git a/blob/blob_test.go b/blob/blob_test.go new file mode 100644 index 0000000..b128fc2 --- /dev/null +++ b/blob/blob_test.go @@ -0,0 +1,122 @@ +package blob + +import "testing" + +const testScheme = Scheme("test") + +func TestSchemeString(t *testing.T) { + want := "test://" + got := testScheme.String() + + if got != want { + t.Errorf("Scheme.String() = %q, want %q", got, want) + } +} + +func TestHasScheme(t *testing.T) { + tests := []struct { + desc string + uri string + want bool + }{ + { + desc: "has scheme", + uri: "test://container/path/to/obj", + want: true, + }, + { + desc: "does not have scheme", + uri: "otherscheme://container/path/to/obj", + want: false, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + got := HasScheme(testScheme, test.uri) + if got != test.want { + t.Errorf("HasScheme = %t, want %t", got, test.want) + } + }) + } +} + +func TestJoin(t *testing.T) { + got := Join(testScheme, "container", "path", "to", "obj") + want := "test://container/path/to/obj" + + if got != want { + t.Errorf("Join = %q, want %q", got, want) + } +} + +func TestJoin_TrailingSlash(t *testing.T) { + got := Join(testScheme, "container", "path", "to", "obj", "" /* traiiling slash */) + want := "test://container/path/to/obj/" + + if got != want { + t.Errorf("Join = %q, want %q", got, want) + } +} + +func TestSplitURI(t *testing.T) { + tests := []struct { + desc string + in string + wantNamespace string + wantObject string + wantOK bool + }{ + { + desc: "valid, has namespace and object", + in: "test://container/path/to/obj", + wantNamespace: "container", + wantObject: "path/to/obj", + wantOK: true, + }, + { + desc: "valid, has namespace and no object", + in: "test://container", + wantNamespace: "container", + wantObject: "", + wantOK: true, + }, + { + desc: "valid, has namespace (with trailing slash) and no object", + in: "test://container/", + wantNamespace: "container", + wantObject: "", + wantOK: true, + }, + { + desc: "invalid, wrong scheme", + in: "otherscheme://container/path/to/obj", + wantNamespace: "", + wantObject: "", + wantOK: false, + }, + { + desc: "invalid, no namespace", + in: "test://", + wantNamespace: "", + wantObject: "", + wantOK: false, + }, + { + desc: "invalid, generally malformed", + in: "not even a scheme!", + wantNamespace: "", + wantObject: "", + wantOK: false, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + gotNS, gotObj, gotOK := SplitURI(testScheme, test.in) + if gotNS != test.wantNamespace || gotObj != test.wantObject || gotOK != test.wantOK { + t.Errorf("SplitURI = %q, %q, %t, want %q, %q, %t", gotNS, gotObj, gotOK, test.wantNamespace, test.wantObject, test.wantOK) + } + }) + } +} diff --git a/cmd/runner/BUILD.bazel b/cmd/runner/BUILD.bazel index 5ceddda..fe5b6e4 100644 --- a/cmd/runner/BUILD.bazel +++ b/cmd/runner/BUILD.bazel @@ -8,11 +8,13 @@ go_library( importpath = "github.com/RMI/pacta/cmd/runner", visibility = ["//visibility:private"], deps = [ + "//azure/azblob", "//azure/azlog", - "//cmd/runner/taskrunner", - "//executors/local", + "//blob", "//pacta", "//task", + "@com_github_azure_azure_sdk_for_go_sdk_azcore//:azcore", + "@com_github_azure_azure_sdk_for_go_sdk_azidentity//:azidentity", "@com_github_namsral_flag//:flag", "@org_uber_go_zap//:zap", "@org_uber_go_zap//zapcore", diff --git a/cmd/runner/main.go b/cmd/runner/main.go index 9cd706f..197e2ab 100644 --- a/cmd/runner/main.go +++ b/cmd/runner/main.go @@ -1,15 +1,24 @@ package main import ( + "bufio" + "bytes" "context" "errors" "fmt" + "io" + "io/fs" "log" "os" + "os/exec" + "path/filepath" + "strings" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/RMI/pacta/azure/azblob" "github.com/RMI/pacta/azure/azlog" - "github.com/RMI/pacta/cmd/runner/taskrunner" - "github.com/RMI/pacta/executors/local" + "github.com/RMI/pacta/blob" "github.com/RMI/pacta/pacta" "github.com/RMI/pacta/task" "github.com/namsral/flag" @@ -35,6 +44,11 @@ func run(args []string) error { var ( env = fs.String("env", "", "The environment we're running in.") + azStorageAccount = fs.String("azure_storage_account", "", "The storage account to authenticate against for blob operations") + azSourcePortfolioContainer = fs.String("azure_source_portfolio_container", "", "The container in the storage account where we read raw portfolios from") + azDestPortfolioContainer = fs.String("azure_dest_portfolio_container", "", "The container in the storage account where we read/write processed portfolios") + azReportContainer = fs.String("azure_report_container", "", "The container in the storage account where we write generated portfolio reports to") + minLogLevel zapcore.Level = zapcore.WarnLevel ) fs.Var(&minLogLevel, "min_log_level", "If set, retains logs at the given level and above. Options: 'debug', 'info', 'warn', 'error', 'dpanic', 'panic', 'fatal' - default warn.") @@ -53,16 +67,272 @@ func run(args []string) error { if err != nil { return fmt.Errorf("failed to init logger: %w", err) } + defer logger.Sync() + + var creds azcore.TokenCredential + // This is necessary because the default timeout is too low in + // azidentity.NewDefaultAzureCredentials, so it times out and fails to run. + if azClientID := os.Getenv("AZURE_CLIENT_ID"); azClientID != "" { + logger.Info("Loading user managed credentials", zap.String("client_id", azClientID)) + if creds, err = azidentity.NewManagedIdentityCredential(&azidentity.ManagedIdentityCredentialOptions{ + ID: azidentity.ClientID(azClientID), + }); err != nil { + return fmt.Errorf("failed to load Azure credentials: %w", err) + } + } else { + logger.Info("Loading default credentials") + if creds, err = azidentity.NewDefaultAzureCredential(nil); err != nil { + return fmt.Errorf("failed to load Azure credentials: %w", err) + } + } - h := taskrunner.New(&local.Executor{}, logger) + blobClient, err := azblob.NewClient(creds, *azStorageAccount) + if err != nil { + return fmt.Errorf("failed to init blob client: %w", err) + } + + h := handler{ + blob: blobClient, + sourcePortfolioContainer: *azSourcePortfolioContainer, + destPortfolioContainer: *azDestPortfolioContainer, + reportContainer: *azReportContainer, + } - portfolioID := pacta.PortfolioID(os.Getenv("PORTFOLIO_ID")) - logger.Info("running PACTA task", zap.String("portfolio_id", string(portfolioID))) - if err := h.Execute(ctx, &task.StartRunRequest{ - PortfolioID: portfolioID, - }); err != nil { - return fmt.Errorf("failed to run task: %w", err) + validTasks := map[task.Type]func(context.Context) error{ + task.ProcessPortfolio: toRunFn(processPortfolioReq, h.processPortfolio), + task.CreateReport: toRunFn(createReportReq, h.createReport), } + taskType := task.Type(os.Getenv("TASK_TYPE")) + if taskType == "" { + return errors.New("no TASK_TYPE given") + } + + taskFn, ok := validTasks[taskType] + if !ok { + return fmt.Errorf("unknown task type %q", taskType) + } + + logger.Info("running PACTA task", zap.String("task_type", string(taskType))) + + if err := taskFn(ctx); err != nil { + return fmt.Errorf("error running task: %w", err) + } + + logger.Info("ran PACTA task successfully", zap.String("task_type", string(taskType))) + return nil } + +type Blob interface { + ReadBlob(ctx context.Context, uri string) (io.ReadCloser, error) + WriteBlob(ctx context.Context, uri string, r io.Reader) error + Scheme() blob.Scheme +} + +type handler struct { + blob Blob + sourcePortfolioContainer string + destPortfolioContainer string + reportContainer string +} + +func processPortfolioReq() (*task.ProcessPortfolioRequest, error) { + pID := os.Getenv("PORTFOLIO_ID") + if pID == "" { + return nil, errors.New("no PORTFOLIO_ID was given") + } + + return &task.ProcessPortfolioRequest{ + PortfolioID: pacta.PortfolioID(pID), + }, nil +} + +func (h *handler) uploadDirectory(ctx context.Context, dirPath, container string) error { + base := filepath.Base(dirPath) + + return filepath.WalkDir(dirPath, func(path string, info fs.DirEntry, err error) error { + if info.IsDir() { + return nil + } + + // This is a file, let's upload it to the container + uri := blob.Join(h.blob.Scheme(), container, base, strings.TrimPrefix(path, dirPath)) + if err := h.uploadBlob(ctx, path, uri); err != nil { + return fmt.Errorf("failed to upload blob: %w", err) + } + return nil + }) +} + +func (h *handler) uploadBlob(ctx context.Context, srcPath, destURI string) error { + srcF, err := os.Open(srcPath) + if err != nil { + return fmt.Errorf("failed to open file for upload: %w", err) + } + defer srcF.Close() // Best-effort in case something fails + + if err := h.blob.WriteBlob(ctx, destURI, srcF); err != nil { + return fmt.Errorf("failed to write file to blob storage: %w", err) + } + + if err := srcF.Close(); err != nil { + return fmt.Errorf("failed to close source file: %w", err) + } + + return nil +} + +func (h *handler) downloadBlob(ctx context.Context, srcURI, destPath string) error { + // Make sure the destination exists + if err := os.MkdirAll(filepath.Dir(destPath), 0600); err != nil { + return fmt.Errorf("failed to create directory to download blob to: %w", err) + } + + destF, err := os.Create(destPath) + if err != nil { + return fmt.Errorf("failed to create dest file: %w", err) + } + defer destF.Close() // Best-effort in case something fails + + br, err := h.blob.ReadBlob(ctx, srcURI) + if err != nil { + return fmt.Errorf("failed to read raw portfolio: %w", err) + } + defer br.Close() // Best-effort in case something fails + + if _, err := io.Copy(destF, br); err != nil { + return fmt.Errorf("failed to load raw portfolio: %w", err) + } + + if err := br.Close(); err != nil { + return fmt.Errorf("failed to close blob reader: %w", err) + } + + if err := destF.Close(); err != nil { + return fmt.Errorf("failed to close dest file: %w", err) + } + + return nil +} + +func (h *handler) processPortfolio(ctx context.Context, req *task.ProcessPortfolioRequest) error { + baseName := string(req.PortfolioID) + ".csv" + + // Load the portfolio from blob storage, place it in /mnt/raw_portfolios, where + // the `process_portfolios.R` script expects it to be. + srcURI := blob.Join(h.blob.Scheme(), h.sourcePortfolioContainer, baseName) + destPath := filepath.Join("/", "mnt", "raw_portfolios", baseName) + + if err := h.downloadBlob(ctx, srcURI, destPath); err != nil { + return fmt.Errorf("failed to download raw portfolio blob: %w", err) + } + + processedDir := filepath.Join("/", "mnt", "processed_portfolios") + if err := os.MkdirAll(processedDir, 0600); err != nil { + return fmt.Errorf("failed to create directory to download blob to: %w", err) + } + + var stdout, stderr bytes.Buffer + cmd := exec.CommandContext(ctx, "/usr/local/bin/Rscript", "/app/process_portfolios.R") + cmd.Stdout = io.MultiWriter(os.Stdout, &stdout) + cmd.Stderr = io.MultiWriter(os.Stderr, &stderr) + + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to run process_portfolios script: %w", err) + } + + sc := bufio.NewScanner(&stderr) + + // TODO: Load from the output database file (or similar, like reading the processed_portfolios dir) instead of parsing stderr + var paths []string + for sc.Scan() { + line := sc.Text() + idx := strings.Index(line, "writing to file: " /* 17 chars */) + if idx == -1 { + continue + } + paths = append(paths, strings.TrimSpace(line[idx+17:])) + } + + // NOTE: This code could benefit from some concurrency, but I'm opting not to prematurely optimize. + for _, p := range paths { + destURI := blob.Join(h.blob.Scheme(), h.destPortfolioContainer, filepath.Base(p)) + if err := h.uploadBlob(ctx, p, destURI); err != nil { + return fmt.Errorf("failed to copy processed portfolio from %q to %q: %w", p, destURI, err) + } + } + + return nil +} + +func createReportReq() (*task.CreateReportRequest, error) { + pID := os.Getenv("PORTFOLIO_ID") + if pID == "" { + return nil, errors.New("no PORTFOLIO_ID was given") + } + + return &task.CreateReportRequest{ + PortfolioID: pacta.PortfolioID(pID), + }, nil +} + +func (h *handler) createReport(ctx context.Context, req *task.CreateReportRequest) error { + // TODO: Download portfolio JSON from Azure storage, and store it in /mnt/processed_portfolios/{id}.json + baseName := string(req.PortfolioID) + ".json" + + // Load the processed portfolio from blob storage, place it in /mnt/ + // processed_portfolios, where the `create_report.R` script expects it + // to be. + srcURI := blob.Join(h.blob.Scheme(), h.destPortfolioContainer, baseName) + destPath := filepath.Join("/", "mnt", "processed_portfolios", baseName) + + if err := h.downloadBlob(ctx, srcURI, destPath); err != nil { + return fmt.Errorf("failed to download processed portfolio blob: %w", err) + } + + reportDir := filepath.Join("/", "mnt", "reports") + if err := os.MkdirAll(reportDir, 0600); err != nil { + return fmt.Errorf("failed to create directory for reports to get copied to: %w", err) + } + + cmd := exec.CommandContext(ctx, "/usr/local/bin/Rscript", "/app/create_report.R") + cmd.Env = append(cmd.Env, + "PORTFOLIO="+string(req.PortfolioID), + "HOME=/root", /* Required by pandoc */ + ) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to run pacta test CLI: %w", err) + } + + // Download outputs from from /out and upload them to Azure + dirEntries, err := os.ReadDir(reportDir) + if err != nil { + return fmt.Errorf("failed to read report directory: %w", err) + } + + for _, dirEntry := range dirEntries { + if !dirEntry.IsDir() { + continue + } + dirPath := filepath.Join(reportDir, dirEntry.Name()) + if err := h.uploadDirectory(ctx, dirPath, h.reportContainer); err != nil { + return fmt.Errorf("failed to upload report directory: %w", err) + } + } + + return nil +} + +func toRunFn[T any](reqFn func() (T, error), runFn func(context.Context, T) error) func(context.Context) error { + return func(ctx context.Context) error { + req, err := reqFn() + if err != nil { + return fmt.Errorf("failed to format request: %w", err) + } + return runFn(ctx, req) + } +} diff --git a/cmd/runner/taskrunner/BUILD.bazel b/cmd/runner/taskrunner/BUILD.bazel index 6e9e6ee..f46a1bb 100644 --- a/cmd/runner/taskrunner/BUILD.bazel +++ b/cmd/runner/taskrunner/BUILD.bazel @@ -6,7 +6,6 @@ go_library( importpath = "github.com/RMI/pacta/cmd/runner/taskrunner", visibility = ["//visibility:public"], deps = [ - "//pacta", "//task", "@org_uber_go_zap//:zap", ], diff --git a/cmd/runner/taskrunner/taskrunner.go b/cmd/runner/taskrunner/taskrunner.go index d80d6bf..a6ca81d 100644 --- a/cmd/runner/taskrunner/taskrunner.go +++ b/cmd/runner/taskrunner/taskrunner.go @@ -5,39 +5,115 @@ package taskrunner import ( "context" + "errors" "fmt" - "github.com/RMI/pacta/pacta" "github.com/RMI/pacta/task" "go.uber.org/zap" ) -type Executor interface { - // TODO: Update this interface to include relevant inputs and outputs - ProcessPortfolio(ctx context.Context) (*pacta.PortfolioResult, error) +type Config struct { + // ConfigPath should be a full path to a config file in the runner image, + // like: /configs/{local,dev}.conf + ConfigPath string + + // BaseImage is the runner image to execute, not specifying a tag. + BaseImage *task.BaseImage + + Logger *zap.Logger + + Runner Runner } -type Handler struct { - logger *zap.Logger - executor Executor +func (c *Config) validate() error { + if c.ConfigPath == "" { + return errors.New("no runner config path given") + } + + if err := validateImage(c.BaseImage); err != nil { + return fmt.Errorf("invalid base image: %w", err) + } + + if c.Logger == nil { + return errors.New("no logger given") + } + + if c.Runner == nil { + return errors.New("no runner given") + } + + return nil } -func New(executor Executor, logger *zap.Logger) *Handler { - return &Handler{ - logger: logger, - executor: executor, +func validateImage(bi *task.BaseImage) error { + if bi.Name == "" { + return errors.New("no name given") } + if bi.Registry == "" { + return errors.New("no registry given") + } + return nil +} + +type Runner interface { + Run(ctx context.Context, cfg *task.Config) (task.ID, error) } -func (h *Handler) Execute(ctx context.Context, req *task.StartRunRequest) error { - // TODO: Add logic for loading portfolio blobs and putting them in the right places. +type TaskRunner struct { + logger *zap.Logger + runner Runner + baseImage *task.BaseImage + configPath string +} - res, err := h.executor.ProcessPortfolio(ctx) - if err != nil { - return fmt.Errorf("failed to process portfolio: %w", err) +func New(cfg *Config) (*TaskRunner, error) { + if err := cfg.validate(); err != nil { + return nil, fmt.Errorf("invalid config given: %w", err) } - h.logger.Info("processed portfolio, result", zap.Any("result", res)) + return &TaskRunner{ + logger: cfg.Logger, + runner: cfg.Runner, + baseImage: cfg.BaseImage, + configPath: cfg.ConfigPath, + }, nil +} - return nil +func (tr *TaskRunner) ProcessPortfolio(ctx context.Context, req *task.ProcessPortfolioRequest) (task.ID, error) { + return tr.run(ctx, []task.EnvVar{ + { + Key: "TASK_TYPE", + Value: string(task.ProcessPortfolio), + }, + { + Key: "PORTFOLIO_ID", + Value: string(req.PortfolioID), + }, + }) +} + +func (tr *TaskRunner) CreateReport(ctx context.Context, req *task.CreateReportRequest) (task.ID, error) { + return tr.run(ctx, []task.EnvVar{ + { + Key: "TASK_TYPE", + Value: string(task.CreateReport), + }, + { + Key: "PORTFOLIO_ID", + Value: string(req.PortfolioID), + }, + }) +} + +func (tr *TaskRunner) run(ctx context.Context, env []task.EnvVar) (task.ID, error) { + return tr.runner.Run(ctx, &task.Config{ + Env: env, + Flags: []string{"--config=" + tr.configPath}, + Command: []string{"/runner"}, + Image: &task.Image{ + Base: *tr.baseImage, + // TODO: Take in the image digest as part of the task definition, as this can change per request. + Tag: "latest", + }, + }) } diff --git a/cmd/server/BUILD.bazel b/cmd/server/BUILD.bazel index 853b6a6..f8d79d8 100644 --- a/cmd/server/BUILD.bazel +++ b/cmd/server/BUILD.bazel @@ -8,11 +8,12 @@ go_library( importpath = "github.com/RMI/pacta/cmd/server", visibility = ["//visibility:private"], deps = [ + "//azure/azblob", "//azure/aztask", "//cmd/runner/taskrunner", "//cmd/server/pactasrv", "//db/sqldb", - "//executors/docker", + "//dockertask", "//oapierr", "//openapi:pacta_generated", "//secrets", @@ -20,7 +21,6 @@ go_library( "@com_github_azure_azure_sdk_for_go_sdk_azcore//:azcore", "@com_github_azure_azure_sdk_for_go_sdk_azidentity//:azidentity", "@com_github_deepmap_oapi_codegen//pkg/chi-middleware", - "@com_github_docker_docker//client", "@com_github_go_chi_chi_v5//:chi", "@com_github_go_chi_chi_v5//middleware", "@com_github_go_chi_httprate//:httprate", diff --git a/cmd/server/configs/local.conf b/cmd/server/configs/local.conf index 2b13a54..c5fced3 100644 --- a/cmd/server/configs/local.conf +++ b/cmd/server/configs/local.conf @@ -12,6 +12,9 @@ secret_postgres_password UNUSED secret_auth_public_key_id 2023-08-11 secret_auth_public_key_data -----BEGIN PUBLIC KEY-----\nMCowBQYDK2VwAyEAP/Sv7H5TRozqXeQ2zV9W4V6Zkb/U5XWEjCQbOwAl0nc=\n-----END PUBLIC KEY----- +secret_azure_storage_account rmipactalocal +secret_azure_source_portfolio_container uploadedportfolios + secret_runner_config_location centralus secret_runner_config_config_path /configs/local.conf secret_runner_config_identity_name pacta-runner-local diff --git a/cmd/server/main.go b/cmd/server/main.go index d619352..53f89bc 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -9,23 +9,22 @@ import ( "math/rand" "net/http" "os" - "strconv" "time" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/RMI/pacta/azure/azblob" "github.com/RMI/pacta/azure/aztask" "github.com/RMI/pacta/cmd/runner/taskrunner" "github.com/RMI/pacta/cmd/server/pactasrv" "github.com/RMI/pacta/db/sqldb" - "github.com/RMI/pacta/executors/docker" + "github.com/RMI/pacta/dockertask" "github.com/RMI/pacta/oapierr" oapipacta "github.com/RMI/pacta/openapi/pacta" "github.com/RMI/pacta/secrets" "github.com/RMI/pacta/task" "github.com/Silicon-Ally/cryptorand" "github.com/Silicon-Ally/zaphttplog" - "github.com/docker/docker/client" "github.com/go-chi/chi/v5" "github.com/go-chi/httprate" "github.com/go-chi/jwtauth/v5" @@ -75,6 +74,9 @@ func run(args []string) error { authKeyID = fs.String("secret_auth_public_key_id", "", "Key ID (kid) of the JWT tokens to allow") authKeyData = fs.String("secret_auth_public_key_data", "", "PEM-encoded Ed25519 public key to verify JWT tokens with, contains literal \\n characters that will need to be replaced before parsing") + azStorageAccount = fs.String("secret_azure_storage_account", "", "The storage account to authenticate against for blob operations") + azSourcePortfolioContainer = fs.String("secret_azure_source_portfolio_container", "", "The container in the storage account where we write raw portfolios to") + runnerConfigLocation = fs.String("secret_runner_config_location", "", "Location (like 'centralus') where the runner jobs should be executed") runnerConfigConfigPath = fs.String("secret_runner_config_config_path", "", "Config path (like '/configs/dev.conf') where the runner jobs should read their base config from") @@ -110,6 +112,7 @@ func run(args []string) error { return fmt.Errorf("failed to init logger: %w", err) } } + defer logger.Sync() sec, err := secrets.LoadPACTA(&secrets.RawPACTAConfig{ PostgresConfig: &secrets.RawPostgresConfig{ @@ -198,13 +201,23 @@ func run(args []string) error { } } - var taskRunner pactasrv.TaskRunner + // What is PACTA portfolio running? A two-step process: + // - ProcessPortfolio -> Produces a bunch of JSON files + // - CreateReport -> A single JSON in, an HTML report out + + // And how do we plan on running it? + // - Locally + // - Against local machine: Use Docker API to spin up a Docker container and run the task + // - Against Azure: Use Azure API to spin up a Container Apps Job and run the task + // - On Azure + // - Spin up a Container Apps Job and run it + + var runner taskrunner.Runner if *useAZRunner { logger.Info("initializing Azure task runner client") - tmp, err := aztask.NewTaskRunner(creds, &aztask.Config{ - Location: runCfg.Location, - ConfigPath: runCfg.ConfigPath, - Rand: rand.New(cryptorand.New()), + tmp, err := aztask.NewRunner(creds, &aztask.Config{ + Location: runCfg.Location, + Rand: rand.New(cryptorand.New()), Identity: &aztask.RunnerIdentity{ Name: runCfg.Identity.Name, SubscriptionID: runCfg.Identity.SubscriptionID, @@ -212,31 +225,43 @@ func run(args []string) error { ClientID: runCfg.Identity.ClientID, ManagedEnvironment: runCfg.Identity.ManagedEnvironment, }, - Image: &aztask.RunnerImage{ - Registry: runCfg.Image.Registry, - Name: runCfg.Image.Name, - }, }) if err != nil { - return fmt.Errorf("failed to init task runner: %w", err) + return fmt.Errorf("failed to init Azure runner: %w", err) } - taskRunner = tmp + runner = tmp } else { - cli, err := client.NewEnvClient() + tmp, err := dockertask.NewRunner(logger) if err != nil { - return fmt.Errorf("failed to initialize Docker client: %w", err) + return fmt.Errorf("failed to init docker runner: %w", err) } + runner = tmp + } - dockerExec := docker.NewExecutor(cli, logger) - taskRunner = &localRunner{ - handler: taskrunner.New(dockerExec, logger), - } + tr, err := taskrunner.New(&taskrunner.Config{ + ConfigPath: runCfg.ConfigPath, + BaseImage: &task.BaseImage{ + Registry: runCfg.Image.Registry, + Name: runCfg.Image.Name, + }, + Logger: logger, + Runner: runner, + }) + if err != nil { + return fmt.Errorf("failed to init task runner: %w", err) + } + + blobClient, err := azblob.NewClient(creds, *azStorageAccount) + if err != nil { + return fmt.Errorf("failed to init blob client: %w", err) } // Create an instance of our handler which satisfies each generated interface srv := &pactasrv.Server{ - DB: db, - TaskRunner: taskRunner, + Blob: blobClient, + PorfolioUploadURI: *azSourcePortfolioContainer, + DB: db, + TaskRunner: tr, } pactaStrictHandler := oapipacta.NewStrictHandlerWithOptions(srv, nil /* middleware */, oapipacta.StrictHTTPServerOptions{ @@ -370,21 +395,3 @@ func responseErrorHandlerFuncForService(logger *zap.Logger, svc string) func(w h http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) } } - -// localRunner is a thin shim around *taskrunner.Handler that satisfies the -// task runner interface, allowing us to run the handler in-process, usually for -// local development. -type localRunner struct { - handler *taskrunner.Handler - - taskID int -} - -func (l *localRunner) StartRun(ctx context.Context, req *task.StartRunRequest) (task.ID, error) { - if err := l.handler.Execute(ctx, req); err != nil { - return "", fmt.Errorf("error from handler: %w", err) - } - - l.taskID++ - return task.ID("task:" + strconv.Itoa(l.taskID)), nil -} diff --git a/cmd/server/pactasrv/BUILD.bazel b/cmd/server/pactasrv/BUILD.bazel index 86288b4..4ee9a4d 100644 --- a/cmd/server/pactasrv/BUILD.bazel +++ b/cmd/server/pactasrv/BUILD.bazel @@ -6,11 +6,13 @@ go_library( "initiative.go", "pacta_version.go", "pactasrv.go", + "portfolio.go", "user.go", ], importpath = "github.com/RMI/pacta/cmd/server/pactasrv", visibility = ["//visibility:public"], deps = [ + "//blob", "//cmd/server/pactasrv/conv", "//db", "//oapierr", @@ -18,6 +20,7 @@ go_library( "//pacta", "//task", "@com_github_go_chi_jwtauth_v5//:jwtauth", + "@com_github_google_uuid//:uuid", "@org_uber_go_zap//:zap", ], ) diff --git a/cmd/server/pactasrv/pactasrv.go b/cmd/server/pactasrv/pactasrv.go index 9e6bf6b..6870180 100644 --- a/cmd/server/pactasrv/pactasrv.go +++ b/cmd/server/pactasrv/pactasrv.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "github.com/RMI/pacta/blob" "github.com/RMI/pacta/db" "github.com/RMI/pacta/oapierr" "github.com/RMI/pacta/pacta" @@ -17,7 +18,8 @@ var ( ) type TaskRunner interface { - StartRun(context.Context, *task.StartRunRequest) (task.ID, error) + ProcessPortfolio(ctx context.Context, req *task.ProcessPortfolioRequest) (task.ID, error) + CreateReport(ctx context.Context, req *task.CreateReportRequest) (task.ID, error) } type DB interface { @@ -70,9 +72,21 @@ type DB interface { DeleteUser(tx db.Tx, id pacta.UserID) error } +type Blob interface { + Scheme() blob.Scheme + + // For uploading portfolios + SignedUploadURL(ctx context.Context, uri string) (string, error) + // For downloading reports + SignedDownloadURL(ctx context.Context, uri string) (string, error) +} + type Server struct { DB DB TaskRunner TaskRunner + + Blob Blob + PorfolioUploadURI string } func mapAll[I any, O any](is []I, f func(I) (O, error)) ([]O, error) { diff --git a/cmd/server/pactasrv/portfolio.go b/cmd/server/pactasrv/portfolio.go new file mode 100644 index 0000000..5db0baa --- /dev/null +++ b/cmd/server/pactasrv/portfolio.go @@ -0,0 +1,22 @@ +package pactasrv + +import ( + "context" + + "github.com/RMI/pacta/blob" + "github.com/RMI/pacta/oapierr" + api "github.com/RMI/pacta/openapi/pacta" + "github.com/google/uuid" + "go.uber.org/zap" +) + +func (s *Server) CreatePortfolioAsset(ctx context.Context, req api.CreatePortfolioAssetRequestObject) (api.CreatePortfolioAssetResponseObject, error) { + uri := blob.Join(s.Blob.Scheme(), s.PorfolioUploadURI, uuid.NewString()) + signed, err := s.Blob.SignedUploadURL(ctx, uri) + if err != nil { + return nil, oapierr.Internal("failed to sign blob URI", zap.String("uri", uri), zap.Error(err)) + } + return api.CreatePortfolioAsset200JSONResponse{ + UploadUrl: signed, + }, nil +} diff --git a/executors/docker/BUILD.bazel b/dockertask/BUILD.bazel similarity index 68% rename from executors/docker/BUILD.bazel rename to dockertask/BUILD.bazel index 352d1d7..cca1e24 100644 --- a/executors/docker/BUILD.bazel +++ b/dockertask/BUILD.bazel @@ -1,16 +1,15 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( - name = "docker", - srcs = ["docker.go"], - importpath = "github.com/RMI/pacta/executors/docker", + name = "dockertask", + srcs = ["dockertask.go"], + importpath = "github.com/RMI/pacta/dockertask", visibility = ["//visibility:public"], deps = [ - "//pacta", + "//task", "@com_github_docker_docker//api/types", "@com_github_docker_docker//api/types/container", "@com_github_docker_docker//client", - "@com_github_docker_docker//pkg/stdcopy", "@com_github_opencontainers_image_spec//specs-go/v1:specs-go", "@org_uber_go_zap//:zap", ], diff --git a/dockertask/dockertask.go b/dockertask/dockertask.go new file mode 100644 index 0000000..cf66e5d --- /dev/null +++ b/dockertask/dockertask.go @@ -0,0 +1,110 @@ +// Package dockertask implements PACTA processing in a container against a +// local Docker daemon. This implementation is used for local testing, where +// one likely doesn't have a functional portfolio-handling environment on the +// host machine. +package dockertask + +import ( + "context" + "fmt" + + "github.com/RMI/pacta/task" + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/client" + "go.uber.org/zap" + + specs "github.com/opencontainers/image-spec/specs-go/v1" +) + +type Runner struct { + client *client.Client + logger *zap.Logger +} + +func NewRunner(logger *zap.Logger) (*Runner, error) { + cli, err := client.NewEnvClient() + if err != nil { + return nil, fmt.Errorf("failed to initialize Docker client: %w", err) + } + + return &Runner{client: cli, logger: logger}, nil +} + +func (r *Runner) Run(ctx context.Context, taskCfg *task.Config) (task.ID, error) { + var env []string + for _, e := range taskCfg.Env { + env = append(env, e.Key+"="+e.Value) + } + cfg := &container.Config{ + Image: taskCfg.Image.String(), + + // Run the script, tell it to output data to our mounted location. + Cmd: append(taskCfg.Command, taskCfg.Flags...), + + Env: env, + + AttachStdin: false, + Tty: false, + } + + hostCfg := &container.HostConfig{ + // TODO: Add relevant inputs + outputs, likely as mounts + } + platform := &specs.Platform{ + Architecture: "amd64", + OS: "linux", + } + + resp, err := r.client.ContainerCreate(ctx, cfg, hostCfg, nil /* net config */, platform, "" /* random name */) + if err != nil { + return "", fmt.Errorf("failed to create PACTA container: %w", err) + } + defer func() { + err := r.client.ContainerRemove(ctx, resp.ID, types.ContainerRemoveOptions{ + RemoveVolumes: true, + RemoveLinks: false, + Force: true, + }) + if err != nil { + r.logger.Error("failed to clean up container", + zap.String("container_id", resp.ID), + zap.Error(err)) + } + }() + + if err := r.client.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil { + return "", fmt.Errorf("failed to start PACTA container: %w", err) + } + + waitC, errC := r.client.ContainerWait(ctx, resp.ID, container.WaitConditionNotRunning) + + select { + case resp := <-waitC: + if resp.Error != nil { + return "", fmt.Errorf("error in container wait response: %+v", resp.Error) + } + case err := <-errC: + return "", fmt.Errorf("error while waiting for container to complete: %w", err) + } + + // If we're here, container exited successfully, let's load the logs. + // logRC, err := r.client.ContainerLogs(ctx, resp.ID, types.ContainerLogsOptions{ + // ShowStdout: true, + // ShowStderr: true, + // Tail: "all", + // Details: true, + // }) + // if err != nil { + // return "", fmt.Errorf("failed to read container logs: %w", err) + // } + // defer func() { + // if err := logRC.Close(); err != nil { + // r.logger.Warn("failed to close contrainer log reader", + // zap.String("container_id", resp.ID), + // zap.Error(err)) + // } + // }() + + return task.ID(resp.ID), nil +} diff --git a/executors/docker/docker.go b/executors/docker/docker.go deleted file mode 100644 index 3f240c9..0000000 --- a/executors/docker/docker.go +++ /dev/null @@ -1,110 +0,0 @@ -// Package docker implements an executor that runs PACTA processing in a -// container against a local Docker daemon. This implementation is used for -// local testing, where one likely doesn't have a functional portfolio-handling -// environment on the host machine. -package docker - -import ( - "bytes" - "context" - "fmt" - - "github.com/RMI/pacta/pacta" - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" - "github.com/docker/docker/client" - "github.com/docker/docker/pkg/stdcopy" - "go.uber.org/zap" - - specs "github.com/opencontainers/image-spec/specs-go/v1" -) - -type Executor struct { - client *client.Client - logger *zap.Logger -} - -func NewExecutor(dockerClient *client.Client, logger *zap.Logger) *Executor { - return &Executor{client: dockerClient, logger: logger} -} - -func (e *Executor) ProcessPortfolio(ctx context.Context) (*pacta.PortfolioResult, error) { - cfg := &container.Config{ - // Use our runner image, which should contain PACTA processing infra. - Image: "rmipacta.azurecr.io/runner:latest", - - // Run the script, tell it to output data to our mounted location. - Cmd: []string{"echo", "TODO, the command to execute"}, - - AttachStdin: false, - Tty: false, - } - - hostCfg := &container.HostConfig{ - // TODO: Add relevant inputs + outputs, likely as mounts - } - platform := &specs.Platform{ - Architecture: "amd64", - OS: "linux", - } - - resp, err := e.client.ContainerCreate(ctx, cfg, hostCfg, nil /* net config */, platform, "" /* random name */) - if err != nil { - return nil, fmt.Errorf("failed to create PACTA container: %w", err) - } - defer func() { - err := e.client.ContainerRemove(ctx, resp.ID, types.ContainerRemoveOptions{ - RemoveVolumes: true, - RemoveLinks: false, - Force: true, - }) - if err != nil { - e.logger.Error("failed to clean up container", - zap.String("container_id", resp.ID), - zap.Error(err)) - } - }() - - if err := e.client.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil { - return nil, fmt.Errorf("failed to start PACTA container: %w", err) - } - - waitC, errC := e.client.ContainerWait(ctx, resp.ID, container.WaitConditionNotRunning) - - select { - case resp := <-waitC: - if resp.Error != nil { - return nil, fmt.Errorf("error in container wait response: %+v", resp.Error) - } - case err := <-errC: - return nil, fmt.Errorf("error while waiting for container to complete: %w", err) - } - - // If we're here, container exited successfully, let's load the logs. - logRC, err := e.client.ContainerLogs(ctx, resp.ID, types.ContainerLogsOptions{ - ShowStdout: true, - ShowStderr: true, - Tail: "all", - Details: true, - }) - if err != nil { - return nil, fmt.Errorf("failed to read container logs: %w", err) - } - defer func() { - if err := logRC.Close(); err != nil { - e.logger.Warn("failed to close contrainer log reader", - zap.String("container_id", resp.ID), - zap.Error(err)) - } - }() - - var stdout, stderr bytes.Buffer - if _, err := stdcopy.StdCopy(&stdout, &stderr, logRC); err != nil { - return nil, fmt.Errorf("failed to read logs: %w", err) - } - - return &pacta.PortfolioResult{ - Stdout: stdout.String(), - Stderr: stderr.String(), - }, nil -} diff --git a/executors/local/BUILD.bazel b/executors/local/BUILD.bazel deleted file mode 100644 index 43e0ad6..0000000 --- a/executors/local/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") - -go_library( - name = "local", - srcs = ["local.go"], - importpath = "github.com/RMI/pacta/executors/local", - visibility = ["//visibility:public"], - deps = ["//pacta"], -) diff --git a/executors/local/local.go b/executors/local/local.go deleted file mode 100644 index 9153e29..0000000 --- a/executors/local/local.go +++ /dev/null @@ -1,33 +0,0 @@ -// Package local implements an executor that runs PACTA processing directly -// on the current machine via os/exec. This implementation is used in deployed -// environments, where the runner binary will be baked into a Docker image -// with the portfolio processing code, and so has local access to a functioning -// portfolio-handling environment. -package local - -import ( - "bytes" - "context" - "fmt" - "os/exec" - - "github.com/RMI/pacta/pacta" -) - -type Executor struct{} - -func (e *Executor) ProcessPortfolio(ctx context.Context) (*pacta.PortfolioResult, error) { - var stdout, stderr bytes.Buffer - cmd := exec.CommandContext(ctx, "echo", "TODO, the command to execute") - cmd.Stdout = &stdout - cmd.Stderr = &stderr - - if err := cmd.Run(); err != nil { - return nil, fmt.Errorf("failed to run pacta test CLI: %w", err) - } - - return &pacta.PortfolioResult{ - Stdout: stdout.String(), - Stderr: stderr.String(), - }, nil -} diff --git a/frontend/.nvmrc b/frontend/.nvmrc new file mode 100644 index 0000000..4a58985 --- /dev/null +++ b/frontend/.nvmrc @@ -0,0 +1 @@ +18.18 diff --git a/frontend/openapi/generated/pacta/index.ts b/frontend/openapi/generated/pacta/index.ts index e2ff504..d39e62f 100644 --- a/frontend/openapi/generated/pacta/index.ts +++ b/frontend/openapi/generated/pacta/index.ts @@ -15,6 +15,7 @@ export { Initiative } from './models/Initiative'; export { InitiativeChanges } from './models/InitiativeChanges'; export { InitiativeCreate } from './models/InitiativeCreate'; export { Language } from './models/Language'; +export type { NewPortfolioAsset } from './models/NewPortfolioAsset'; export type { PactaVersion } from './models/PactaVersion'; export type { PactaVersionChanges } from './models/PactaVersionChanges'; export type { PactaVersionCreate } from './models/PactaVersionCreate'; diff --git a/frontend/openapi/generated/pacta/models/NewPortfolioAsset.ts b/frontend/openapi/generated/pacta/models/NewPortfolioAsset.ts new file mode 100644 index 0000000..7633b12 --- /dev/null +++ b/frontend/openapi/generated/pacta/models/NewPortfolioAsset.ts @@ -0,0 +1,12 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +export type NewPortfolioAsset = { + /** + * The signed URL where the file should be uploaded to, using PUT semantics. + */ + upload_url: string; +}; + diff --git a/frontend/openapi/generated/pacta/services/DefaultService.ts b/frontend/openapi/generated/pacta/services/DefaultService.ts index 832bf1d..8365463 100644 --- a/frontend/openapi/generated/pacta/services/DefaultService.ts +++ b/frontend/openapi/generated/pacta/services/DefaultService.ts @@ -5,6 +5,7 @@ import type { Initiative } from '../models/Initiative'; import type { InitiativeChanges } from '../models/InitiativeChanges'; import type { InitiativeCreate } from '../models/InitiativeCreate'; +import type { NewPortfolioAsset } from '../models/NewPortfolioAsset'; import type { PactaVersion } from '../models/PactaVersion'; import type { PactaVersionChanges } from '../models/PactaVersionChanges'; import type { PactaVersionCreate } from '../models/PactaVersionCreate'; @@ -290,4 +291,20 @@ export class DefaultService { }); } + /** + * Test endpoint, creates a new portfolio asset + * Creates a new asset for a portfolio + * + * Returns a signed URL where the portfolio can be uploaded to. + * + * @returns NewPortfolioAsset The asset can now be uploaded via the given signed URL. + * @throws ApiError + */ + public createPortfolioAsset(): CancelablePromise { + return this.httpRequest.request({ + method: 'POST', + url: '/test:createPortfolioAsset', + }); + } + } diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 6964d9f..4722b88 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -7,6 +7,7 @@ "name": "nuxt-app", "dependencies": { "@azure/msal-browser": "^3.2.0", + "axios": "^1.5.1", "primeflex": "^3.3.1", "primeicons": "^6.0.1", "primevue": "^3.32.0", @@ -23,7 +24,6 @@ "eslint": "^8.51.0", "eslint-config-standard-with-typescript": "^38.0.0", "eslint-plugin-import": "^2.28.1", - "eslint-plugin-json": "^3.1.0", "eslint-plugin-n": "^16.0.2", "eslint-plugin-promise": "^6.1.1", "eslint-plugin-vue": "^9.17.0", @@ -5417,8 +5417,7 @@ "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "dev": true + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" }, "node_modules/autoprefixer": { "version": "10.4.14", @@ -5466,13 +5465,13 @@ } }, "node_modules/axios": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.27.2.tgz", - "integrity": "sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==", - "dev": true, + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.5.1.tgz", + "integrity": "sha512-Q28iYCWzNHjAm+yEAot5QaAMxhMghWLFVf7rRdwhUI+c2jix2DUXjAHXVi+s1ibs3mjPO/cCgbA++3BjD0vP/A==", "dependencies": { - "follow-redirects": "^1.14.9", - "form-data": "^4.0.0" + "follow-redirects": "^1.15.0", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" } }, "node_modules/balanced-match": { @@ -6462,7 +6461,6 @@ "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "dev": true, "dependencies": { "delayed-stream": "~1.0.0" }, @@ -7462,7 +7460,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "dev": true, "engines": { "node": ">=0.4.0" } @@ -8364,19 +8361,6 @@ "semver": "bin/semver.js" } }, - "node_modules/eslint-plugin-json": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/eslint-plugin-json/-/eslint-plugin-json-3.1.0.tgz", - "integrity": "sha512-MrlG2ynFEHe7wDGwbUuFPsaT2b1uhuEFhJ+W1f1u+1C2EkXmTYJp4B1aAdQQ8M+CC3t//N/oRKiIVw14L2HR1g==", - "dev": true, - "dependencies": { - "lodash": "^4.17.21", - "vscode-json-languageservice": "^4.1.6" - }, - "engines": { - "node": ">=12.0" - } - }, "node_modules/eslint-plugin-n": { "version": "16.0.2", "resolved": "https://registry.npmjs.org/eslint-plugin-n/-/eslint-plugin-n-16.0.2.tgz", @@ -9013,7 +8997,6 @@ "version": "1.15.2", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==", - "dev": true, "funding": [ { "type": "individual", @@ -9070,7 +9053,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", - "dev": true, "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", @@ -11922,7 +11904,6 @@ "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "dev": true, "engines": { "node": ">= 0.6" } @@ -11931,7 +11912,6 @@ "version": "2.1.35", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "dev": true, "dependencies": { "mime-db": "1.52.0" }, @@ -14571,6 +14551,11 @@ "integrity": "sha512-/XJ368cyBJ7fzLMwLKv1e4vLxOju2MNAIokcr7meSaNcVbWz/CPcW22cP04mwxOErdA5mwjA8Q6w/cdAQxVn7Q==", "dev": true }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, "node_modules/prr": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz", @@ -18335,19 +18320,6 @@ "vite": "^3.0.0-0 || ^4.0.0-0" } }, - "node_modules/vscode-json-languageservice": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/vscode-json-languageservice/-/vscode-json-languageservice-4.2.1.tgz", - "integrity": "sha512-xGmv9QIWs2H8obGbWg+sIPI/3/pFgj/5OWBhNzs00BkYQ9UaB2F6JJaGB/2/YOZJ3BvLXQTC4Q7muqU25QgAhA==", - "dev": true, - "dependencies": { - "jsonc-parser": "^3.0.0", - "vscode-languageserver-textdocument": "^1.0.3", - "vscode-languageserver-types": "^3.16.0", - "vscode-nls": "^5.0.0", - "vscode-uri": "^3.0.3" - } - }, "node_modules/vscode-jsonrpc": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-6.0.0.tgz", @@ -18427,12 +18399,6 @@ "integrity": "sha512-k8luDIWJWyenLc5ToFQQMaSrqCHiLwyKPHKPQZ5zz21vM+vIVUSvsRpcbiECH4WR88K2XZqc4ScRcZ7nk/jbeA==", "dev": true }, - "node_modules/vscode-nls": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/vscode-nls/-/vscode-nls-5.2.0.tgz", - "integrity": "sha512-RAaHx7B14ZU04EU31pT+rKz2/zSl7xMsfIZuo8pd+KZO6PXtQmpevpq3vxvWNcrGbdmhM/rr5Uw5Mz+NBfhVng==", - "dev": true - }, "node_modules/vscode-uri": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/vscode-uri/-/vscode-uri-3.0.7.tgz", @@ -18570,6 +18536,16 @@ "node": ">=12.0.0" } }, + "node_modules/wait-on/node_modules/axios": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.27.2.tgz", + "integrity": "sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==", + "dev": true, + "dependencies": { + "follow-redirects": "^1.14.9", + "form-data": "^4.0.0" + } + }, "node_modules/wcwidth": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz", @@ -22687,8 +22663,7 @@ "asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "dev": true + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" }, "autoprefixer": { "version": "10.4.14", @@ -22711,13 +22686,13 @@ "dev": true }, "axios": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.27.2.tgz", - "integrity": "sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==", - "dev": true, + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.5.1.tgz", + "integrity": "sha512-Q28iYCWzNHjAm+yEAot5QaAMxhMghWLFVf7rRdwhUI+c2jix2DUXjAHXVi+s1ibs3mjPO/cCgbA++3BjD0vP/A==", "requires": { - "follow-redirects": "^1.14.9", - "form-data": "^4.0.0" + "follow-redirects": "^1.15.0", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" } }, "balanced-match": { @@ -23434,7 +23409,6 @@ "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "dev": true, "requires": { "delayed-stream": "~1.0.0" } @@ -24174,8 +24148,7 @@ "delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "dev": true + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==" }, "delegates": { "version": "1.0.0", @@ -24974,16 +24947,6 @@ } } }, - "eslint-plugin-json": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/eslint-plugin-json/-/eslint-plugin-json-3.1.0.tgz", - "integrity": "sha512-MrlG2ynFEHe7wDGwbUuFPsaT2b1uhuEFhJ+W1f1u+1C2EkXmTYJp4B1aAdQQ8M+CC3t//N/oRKiIVw14L2HR1g==", - "dev": true, - "requires": { - "lodash": "^4.17.21", - "vscode-json-languageservice": "^4.1.6" - } - }, "eslint-plugin-n": { "version": "16.0.2", "resolved": "https://registry.npmjs.org/eslint-plugin-n/-/eslint-plugin-n-16.0.2.tgz", @@ -25355,8 +25318,7 @@ "follow-redirects": { "version": "1.15.2", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", - "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==", - "dev": true + "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==" }, "for-each": { "version": "0.3.3", @@ -25389,7 +25351,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", - "dev": true, "requires": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", @@ -27582,14 +27543,12 @@ "mime-db": { "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "dev": true + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==" }, "mime-types": { "version": "2.1.35", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "dev": true, "requires": { "mime-db": "1.52.0" } @@ -29513,6 +29472,11 @@ "integrity": "sha512-/XJ368cyBJ7fzLMwLKv1e4vLxOju2MNAIokcr7meSaNcVbWz/CPcW22cP04mwxOErdA5mwjA8Q6w/cdAQxVn7Q==", "dev": true }, + "proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, "prr": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz", @@ -32239,19 +32203,6 @@ "shell-quote": "^1.8.0" } }, - "vscode-json-languageservice": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/vscode-json-languageservice/-/vscode-json-languageservice-4.2.1.tgz", - "integrity": "sha512-xGmv9QIWs2H8obGbWg+sIPI/3/pFgj/5OWBhNzs00BkYQ9UaB2F6JJaGB/2/YOZJ3BvLXQTC4Q7muqU25QgAhA==", - "dev": true, - "requires": { - "jsonc-parser": "^3.0.0", - "vscode-languageserver-textdocument": "^1.0.3", - "vscode-languageserver-types": "^3.16.0", - "vscode-nls": "^5.0.0", - "vscode-uri": "^3.0.3" - } - }, "vscode-jsonrpc": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-6.0.0.tgz", @@ -32321,12 +32272,6 @@ "integrity": "sha512-k8luDIWJWyenLc5ToFQQMaSrqCHiLwyKPHKPQZ5zz21vM+vIVUSvsRpcbiECH4WR88K2XZqc4ScRcZ7nk/jbeA==", "dev": true }, - "vscode-nls": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/vscode-nls/-/vscode-nls-5.2.0.tgz", - "integrity": "sha512-RAaHx7B14ZU04EU31pT+rKz2/zSl7xMsfIZuo8pd+KZO6PXtQmpevpq3vxvWNcrGbdmhM/rr5Uw5Mz+NBfhVng==", - "dev": true - }, "vscode-uri": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/vscode-uri/-/vscode-uri-3.0.7.tgz", @@ -32426,6 +32371,18 @@ "lodash": "^4.17.21", "minimist": "^1.2.7", "rxjs": "^7.8.0" + }, + "dependencies": { + "axios": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.27.2.tgz", + "integrity": "sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==", + "dev": true, + "requires": { + "follow-redirects": "^1.14.9", + "form-data": "^4.0.0" + } + } } }, "wcwidth": { diff --git a/frontend/package.json b/frontend/package.json index 2442f55..11e6188 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -40,6 +40,7 @@ }, "dependencies": { "@azure/msal-browser": "^3.2.0", + "axios": "^1.5.1", "primeflex": "^3.3.1", "primeicons": "^6.0.1", "primevue": "^3.32.0", diff --git a/frontend/pages/admin/index.vue b/frontend/pages/admin/index.vue index fc2e2e9..4f6e95a 100644 --- a/frontend/pages/admin/index.vue +++ b/frontend/pages/admin/index.vue @@ -21,6 +21,12 @@ const adminItems: AdminItem[] = [ desc: 'Create, update + manage Initiatives', href: '/admin/initiative', }, + { + title: 'Test Portfolio Processing', + icon: 'pi pi-file-o', + desc: 'Test out portfolio processing with an uploaded portfolio', + href: '/admin/portfolio_test' + }, ] diff --git a/frontend/pages/admin/portfolio_test.vue b/frontend/pages/admin/portfolio_test.vue new file mode 100644 index 0000000..1d166b8 --- /dev/null +++ b/frontend/pages/admin/portfolio_test.vue @@ -0,0 +1,72 @@ + + + diff --git a/frontend/plugins/axios.ts b/frontend/plugins/axios.ts new file mode 100644 index 0000000..bbfd380 --- /dev/null +++ b/frontend/plugins/axios.ts @@ -0,0 +1,9 @@ +import axios from 'axios' + +export default defineNuxtPlugin(() => { + return { + provide: { + axios: axios.create(), + }, + } +}) diff --git a/go.mod b/go.mod index f6b5d01..0cb6dcf 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appcontainers/armappcontainers/v2 v2.0.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 github.com/Silicon-Ally/cryptorand v1.0.1 github.com/Silicon-Ally/idgen v1.0.1 github.com/Silicon-Ally/testpgx v0.0.4 diff --git a/go.sum b/go.sum index dc45895..2b5b5ae 100644 --- a/go.sum +++ b/go.sum @@ -19,6 +19,8 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInm github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appcontainers/armappcontainers/v2 v2.0.0 h1:1PD0CnFSl1m1TCwudP3cIiyTABCWVzHXtYc6Vi5J0JY= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appcontainers/armappcontainers/v2 v2.0.0/go.mod h1:xCGT95xV5ei4ahSgJWy31pPGE3xWfaWpr9uRzwTzsmg= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 h1:u/LLAOFgsMv7HmNL4Qufg58y+qElGOt5qv0z1mURkRY= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0/go.mod h1:2e8rMJtl2+2j+HXbTBwnyGpm5Nou7KhvSfxOq8JpTag= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 h1:WpB/QDNLpMw72xHJc34BNNykqSOeEJDAWkhf0u12/Jk= github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= diff --git a/openapi/pacta.yaml b/openapi/pacta.yaml index 7f0ee1f..9190663 100644 --- a/openapi/pacta.yaml +++ b/openapi/pacta.yaml @@ -276,6 +276,21 @@ paths: responses: '204': description: user deleted + /test:createPortfolioAsset: + post: + summary: Test endpoint, creates a new portfolio asset + description: | + Creates a new asset for a portfolio + + Returns a signed URL where the portfolio can be uploaded to. + operationId: createPortfolioAsset + responses: + '200': + description: The asset can now be uploaded via the given signed URL. + content: + application/json: + schema: + $ref: '#/components/schemas/NewPortfolioAsset' components: responses: @@ -559,6 +574,15 @@ components: type: boolean description: Whether the given user is a super admin + NewPortfolioAsset: + type: object + required: + - upload_url + properties: + upload_url: + type: string + description: The signed URL where the file should be uploaded to, using PUT semantics. + Error: type: object required: diff --git a/pacta/pacta.go b/pacta/pacta.go index 54d3f17..44144f6 100644 --- a/pacta/pacta.go +++ b/pacta/pacta.go @@ -705,9 +705,3 @@ func cloneAll[T cloneable[T]](in []T) []T { } return out } - -// TODO: Populate with relevant output from portfolio analysis -type PortfolioResult struct { - Stdout string - Stderr string -} diff --git a/scripts/run_db.sh b/scripts/run_db.sh index 1b7a095..ab32abc 100755 --- a/scripts/run_db.sh +++ b/scripts/run_db.sh @@ -65,6 +65,9 @@ chmod 766 "$SOCKET_DIR" # appropriately. set_kv "SOCKET_DIR" "$SOCKET_DIR/sub" +# Needed for rootless Podman setups +mkdir -p "$SOCKET_DIR/sub" + # We turn off ports and listen solely on our Unix socket. docker run \ --name local-postgres \ diff --git a/task/task.go b/task/task.go index 3f131cf..d4d27ba 100644 --- a/task/task.go +++ b/task/task.go @@ -1,12 +1,66 @@ // Package task holds domain types for asynchronous PACTA work, like analyzing profiles package task -import "github.com/RMI/pacta/pacta" +import ( + "bytes" + + "github.com/RMI/pacta/pacta" +) // TaskID uniquely identifies a task being processed. type ID string -type StartRunRequest struct { - // This is just an example, this will likely change over time. +type Type string + +const ( + ProcessPortfolio = Type("process_portfolio") + CreateReport = Type("create_report") +) + +type ProcessPortfolioRequest struct { PortfolioID pacta.PortfolioID } + +type CreateReportRequest struct { + PortfolioID pacta.PortfolioID +} + +type EnvVar struct { + Key string + Value string +} + +type BaseImage struct { + // Like 'rmipacta.azurecr.io' + Registry string + // Like 'runner' + Name string +} + +type Image struct { + Base BaseImage + // Like 'latest' + Tag string +} + +func (i *BaseImage) WithTag(tag string) string { + var buf bytes.Buffer + // /: + buf.WriteString(i.Registry) + buf.WriteRune('/') + buf.WriteString(i.Name) + buf.WriteRune(':') + buf.WriteString(tag) + return buf.String() +} + +func (i *Image) String() string { + return i.Base.WithTag(i.Tag) +} + +type Config struct { + Env []EnvVar + Image *Image + Flags []string + Command []string +}