Get the create-report async task in shape

cmd/runner/README.md

@@ -15,11 +15,18 @@ bazel run //scripts:run_server -- --use_azure_runner
 
 ### Creating a new docker image to run locally
 
+When developing the runner, you have two options:
+
+* **Test against local Docker** - Run the server **without** the  `--use_azure_runner`, which means async tasks will run locally, using `docker run ...`. To test local runner changes, you can build and tag a runner image locally with `bazel run //scripts:build_and_load_runner`.
+  * After running the script, the updated runner will immediately be available, no need to restart the server.
+  * This is the option you'll want to use most of the time.
+* **Test against Azure Container Apps Jobs** - Run the server **with** the  `--use_azure_runner`, which means async tasks will be run on Azure, created via the Azure API. To test changes here, you can build and tag a runner image locally with `bazel run //scripts:build_and_load_runner`, and then push it to Azure with `docker push rmisa.azurecr.io/runner:latest`
+  * You generally won't need to use this option unless you're testing something very specific about the runner's integration with Azure, as the runner code is identical whether run locally or on Azure.
+
+### Cleaning up old runner containers
+
+By default, we don't auto-remove stopped containers (i.e. finished runner tasks), to give developers a chance to review the logs (e.g. with `docker logs <sha>`). To clean up all completed runs at once, run:
+
 ```bash
-# Build the runner binary
-bazel build --@io_bazel_rules_go//go/config:pure //cmd/runner:image_tarball
-# Load the new image into docker, which will output a SHA256 value
-docker load < bazel-bin/cmd/runner/image_tarball/tarball.tar
-# Tag the runner image in order for it to be picked up locally. Don't push this to the registry!
-docker tag <SHA from previous step> rmisa.azurecr.io/runner
-```
+docker rm $(docker ps -a -q -f "status=exited" -f "ancestor=rmisa.azurecr.io/runner:latest")
+```

cmd/runner/main.go

@@ -180,24 +180,68 @@ func parsePortfolioReq() (*task.ParsePortfolioRequest, error) {
 	return &task, nil
 }
 
-func (h *handler) uploadDirectory(ctx context.Context, dirPath, container string) error {
+func (h *handler) uploadDirectory(ctx context.Context, dirPath, container string) ([]*task.AnalysisArtifact, error) {
 	base := filepath.Base(dirPath)
 
-	return filepath.WalkDir(dirPath, func(path string, info fs.DirEntry, err error) error {
+	var artifacts []*task.AnalysisArtifact
+	err := filepath.WalkDir(dirPath, func(path string, info fs.DirEntry, err error) error {
 		if info.IsDir() {
 			return nil
 		}
 
 		// This is a file, let's upload it to the container
-		uri := blob.Join(h.blob.Scheme(), container, base, strings.TrimPrefix(path, dirPath))
+		uri := blob.Join(h.blob.Scheme(), container, base, strings.TrimPrefix(path, dirPath+"/"))
 		if err := h.uploadBlob(ctx, path, uri); err != nil {
 			return fmt.Errorf("failed to upload blob: %w", err)
 		}
+
+		fn := filepath.Base(path)
+		// Returns pacta.FileType_UNKNOWN for unrecognized extensions, which we'll serve as binary blobs.
+		ft := fileTypeFromExt(filepath.Ext(fn))
+		if ft == pacta.FileType_UNKNOWN {
+			h.logger.Error("unhandled file extension", zap.String("dir", dirPath), zap.String("file_ext", filepath.Ext(fn)))
+		}
+		artifacts = append(artifacts, &task.AnalysisArtifact{
+			BlobURI:  pacta.BlobURI(uri),
+			FileName: fn,
+			FileType: ft,
+		})
 		return nil
 	})
+	if err != nil {
+		return nil, fmt.Errorf("error while walking dir/uploading blobs: %w", err)
+	}
+	return artifacts, nil
+}
+
+func fileTypeFromExt(ext string) pacta.FileType {
+	switch ext {
+	case ".csv":
+		return pacta.FileType_CSV
+	case ".yaml":
+		return pacta.FileType_YAML
+	case ".zip":
+		return pacta.FileType_ZIP
+	case ".html":
+		return pacta.FileType_HTML
+	case ".json":
+		return pacta.FileType_JSON
+	case ".txt":
+		return pacta.FileType_TEXT
+	case ".css":
+		return pacta.FileType_CSS
+	case ".js":
+		return pacta.FileType_JS
+	case ".ttf":
+		return pacta.FileType_TTF
+	default:
+		return pacta.FileType_UNKNOWN
+	}
 }
 
 func (h *handler) uploadBlob(ctx context.Context, srcPath, destURI string) error {
+	h.logger.Info("uploading blob", zap.String("src", srcPath), zap.String("dest", destURI))
+
 	srcF, err := os.Open(srcPath)
 	if err != nil {
 		return fmt.Errorf("failed to open file for upload: %w", err)
@@ -388,13 +432,16 @@ func (h *handler) createAudit(ctx context.Context, taskID task.ID, req *task.Cre
 
 func (h *handler) createReport(ctx context.Context, taskID task.ID, req *task.CreateReportRequest) error {
 	fileNames := []string{}
-	for i, blobURI := range req.BlobURIs {
+	for _, blobURI := range req.BlobURIs {
 		// Load the parsed portfolio from blob storage, place it in /mnt/
 		// processed_portfolios, where the `create_report.R` script expects it
 		// to be.
-		fileName := fmt.Sprintf("%d.json", i)
-		fileNames = append(fileNames, fileName)
-		destPath := filepath.Join("/", "mnt", "processed_portfolios", fileName)
+		fileNameWithExt := filepath.Base(string(blobURI))
+		if !strings.HasSuffix(fileNameWithExt, ".json") {
+			return fmt.Errorf("given blob wasn't a JSON-formatted portfolio, %q", fileNameWithExt)
+		}
+		fileNames = append(fileNames, strings.TrimSuffix(fileNameWithExt, ".json"))
+		destPath := filepath.Join("/", "mnt", "processed_portfolios", fileNameWithExt)
 		if err := h.downloadBlob(ctx, string(blobURI), destPath); err != nil {
 			return fmt.Errorf("failed to download processed portfolio blob: %w", err)
 		}
@@ -423,16 +470,40 @@ func (h *handler) createReport(ctx context.Context, taskID task.ID, req *task.Cr
 		return fmt.Errorf("failed to read report directory: %w", err)
 	}
 
+	var artifacts []*task.AnalysisArtifact
 	for _, dirEntry := range dirEntries {
 		if !dirEntry.IsDir() {
 			continue
 		}
 		dirPath := filepath.Join(reportDir, dirEntry.Name())
-		if err := h.uploadDirectory(ctx, dirPath, h.reportContainer); err != nil {
+		tmp, err := h.uploadDirectory(ctx, dirPath, h.reportContainer)
+		if err != nil {
 			return fmt.Errorf("failed to upload report directory: %w", err)
 		}
+		artifacts = tmp
+	}
+
+	events := []publisher.Event{
+		{
+			Data: task.CreateReportResponse{
+				TaskID:    taskID,
+				Request:   req,
+				Artifacts: artifacts,
+			},
+			DataVersion: to.Ptr("1.0"),
+			EventType:   to.Ptr("created-report"),
+			EventTime:   to.Ptr(time.Now()),
+			ID:          to.Ptr(string(taskID)),
+			Subject:     to.Ptr(string(taskID)),
+		},
+	}
+
+	if _, err := h.pubsub.PublishEvents(ctx, events, nil); err != nil {
+		return fmt.Errorf("failed to publish event: %w", err)
 	}
 
+	h.logger.Info("created report", zap.String("task_id", string(taskID)))
+
 	return nil
 }
 

cmd/server/pactasrv/analysis.go

@@ -255,6 +255,12 @@ func (s *Server) RunAnalysis(ctx context.Context, request api.RunAnalysisRequest
 		return nil, oapierr.Internal("unknown analysis type", zap.String("analysis_type", string(analysisType)))
 	}
 
+	now := s.Now()
+	if err := s.DB.UpdateAnalysis(s.DB.NoTxn(ctx), analysisID, db.SetAnalysisRanAt(now)); err != nil {
+		// Just log the error, it's non-critical
+		s.Logger.Error("failed to set ranAt time on analysis", zap.String("analysis_id", string(analysisID)), zap.Time("ran_at", now))
+	}
+
 	return api.RunAnalysis200JSONResponse{AnalysisId: string(analysisID)}, nil
 }
 

db/sqldb/golden/human_readable_schema.sql

@@ -47,7 +47,12 @@ CREATE TYPE file_type AS ENUM (
     'yaml',
     'zip',
     'html',
-    'json');
+    'json',
+    'txt',
+    'css',
+    'js',
+    'ttf',
+    'unknown');
 CREATE TYPE language AS ENUM (
     'en',
     'de',

db/sqldb/golden/schema_dump.sql

@@ -116,7 +116,12 @@ CREATE TYPE public.file_type AS ENUM (
     'yaml',
     'zip',
     'html',
-    'json'
+    'json',
+    'txt',
+    'css',
+    'js',
+    'ttf',
+    'unknown'
 );
 
 

db/sqldb/migrations/0011_add_report_file_types.down.sql

@@ -0,0 +1,13 @@
+BEGIN;
+
+-- There isn't a way to delete a value from an enum, so this is the workaround
+-- https://stackoverflow.com/a/56777227/17909149
+DROP TYPE file_type;
+CREATE TYPE file_type AS ENUM (
+    'csv',
+    'yaml',
+    'zip',
+    'html',
+    'json');
+
+COMMIT;

db/sqldb/migrations/0011_add_report_file_types.up.sql

@@ -0,0 +1,9 @@
+BEGIN;
+
+ALTER TYPE file_type ADD VALUE 'txt';
+ALTER TYPE file_type ADD VALUE 'css';
+ALTER TYPE file_type ADD VALUE 'js';
+ALTER TYPE file_type ADD VALUE 'ttf';
+ALTER TYPE file_type ADD VALUE 'unknown';
+
+COMMIT;

db/sqldb/sqldb_test.go

@@ -92,6 +92,8 @@ func TestSchemaHistory(t *testing.T) {
 		{ID: 8, Version: 8},   // 0008_indexes_on_blob_ids
 		{ID: 9, Version: 9},   // 0009_support_user_merge
 		{ID: 10, Version: 10}, // 0010_audit_log_enum_values
+		{ID: 11, Version: 11}, // 0011_add_report_file_types
+
 	}
 
 	if diff := cmp.Diff(want, got); diff != "" {

frontend/i18n.config.ts

@@ -5,10 +5,11 @@ export default defineI18nConfig(() => ({
   missing: (locale, key, vm) => {
     // TODO(grady) figure out how to skip this if we're in production + just log.
     // Consider using process.env.NODE_ENV == 'prod', etc.
-    const fn = inject('handleMissingTranslation')
+    const fn = vm?.appContext.app.$nuxt.$missingTranslations.handleMissingTranslation
     if (fn) {
-      const callable = fn as (locale: string, key: string) => void
-      callable(locale, key)
+      fn(locale, key)
+    } else {
+      console.warn(`No handleMissingTranslation function found, can't handle ${locale} ${key}`)
     }
   },
 }))

frontend/plugins/handle-missing-translation.ts

@@ -16,13 +16,13 @@ export default defineNuxtPlugin((nuxtApp) => {
     }
   }
 
-  nuxtApp.vueApp.provide('handleMissingTranslation', handleMissingTranslation)
   const values = computed(() => {
     return missingTranslations.value
   })
   return {
     provide: {
       missingTranslations: {
+        handleMissingTranslation,
         values,
         numberMissing,
       },

pacta/pacta.go

@@ -206,6 +206,13 @@ const (
 	FileType_ZIP  = "zip"
 	FileType_HTML = "html"
 	FileType_JSON = "json"
+
+	// All for serving reports
+	FileType_TEXT    = "txt"
+	FileType_CSS     = "css"
+	FileType_JS      = "js"
+	FileType_TTF     = "ttf"
+	FileType_UNKNOWN = "unknown"
 )
 
 var FileTypeValues = []FileType{
@@ -215,6 +222,11 @@ var FileTypeValues = []FileType{
 	FileType_JSON,
 	FileType_HTML,
 	FileType_JSON,
+	FileType_TEXT,
+	FileType_CSS,
+	FileType_JS,
+	FileType_TTF,
+	FileType_UNKNOWN,
 }
 
 func ParseFileType(s string) (FileType, error) {
@@ -233,6 +245,16 @@ func ParseFileType(s string) (FileType, error) {
 		return FileType_HTML, nil
 	case "json":
 		return FileType_JSON, nil
+	case "txt":
+		return FileType_TEXT, nil
+	case "css":
+		return FileType_CSS, nil
+	case "js":
+		return FileType_JS, nil
+	case "ttf":
+		return FileType_TTF, nil
+	case "unknown":
+		return FileType_UNKNOWN, nil
 	}
 	return "", fmt.Errorf("unknown pacta.FileType: %q", s)
 }

scripts/BUILD.bazel

@@ -47,3 +47,8 @@ sh_binary(
         "//scripts/shared:migrate",
     ],
 )
+
+sh_binary(
+    name = "build_and_load_runner",
+    srcs = ["build_and_load_runner.sh"],
+)

scripts/build_and_load_runner.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -euo pipefail
+
+ROOT="$BUILD_WORKSPACE_DIRECTORY"
+cd "$ROOT"
+
+# Build the image
+bazel build  --@io_bazel_rules_go//go/config:pure //cmd/runner:image_tarball
+
+# Load it into Docker, capture output
+LOAD_OUTPUT=$(docker load < bazel-bin/cmd/runner/image_tarball/tarball.tar)
+
+# Extract the SHA
+IMAGE_ID=$(echo $LOAD_OUTPUT | grep -oP 'sha256:\K\w+')
+
+# Tag the image
+docker tag $IMAGE_ID rmisa.azurecr.io/runner:latest
+
+echo "Tagged $IMAGE_ID as rmisa.azurecr.io/runner:latest"
+