Merge pull request #16251 from CDCgov/deployment/2024-10-17

Deployment of 2024-10-17

.github/workflows/deploy_terraform.yml

@@ -11,7 +11,7 @@ on:
 jobs:
   pre_job:
     name: Set Build Environment
-    concurrency: 
+    concurrency:
       group: ${{ github.workflow }}-${{ needs.pre_job.outputs.env_name }}
       cancel-in-progress: true
     runs-on: ubuntu-latest
@@ -28,7 +28,7 @@ jobs:
   confirm_changes:
     name: Check Terraform Stats - ${{ needs.pre_job.outputs.env_name }}
     if: ${{ needs.pre_job.outputs.tf_change == 'true' }}
-    concurrency: 
+    concurrency:
       group: ${{ github.workflow }}-${{ needs.pre_job.outputs.env_name }}
       cancel-in-progress: true
     needs:
@@ -60,7 +60,7 @@ jobs:
 
   approve_deploy:
     name: Approve Deploy - ${{ needs.pre_job.outputs.env_name }}
-    concurrency: 
+    concurrency:
       group: ${{ github.workflow }}-${{ needs.pre_job.outputs.env_name }}
       cancel-in-progress: true
     needs:
@@ -75,7 +75,7 @@ jobs:
 
   run_deploy:
     name: Run Deploy - ${{ needs.pre_job.outputs.env_name }}
-    concurrency: 
+    concurrency:
       group: ${{ github.workflow }}-${{ needs.pre_job.outputs.env_name }}
       cancel-in-progress: true
     needs:
@@ -108,4 +108,7 @@ jobs:
       - name: Run Terraform
         run: |
           terraform init -input=false
-          terraform apply -input=false -no-color -lock-timeout=600s -auto-approve
+          terraform validate
+          terraform fmt -recursive
+          terraform plan -out ${{ needs.pre_job.outputs.env_name }}-tf.plan
+          # terraform apply -input=false -no-color -lock-timeout=600s -auto-approve ${{ needs.pre_job.outputs.env_name }}-tf.plan

operations/app/terraform/modules/function_app/locals.tf

@@ -19,6 +19,7 @@ locals {
     "RS_OKTA_ClientId" = var.RS_OKTA_clientId
     "ETOR_TI_baseurl"  = var.etor_ti_base_url
     "cdctiautomated"   = var.cdctiautomated_sa
+    "JAVA_OPTS"        = var.JAVA_OPTS
     # Manage client secrets via a Key Vault
     "CREDENTIAL_STORAGE_METHOD" = "AZURE"
     "CREDENTIAL_KEY_VAULT_NAME" = var.client_config_key_vault_name

operations/app/terraform/modules/function_app/~inputs.tf

@@ -93,6 +93,7 @@ variable "etor_ti_base_url" {}
 variable "cdctiautomated_sa" {
   default = ""
 }
+variable "JAVA_OPTS" {}
 
 variable "subnets" {
   description = "A set of all available subnet combinations"

operations/app/terraform/modules/storage/main.tf

@@ -36,6 +36,13 @@ resource "azurerm_storage_account" "storage_account" {
   tags = {
     environment = var.environment
   }
+
+  timeouts {
+    create = var.timeout_create
+    read   = var.timeout_read
+    delete = var.timeout_delete
+    update = var.timeout_update
+  }
 }
 
 resource "azurerm_storage_queue" "storage_queue" {
@@ -189,6 +196,13 @@ resource "azurerm_storage_account" "storage_public" {
   tags = {
     environment = var.environment
   }
+
+  timeouts {
+    create = var.timeout_create
+    read   = var.timeout_read
+    delete = var.timeout_delete
+    update = var.timeout_update
+  }
 }
 
 resource "azurerm_storage_share" "gh_locks" {
@@ -244,6 +258,13 @@ resource "azurerm_storage_account" "storage_partner" {
   tags = {
     environment = var.environment
   }
+
+  timeouts {
+    create = var.timeout_create
+    read   = var.timeout_read
+    delete = var.timeout_delete
+    update = var.timeout_update
+  }
 }
 
 # Grant the storage account Key Vault access, to access encryption keys

operations/app/terraform/modules/storage/~inputs.tf

@@ -58,4 +58,29 @@ variable "storage_queue_name" {
   description = "Default storage queue names that will be created in the storage account."
   type        = list(string)
   default     = ["proces"]
+}
+
+# TF timeouts for storage operations
+variable "timeout_create" {
+  description = "Timeout for create operations"
+  type        = string
+  default     = "60m" # module default 60m
+}
+
+variable "timeout_read" {
+  description = "Timeout for read operations"
+  type        = string
+  default     = "60m" # module default 5m
+}
+
+variable "timeout_update" {
+  description = "Timeout for update operations"
+  type        = string
+  default     = "60m" # module default 60m
+}
+
+variable "timeout_delete" {
+  description = "Timeout for delete operations"
+  type        = string
+  default     = "60m" # module default 60m
 }

operations/app/terraform/vars/demo/locals.tf

@@ -17,6 +17,7 @@ locals {
     RS_okta_redirect_url  = "https://prime-data-hub-XXXXXXX.azurefd.net/download"
     RS_OKTA_scope         = "reportstream_dev"
     etor_ti_base_url      = "https://cdcti-stg-api.azurewebsites.net"
+    JAVA_OPTS             = "-Dfile.encoding=UTF-8"
   }
   key_vault = {
     app_config_kv_name    = "pdh${local.init.environment}-appconfig${local.init.random_id}"

operations/app/terraform/vars/demo/main.tf

@@ -183,6 +183,7 @@ module "function_app" {
   RS_OKTA_clientId                  = data.azurerm_key_vault_secret.RS_OKTA_clientId.value
   RS_OKTA_authKey                   = data.azurerm_key_vault_secret.RS_OKTA_authKey.value
   etor_ti_base_url                  = local.init.etor_ti_base_url
+  JAVA_OPTS                         = local.init.JAVA_OPTS
 }
 
 module "front_door" {

operations/app/terraform/vars/prod/locals.tf

@@ -16,7 +16,7 @@ locals {
     storage_queue_name    = ["process"]
     sftp_container_module = false
     etor_ti_base_url      = "https://cdcti-prd-api.azurewebsites.net"
-
+    JAVA_OPTS             = "-Dfile.encoding=UTF-8"
   }
   key_vault = {
     app_config_kv_name    = "pdh${local.init.environment}-appconfig"

operations/app/terraform/vars/prod/main.tf

@@ -161,6 +161,7 @@ module "function_app" {
   RS_OKTA_clientId                  = data.azurerm_key_vault_secret.RS_OKTA_clientId.value
   RS_OKTA_authKey                   = data.azurerm_key_vault_secret.RS_OKTA_authKey.value
   etor_ti_base_url                  = local.init.etor_ti_base_url
+  JAVA_OPTS                         = local.init.JAVA_OPTS
 }
 
 module "front_door" {

operations/app/terraform/vars/staging/locals.tf

@@ -16,6 +16,7 @@ locals {
     storage_queue_name    = ["process", "batch", "batch-poison", "elr-fhir-convert", "process-poison", "send", "send-poison", "elr-fhir-convert", "elr-fhir-convert-poison", "elr-fhir-route", "elr-fhir-translate", "elr-fhir-translate-poison", "process-elr"]
     sftp_container_module = true
     etor_ti_base_url      = "https://cdcti-stg-api.azurewebsites.net"
+    JAVA_OPTS             = "-Dfile.encoding=UTF-8"
   }
   key_vault = {
     app_config_kv_name    = "pdh${local.init.environment}-appconfig"

operations/app/terraform/vars/staging/main.tf

@@ -161,6 +161,7 @@ module "function_app" {
   RS_OKTA_authKey                   = data.azurerm_key_vault_secret.RS_OKTA_authKey.value
   etor_ti_base_url                  = local.init.etor_ti_base_url
   cdctiautomated_sa                 = data.azurerm_key_vault_secret.cdctiautomated_sa.value
+  JAVA_OPTS                         = local.init.JAVA_OPTS
 }
 
 module "front_door" {

operations/app/terraform/vars/test/locals.tf

@@ -16,6 +16,7 @@ locals {
     storage_queue_name    = ["process"]
     sftp_container_module = true
     etor_ti_base_url      = "https://cdcti-stg-api.azurewebsites.net"
+    JAVA_OPTS             = "-Dfile.encoding=UTF-8"
   }
   key_vault = {
     app_config_kv_name    = "pdh${local.init.environment}-app-config"

operations/app/terraform/vars/test/main.tf

@@ -164,6 +164,7 @@ module "function_app" {
   RS_OKTA_clientId                  = data.azurerm_key_vault_secret.RS_OKTA_clientId.value
   RS_OKTA_authKey                   = data.azurerm_key_vault_secret.RS_OKTA_authKey.value
   etor_ti_base_url                  = local.init.etor_ti_base_url
+  JAVA_OPTS                         = local.init.JAVA_OPTS
 }
 
 module "front_door" {

prime-router/docs/api/reports.yml

@@ -118,6 +118,51 @@ paths:
                   $ref: '#/components/schemas/Report'
         '500':
           description: Internal Server Error
+  /reports/testing/test:
+    post:
+      summary: Evaluates a message based off of the receiver settings specified. Returns any errors, filtering, or the message.
+      security:
+        - OAuth2: [ system_admin ]
+      parameters:
+        - in: query
+          name: receiverName
+          description: The name of the receiver to look for in the current environment's settings
+          schema:
+            type: string
+          required: true
+          example: full-elr
+        - in: query
+          name: organizationName
+          description: The name of the organization to look for the receiver in the current environment's settings
+          required: true
+          schema:
+            type: string
+          example: me-phd
+        - in: query
+          name: senderSchema
+          description: The path to the sender schema
+          required: false
+          schema:
+            type: string
+          example: classpath:/metadata/fhir_transforms/senders/SimpleReport/simple-report-sender-transform.yml
+      requestBody:
+        description: The message to process
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/Report'
+      responses:
+        '200':
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Report'
+        '400':
+          description: Error with one or more filters or finding the receiver.
+        '500':
+          description: Internal Server Error
   /reports/download:
     get:
       summary: Downloads a message based on the report id

prime-router/src/main/kotlin/azure/ReportFunction.kt

@@ -1,8 +1,10 @@
 package gov.cdc.prime.router.azure
 
 import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.databind.SerializationFeature
 import com.fasterxml.jackson.databind.json.JsonMapper
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule
+import com.github.ajalt.clikt.core.CliktError
 import com.google.common.net.HttpHeaders
 import com.microsoft.azure.functions.HttpMethod
 import com.microsoft.azure.functions.HttpRequestMessage
@@ -23,7 +25,6 @@ import gov.cdc.prime.router.Sender
 import gov.cdc.prime.router.Sender.ProcessingType
 import gov.cdc.prime.router.SubmissionReceiver
 import gov.cdc.prime.router.UniversalPipelineReceiver
-import gov.cdc.prime.router.azure.BlobAccess.Companion.defaultBlobMetadata
 import gov.cdc.prime.router.azure.BlobAccess.Companion.getBlobContainer
 import gov.cdc.prime.router.azure.db.enums.TaskAction
 import gov.cdc.prime.router.azure.db.tables.pojos.ReportFile
@@ -32,6 +33,7 @@ import gov.cdc.prime.router.azure.observability.event.ReportStreamEventName
 import gov.cdc.prime.router.azure.observability.event.ReportStreamEventProperties
 import gov.cdc.prime.router.azure.observability.event.ReportStreamEventService
 import gov.cdc.prime.router.cli.PIIRemovalCommands
+import gov.cdc.prime.router.cli.ProcessFhirCommands
 import gov.cdc.prime.router.common.AzureHttpUtils.getSenderIP
 import gov.cdc.prime.router.common.Environment
 import gov.cdc.prime.router.common.JacksonMapperUtilities
@@ -43,6 +45,7 @@ import gov.cdc.prime.router.tokens.authenticationFailure
 import gov.cdc.prime.router.tokens.authorizationFailure
 import kotlinx.serialization.json.Json
 import org.apache.logging.log4j.kotlin.Logging
+import java.io.File
 import java.nio.charset.StandardCharsets
 import java.util.UUID
 
@@ -120,6 +123,113 @@ class ReportFunction(
         return HttpUtilities.unauthorizedResponse(request)
     }
 
+    /**
+     * Run a message through the fhirdata cli
+     *
+     * @see ../../../docs/api/reports.yml
+     */
+    @FunctionName("processFhirDataRequest")
+    fun processFhirDataRequest(
+        @HttpTrigger(
+            name = "processFhirDataRequest",
+            methods = [HttpMethod.POST],
+            authLevel = AuthorizationLevel.ANONYMOUS,
+            route = "reports/testing/test"
+        ) request: HttpRequestMessage<String?>,
+    ): HttpResponseMessage {
+        val claims = AuthenticatedClaims.authenticate(request)
+        if (claims != null && claims.authorized(setOf(Scope.primeAdminScope))) {
+            val receiverName = request.queryParameters["receiverName"]
+            val organizationName = request.queryParameters["organizationName"]
+            val senderSchema = request.queryParameters["senderSchema"]
+            if (receiverName.isNullOrBlank()) {
+                return HttpUtilities.badRequestResponse(
+                    request,
+                    "The receiver name is required"
+                )
+            }
+            if (organizationName.isNullOrBlank()) {
+                return HttpUtilities.badRequestResponse(
+                    request,
+                    "The organization name is required"
+                )
+            }
+            if (request.body.isNullOrBlank()) {
+                return HttpUtilities.badRequestResponse(
+                    request,
+                    "A message to process must be included in the body"
+                )
+            }
+            val file = File("filename.fhir")
+            file.createNewFile()
+            file.bufferedWriter().use { out ->
+                out.write(request.body)
+            }
+
+            try {
+                val result = ProcessFhirCommands().processFhirDataRequest(
+                    file,
+                    Environment.get().envName,
+                    receiverName,
+                    organizationName,
+                    senderSchema,
+                    false
+                )
+                file.delete()
+                val message = if (result.message != null) {
+                    result.message.toString()
+                } else {
+                    null
+                }
+                val bundle = if (result.bundle != null) {
+                    result.bundle.toString()
+                } else {
+                    null
+                }
+                return HttpUtilities.okResponse(
+                    request,
+                    ObjectMapper().configure(SerializationFeature.FAIL_ON_SELF_REFERENCES, false).writeValueAsString(
+                        MessageOrBundleStringified(
+                            message,
+                            bundle,
+                            result.senderTransformPassed,
+                            result.senderTransformErrors,
+                            result.senderTransformWarnings,
+                            result.enrichmentSchemaPassed,
+                            result.enrichmentSchemaErrors,
+                            result.senderTransformWarnings,
+                            result.receiverTransformPassed,
+                            result.receiverTransformErrors,
+                            result.receiverTransformWarnings,
+                            result.filterErrors,
+                            result.filtersPassed
+                        )
+                    )
+                )
+            } catch (exception: CliktError) {
+                file.delete()
+                return HttpUtilities.badRequestResponse(request, "${exception.message}")
+            }
+        }
+        return HttpUtilities.unauthorizedResponse(request)
+    }
+
+    class MessageOrBundleStringified(
+        var message: String? = null,
+        var bundle: String? = null,
+        override var senderTransformPassed: Boolean = true,
+        override var senderTransformErrors: MutableList<String> = mutableListOf(),
+        override var senderTransformWarnings: MutableList<String> = mutableListOf(),
+        override var enrichmentSchemaPassed: Boolean = true,
+        override var enrichmentSchemaErrors: MutableList<String> = mutableListOf(),
+        override var enrichmentSchemaWarnings: MutableList<String> = mutableListOf(),
+        override var receiverTransformPassed: Boolean = true,
+        override var receiverTransformErrors: MutableList<String> = mutableListOf(),
+        override var receiverTransformWarnings: MutableList<String> = mutableListOf(),
+        override var filterErrors: MutableList<String> = mutableListOf(),
+        override var filtersPassed: Boolean = true,
+    ) : ProcessFhirCommands.MessageOrBundleParent()
+
     /**
      * Moved the logic to a separate function for testing purposes
      */