From 446f6d54a4f46ebc8f89eae62fbf310a8ea6752a Mon Sep 17 00:00:00 2001 From: Gabriel Dorsch Date: Mon, 30 Sep 2024 12:08:57 -0400 Subject: [PATCH 01/14] Updated submission history test cases --- .../SubmissionFunctionIntegrationTests.kt | 129 +++++++++++++++++- 1 file changed, 126 insertions(+), 3 deletions(-) diff --git a/prime-router/src/test/kotlin/history/azure/SubmissionFunctionIntegrationTests.kt b/prime-router/src/test/kotlin/history/azure/SubmissionFunctionIntegrationTests.kt index 4df2eccea16..34bd4b30719 100644 --- a/prime-router/src/test/kotlin/history/azure/SubmissionFunctionIntegrationTests.kt +++ b/prime-router/src/test/kotlin/history/azure/SubmissionFunctionIntegrationTests.kt @@ -71,6 +71,128 @@ class SubmissionFunctionIntegrationTests { @Test fun `it should return a history for partially delivered submission`() { + val submittedReport = reportGraph { + topic(Topic.FULL_ELR) + format(MimeFormat.HL7) + sender(UniversalPipelineTestUtils.hl7Sender) + + submission { + action(TaskAction.receive) + reportGraphNode { + action(TaskAction.convert) + log(ActionLog(InvalidParamMessage("log"), type = ActionLogLevel.warning)) + reportGraphNode { + action(TaskAction.destination_filter) + reportGraphNode { + action(TaskAction.none) + receiver(UniversalPipelineTestUtils.universalPipelineOrganization.receivers[1]) + itemCount(0) + } + } + reportGraphNode { + action(TaskAction.destination_filter) + reportGraphNode { + action(TaskAction.receiver_filter) + reportGraphNode { + action(TaskAction.translate) + receiver(UniversalPipelineTestUtils.universalPipelineOrganization.receivers[0]) + reportGraphNode { + action(TaskAction.send) + transportResult("Success") + receiver(UniversalPipelineTestUtils.universalPipelineOrganization.receivers[0]) + } + } + } + reportGraphNode { + action(TaskAction.receiver_filter) + reportGraphNode { + action(TaskAction.none) + receiver(UniversalPipelineTestUtils.universalPipelineOrganization.receivers[1]) + itemCount(0) + } + } + } + } + } + }.generate(ReportStreamTestDatabaseContainer.testDatabaseAccess) + + val httpRequestMessage = MockHttpRequestMessage() + + val func = setupSubmissionFunction() + + val history = func + .getReportDetailedHistory(httpRequestMessage, submittedReport.node.reportId.toString()) + assertThat(history).isNotNull() + val historyNode = JacksonMapperUtilities.defaultMapper.readTree(history.body.toString()) + assertThat( + historyNode.get("overallStatus").asText() + ).isEqualTo(DetailedSubmissionHistory.Status.PARTIALLY_DELIVERED.toString()) + assertThat(historyNode.get("destinations").size()).isEqualTo(2) + assertThat(historyNode.get("errors").size()).isEqualTo(0) + assertThat(historyNode.get("warnings").size()).isEqualTo(1) + } + + // this test remains to prevent breaking queries against old submissions that used the legacy route step + @Test + fun `it should return a history for an partially delivered submission (for legacy route step)`() { + val submittedReport = reportGraph { + topic(Topic.FULL_ELR) + format(MimeFormat.HL7) + sender(UniversalPipelineTestUtils.hl7Sender) + + submission { + action(TaskAction.receive) + reportGraphNode { + action(TaskAction.convert) + log(ActionLog(InvalidParamMessage("log"), type = ActionLogLevel.warning)) + reportGraphNode { + action(TaskAction.route) + reportGraphNode { + action(TaskAction.none) + receiver(UniversalPipelineTestUtils.universalPipelineOrganization.receivers[1]) + itemCount(0) + } + } + reportGraphNode { + action(TaskAction.route) + reportGraphNode { + action(TaskAction.translate) + receiver(UniversalPipelineTestUtils.universalPipelineOrganization.receivers[0]) + reportGraphNode { + action(TaskAction.send) + transportResult("Success") + receiver(UniversalPipelineTestUtils.universalPipelineOrganization.receivers[0]) + } + } + reportGraphNode { + action(TaskAction.none) + receiver(UniversalPipelineTestUtils.universalPipelineOrganization.receivers[1]) + itemCount(0) + } + } + } + } + }.generate(ReportStreamTestDatabaseContainer.testDatabaseAccess) + + val httpRequestMessage = MockHttpRequestMessage() + + val func = setupSubmissionFunction() + + val history = func + .getReportDetailedHistory(httpRequestMessage, submittedReport.node.reportId.toString()) + assertThat(history).isNotNull() + val historyNode = JacksonMapperUtilities.defaultMapper.readTree(history.body.toString()) + assertThat( + historyNode.get("overallStatus").asText() + ).isEqualTo(DetailedSubmissionHistory.Status.PARTIALLY_DELIVERED.toString()) + assertThat(historyNode.get("destinations").size()).isEqualTo(2) + assertThat(historyNode.get("errors").size()).isEqualTo(0) + assertThat(historyNode.get("warnings").size()).isEqualTo(1) + } + + // TODO: https://github.com/CDCgov/prime-reportstream/issues/16054 + @Test + fun `it should return a history for an in-flight submission`() { val submittedReport = reportGraph { topic(Topic.FULL_ELR) format(MimeFormat.HL7) @@ -126,15 +248,16 @@ class SubmissionFunctionIntegrationTests { val historyNode = JacksonMapperUtilities.defaultMapper.readTree(history.body.toString()) assertThat( historyNode.get("overallStatus").asText() - ).isEqualTo(DetailedSubmissionHistory.Status.PARTIALLY_DELIVERED.toString()) + ).isEqualTo(DetailedSubmissionHistory.Status.PARTIALLY_DELIVERED.toString()) // TODO: should be RECEIVED assertThat(historyNode.get("destinations").size()).isEqualTo(2) assertThat(historyNode.get("errors").size()).isEqualTo(0) assertThat(historyNode.get("warnings").size()).isEqualTo(1) } + // TODO: https://github.com/CDCgov/prime-reportstream/issues/16054 // this test remains to prevent breaking queries against old submissions that used the legacy route step @Test - fun `it should return a history for partially delivered submission (for legacy route step)`() { + fun `it should return a history for an in-flight submission (for legacy route step)`() { val submittedReport = reportGraph { topic(Topic.FULL_ELR) format(MimeFormat.HL7) @@ -184,7 +307,7 @@ class SubmissionFunctionIntegrationTests { val historyNode = JacksonMapperUtilities.defaultMapper.readTree(history.body.toString()) assertThat( historyNode.get("overallStatus").asText() - ).isEqualTo(DetailedSubmissionHistory.Status.PARTIALLY_DELIVERED.toString()) + ).isEqualTo(DetailedSubmissionHistory.Status.PARTIALLY_DELIVERED.toString()) // TODO: should be RECEIVED assertThat(historyNode.get("destinations").size()).isEqualTo(2) assertThat(historyNode.get("errors").size()).isEqualTo(0) assertThat(historyNode.get("warnings").size()).isEqualTo(1) From 168de58f5f64c99a27a0b0e01fc534b42a430cba Mon Sep 17 00:00:00 2001 From: devopsmatt Date: Thu, 3 Oct 2024 13:24:31 -0700 Subject: [PATCH 02/14] adding DevSecOps WoW and issue template --- .github/ISSUE_TEMPLATE/DevSecOps-WoW.md | 25 ++++++++ .github/ISSUE_TEMPLATE/DevSecOps-issue.md | 69 +++++++++++++++++++++++ DevSecOps-WoW.md | 25 ++++++++ DevSecOps-issue.md | 69 +++++++++++++++++++++++ 4 files changed, 188 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/DevSecOps-WoW.md create mode 100644 .github/ISSUE_TEMPLATE/DevSecOps-issue.md create mode 100644 DevSecOps-WoW.md create mode 100644 DevSecOps-issue.md diff --git a/.github/ISSUE_TEMPLATE/DevSecOps-WoW.md b/.github/ISSUE_TEMPLATE/DevSecOps-WoW.md new file mode 100644 index 00000000000..0ffe1c7a61b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/DevSecOps-WoW.md @@ -0,0 +1,25 @@ +# CDC ReportStream DevSecOps Ways of Working + +## Team Structure + +- The CDC DevSecOps Team includes multiple members, although no person is assigned to a specific Application Team. +- The CDC DevSecOps Team operates on more of a 'help desk' model where they receive tickets and internally triage/prioritize their work. + +## Requesting DevSecOps Support + +- Quick questions that take <5 min to answer can be asked in Slack and answered by anyone with knowledge, but any issues that need investigation or that may take longer than 5 minutes to gather context and resolve should be opened in ticket form using the [DevSecOps-issue.md](../ISSUE_TEMPLATE/DevSecOps-issue.md) ticket template. +- Please avoid using DMs to communicate directly with the DevSecOps Team - instead post communication in the shared #prime-devops channel and tag the appropriate individual(s). +- Apps Teams will request DevSecOps support via the [DevSecOps-issue.md](../ISSUE_TEMPLATE/DevSecOps-issue.md) Github ticket template. The template will automatically add appropriate labels and assign the ticket to the DevSecOps Team. +- The DevSecOps Team will track all DevSecOps issues on the ReportStream board that are tagged with the `DevSecOps` and `reportstream` labels. +- When new issues are created using the DevSecOps issue template, the DevSecOps Team will receive an automatic Slack notification in a separate DevSecOps alert channel (TBD). + +## Workflow Management/Task Prioritization + +- The DevSecOps Team will operate using a Kanban approach and stay within established Work in Progress (WIP) limits. +- While the DevSecOps Team won't be embedded with specific Apps Teams, they will still operate within the same sprint cadence as the Apps Teams. +- DevSecOps will review new tickets as they come in and prioritize them. +- Every 2 weeks on Wednesday, a DevSecOps Team representative will attend a Scrum of Scrums with the Apps Tech Leads (and TBD) to review the status of in progress items, flag blockers, and discuss prioritization of new issues. Patrick will give ultimate guidance on what issues need to take priority over others in scenarios, whereever it is unclear. +- DevSecOps Team members are not required to attend Apps Team stand ups, sprint planning, refinement, or retro meetings given that they may be doing work across multiple apps at a given time. That said, they will join Apps Team stand ups at their discretion if they need to communicate with the Team, and/or seek clarifications or are requested to join to provide input. They will also pair with Tech Leads to flesh out draft issues, when their input is needed. +- (Eventually) DevSecOps Team members will not factor into app Team velocity calculations. +- The DevSecOps Team will conduct their own internal stand ups for 10-15 minutes and other recurring Agile ceremonies. +- DevSecOps Team members will communicate status of their tickets with the respective App Tech Leads. \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/DevSecOps-issue.md b/.github/ISSUE_TEMPLATE/DevSecOps-issue.md new file mode 100644 index 00000000000..aa0dbad2787 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/DevSecOps-issue.md @@ -0,0 +1,69 @@ +--- +name: DevSecOps ticket +about: For new DevSecOps tickets on the board. +labels: ['DevSecOps', 'reportstream'] +--- +# DevSecOps Issue + +## Summary + +Context and description + +## Environment + +- [ ] Local +- [ ] Dev +- [ ] Stage +- [ ] Prod + +## Priority + +- [ ] **Critical** - affecting prod systems +- [ ] **Major** - blocking major functionality, deployment, etc +- [ ] **Minor** - improvements, bug fixes +- [ ] **Nice-to-have** - feature request + +### Blocks the following + +- List any blockers +- Link any Github relevant issues + +### Blocked by the following + +- Link to any tickets that add context, etc + +## Contact + +**Team**: + +**Slack Team Channel**: + +**Slack Contact (Full Name):** + +**Technical Team Lead:** + +## How to Reproduce + +### Screenshots, links, etc. for context + +## How to Test + +## Definition of Done + +- [ ] + +## Context Links + +**Git Repo:** + +**Relevant Code Links:** + +**Azure**: + +**Site URL**: + +**Other**: + +## Notes + +- \ No newline at end of file diff --git a/DevSecOps-WoW.md b/DevSecOps-WoW.md new file mode 100644 index 00000000000..0ffe1c7a61b --- /dev/null +++ b/DevSecOps-WoW.md @@ -0,0 +1,25 @@ +# CDC ReportStream DevSecOps Ways of Working + +## Team Structure + +- The CDC DevSecOps Team includes multiple members, although no person is assigned to a specific Application Team. +- The CDC DevSecOps Team operates on more of a 'help desk' model where they receive tickets and internally triage/prioritize their work. + +## Requesting DevSecOps Support + +- Quick questions that take <5 min to answer can be asked in Slack and answered by anyone with knowledge, but any issues that need investigation or that may take longer than 5 minutes to gather context and resolve should be opened in ticket form using the [DevSecOps-issue.md](../ISSUE_TEMPLATE/DevSecOps-issue.md) ticket template. +- Please avoid using DMs to communicate directly with the DevSecOps Team - instead post communication in the shared #prime-devops channel and tag the appropriate individual(s). +- Apps Teams will request DevSecOps support via the [DevSecOps-issue.md](../ISSUE_TEMPLATE/DevSecOps-issue.md) Github ticket template. The template will automatically add appropriate labels and assign the ticket to the DevSecOps Team. +- The DevSecOps Team will track all DevSecOps issues on the ReportStream board that are tagged with the `DevSecOps` and `reportstream` labels. +- When new issues are created using the DevSecOps issue template, the DevSecOps Team will receive an automatic Slack notification in a separate DevSecOps alert channel (TBD). + +## Workflow Management/Task Prioritization + +- The DevSecOps Team will operate using a Kanban approach and stay within established Work in Progress (WIP) limits. +- While the DevSecOps Team won't be embedded with specific Apps Teams, they will still operate within the same sprint cadence as the Apps Teams. +- DevSecOps will review new tickets as they come in and prioritize them. +- Every 2 weeks on Wednesday, a DevSecOps Team representative will attend a Scrum of Scrums with the Apps Tech Leads (and TBD) to review the status of in progress items, flag blockers, and discuss prioritization of new issues. Patrick will give ultimate guidance on what issues need to take priority over others in scenarios, whereever it is unclear. +- DevSecOps Team members are not required to attend Apps Team stand ups, sprint planning, refinement, or retro meetings given that they may be doing work across multiple apps at a given time. That said, they will join Apps Team stand ups at their discretion if they need to communicate with the Team, and/or seek clarifications or are requested to join to provide input. They will also pair with Tech Leads to flesh out draft issues, when their input is needed. +- (Eventually) DevSecOps Team members will not factor into app Team velocity calculations. +- The DevSecOps Team will conduct their own internal stand ups for 10-15 minutes and other recurring Agile ceremonies. +- DevSecOps Team members will communicate status of their tickets with the respective App Tech Leads. \ No newline at end of file diff --git a/DevSecOps-issue.md b/DevSecOps-issue.md new file mode 100644 index 00000000000..aa0dbad2787 --- /dev/null +++ b/DevSecOps-issue.md @@ -0,0 +1,69 @@ +--- +name: DevSecOps ticket +about: For new DevSecOps tickets on the board. +labels: ['DevSecOps', 'reportstream'] +--- +# DevSecOps Issue + +## Summary + +Context and description + +## Environment + +- [ ] Local +- [ ] Dev +- [ ] Stage +- [ ] Prod + +## Priority + +- [ ] **Critical** - affecting prod systems +- [ ] **Major** - blocking major functionality, deployment, etc +- [ ] **Minor** - improvements, bug fixes +- [ ] **Nice-to-have** - feature request + +### Blocks the following + +- List any blockers +- Link any Github relevant issues + +### Blocked by the following + +- Link to any tickets that add context, etc + +## Contact + +**Team**: + +**Slack Team Channel**: + +**Slack Contact (Full Name):** + +**Technical Team Lead:** + +## How to Reproduce + +### Screenshots, links, etc. for context + +## How to Test + +## Definition of Done + +- [ ] + +## Context Links + +**Git Repo:** + +**Relevant Code Links:** + +**Azure**: + +**Site URL**: + +**Other**: + +## Notes + +- \ No newline at end of file From b5933c4ced3d99dba9a21abf0398c63f107675bb Mon Sep 17 00:00:00 2001 From: devopsmatt Date: Thu, 3 Oct 2024 13:32:29 -0700 Subject: [PATCH 03/14] mop up --- .github/ISSUE_TEMPLATE/DevSecOps-WoW.md | 25 --------- DevSecOps-issue.md | 69 ------------------------- 2 files changed, 94 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE/DevSecOps-WoW.md delete mode 100644 DevSecOps-issue.md diff --git a/.github/ISSUE_TEMPLATE/DevSecOps-WoW.md b/.github/ISSUE_TEMPLATE/DevSecOps-WoW.md deleted file mode 100644 index 0ffe1c7a61b..00000000000 --- a/.github/ISSUE_TEMPLATE/DevSecOps-WoW.md +++ /dev/null @@ -1,25 +0,0 @@ -# CDC ReportStream DevSecOps Ways of Working - -## Team Structure - -- The CDC DevSecOps Team includes multiple members, although no person is assigned to a specific Application Team. -- The CDC DevSecOps Team operates on more of a 'help desk' model where they receive tickets and internally triage/prioritize their work. - -## Requesting DevSecOps Support - -- Quick questions that take <5 min to answer can be asked in Slack and answered by anyone with knowledge, but any issues that need investigation or that may take longer than 5 minutes to gather context and resolve should be opened in ticket form using the [DevSecOps-issue.md](../ISSUE_TEMPLATE/DevSecOps-issue.md) ticket template. -- Please avoid using DMs to communicate directly with the DevSecOps Team - instead post communication in the shared #prime-devops channel and tag the appropriate individual(s). -- Apps Teams will request DevSecOps support via the [DevSecOps-issue.md](../ISSUE_TEMPLATE/DevSecOps-issue.md) Github ticket template. The template will automatically add appropriate labels and assign the ticket to the DevSecOps Team. -- The DevSecOps Team will track all DevSecOps issues on the ReportStream board that are tagged with the `DevSecOps` and `reportstream` labels. -- When new issues are created using the DevSecOps issue template, the DevSecOps Team will receive an automatic Slack notification in a separate DevSecOps alert channel (TBD). - -## Workflow Management/Task Prioritization - -- The DevSecOps Team will operate using a Kanban approach and stay within established Work in Progress (WIP) limits. -- While the DevSecOps Team won't be embedded with specific Apps Teams, they will still operate within the same sprint cadence as the Apps Teams. -- DevSecOps will review new tickets as they come in and prioritize them. -- Every 2 weeks on Wednesday, a DevSecOps Team representative will attend a Scrum of Scrums with the Apps Tech Leads (and TBD) to review the status of in progress items, flag blockers, and discuss prioritization of new issues. Patrick will give ultimate guidance on what issues need to take priority over others in scenarios, whereever it is unclear. -- DevSecOps Team members are not required to attend Apps Team stand ups, sprint planning, refinement, or retro meetings given that they may be doing work across multiple apps at a given time. That said, they will join Apps Team stand ups at their discretion if they need to communicate with the Team, and/or seek clarifications or are requested to join to provide input. They will also pair with Tech Leads to flesh out draft issues, when their input is needed. -- (Eventually) DevSecOps Team members will not factor into app Team velocity calculations. -- The DevSecOps Team will conduct their own internal stand ups for 10-15 minutes and other recurring Agile ceremonies. -- DevSecOps Team members will communicate status of their tickets with the respective App Tech Leads. \ No newline at end of file diff --git a/DevSecOps-issue.md b/DevSecOps-issue.md deleted file mode 100644 index aa0dbad2787..00000000000 --- a/DevSecOps-issue.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -name: DevSecOps ticket -about: For new DevSecOps tickets on the board. -labels: ['DevSecOps', 'reportstream'] ---- -# DevSecOps Issue - -## Summary - -Context and description - -## Environment - -- [ ] Local -- [ ] Dev -- [ ] Stage -- [ ] Prod - -## Priority - -- [ ] **Critical** - affecting prod systems -- [ ] **Major** - blocking major functionality, deployment, etc -- [ ] **Minor** - improvements, bug fixes -- [ ] **Nice-to-have** - feature request - -### Blocks the following - -- List any blockers -- Link any Github relevant issues - -### Blocked by the following - -- Link to any tickets that add context, etc - -## Contact - -**Team**: - -**Slack Team Channel**: - -**Slack Contact (Full Name):** - -**Technical Team Lead:** - -## How to Reproduce - -### Screenshots, links, etc. for context - -## How to Test - -## Definition of Done - -- [ ] - -## Context Links - -**Git Repo:** - -**Relevant Code Links:** - -**Azure**: - -**Site URL**: - -**Other**: - -## Notes - -- \ No newline at end of file From 69440c6ba22dfd267c218dceb5df22d64350da57 Mon Sep 17 00:00:00 2001 From: matts <22215332+devopsmatt@users.noreply.github.com> Date: Thu, 3 Oct 2024 13:42:50 -0700 Subject: [PATCH 04/14] Update DevSecOps-WoW.md typo fix --- DevSecOps-WoW.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DevSecOps-WoW.md b/DevSecOps-WoW.md index 0ffe1c7a61b..09072914362 100644 --- a/DevSecOps-WoW.md +++ b/DevSecOps-WoW.md @@ -18,8 +18,8 @@ - The DevSecOps Team will operate using a Kanban approach and stay within established Work in Progress (WIP) limits. - While the DevSecOps Team won't be embedded with specific Apps Teams, they will still operate within the same sprint cadence as the Apps Teams. - DevSecOps will review new tickets as they come in and prioritize them. -- Every 2 weeks on Wednesday, a DevSecOps Team representative will attend a Scrum of Scrums with the Apps Tech Leads (and TBD) to review the status of in progress items, flag blockers, and discuss prioritization of new issues. Patrick will give ultimate guidance on what issues need to take priority over others in scenarios, whereever it is unclear. +- Every 2 weeks on Wednesday, a DevSecOps Team representative will attend a Scrum of Scrums with the Apps Tech Leads (and TBD) to review the status of in progress items, flag blockers, and discuss prioritization of new issues. Patrick will give ultimate guidance on what issues need to take priority over others in scenarios, wherever it is unclear. - DevSecOps Team members are not required to attend Apps Team stand ups, sprint planning, refinement, or retro meetings given that they may be doing work across multiple apps at a given time. That said, they will join Apps Team stand ups at their discretion if they need to communicate with the Team, and/or seek clarifications or are requested to join to provide input. They will also pair with Tech Leads to flesh out draft issues, when their input is needed. - (Eventually) DevSecOps Team members will not factor into app Team velocity calculations. - The DevSecOps Team will conduct their own internal stand ups for 10-15 minutes and other recurring Agile ceremonies. -- DevSecOps Team members will communicate status of their tickets with the respective App Tech Leads. \ No newline at end of file +- DevSecOps Team members will communicate status of their tickets with the respective App Tech Leads. From a6abc12bbcdf5b40e92fddab6edea8312b07867e Mon Sep 17 00:00:00 2001 From: matts <22215332+devopsmatt@users.noreply.github.com> Date: Thu, 3 Oct 2024 14:04:17 -0700 Subject: [PATCH 05/14] Update DevSecOps-WoW.md Agreed. The Zenhub metrics can be filtered by labels. Co-authored-by: Arnej <118766341+arnejduranovic@users.noreply.github.com> --- DevSecOps-WoW.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DevSecOps-WoW.md b/DevSecOps-WoW.md index 09072914362..d441e6a1b70 100644 --- a/DevSecOps-WoW.md +++ b/DevSecOps-WoW.md @@ -20,6 +20,6 @@ - DevSecOps will review new tickets as they come in and prioritize them. - Every 2 weeks on Wednesday, a DevSecOps Team representative will attend a Scrum of Scrums with the Apps Tech Leads (and TBD) to review the status of in progress items, flag blockers, and discuss prioritization of new issues. Patrick will give ultimate guidance on what issues need to take priority over others in scenarios, wherever it is unclear. - DevSecOps Team members are not required to attend Apps Team stand ups, sprint planning, refinement, or retro meetings given that they may be doing work across multiple apps at a given time. That said, they will join Apps Team stand ups at their discretion if they need to communicate with the Team, and/or seek clarifications or are requested to join to provide input. They will also pair with Tech Leads to flesh out draft issues, when their input is needed. -- (Eventually) DevSecOps Team members will not factor into app Team velocity calculations. +- DevSecOps Team members will not factor into app Team velocity calculations. - The DevSecOps Team will conduct their own internal stand ups for 10-15 minutes and other recurring Agile ceremonies. - DevSecOps Team members will communicate status of their tickets with the respective App Tech Leads. From 3ffc3dd426ab1cedd0e10ea86213461eb80388df Mon Sep 17 00:00:00 2001 From: matts <22215332+devopsmatt@users.noreply.github.com> Date: Thu, 3 Oct 2024 14:06:40 -0700 Subject: [PATCH 06/14] Update DevSecOps-WoW.md as per Arnej's suggestion Adding: DevSec0ps team shall participate in, and help inform, quarterly program planning exercises --- DevSecOps-WoW.md | 1 + 1 file changed, 1 insertion(+) diff --git a/DevSecOps-WoW.md b/DevSecOps-WoW.md index d441e6a1b70..25369412973 100644 --- a/DevSecOps-WoW.md +++ b/DevSecOps-WoW.md @@ -23,3 +23,4 @@ - DevSecOps Team members will not factor into app Team velocity calculations. - The DevSecOps Team will conduct their own internal stand ups for 10-15 minutes and other recurring Agile ceremonies. - DevSecOps Team members will communicate status of their tickets with the respective App Tech Leads. +- DevSec0ps team shall participate in, and help inform, quarterly program planning exercises From 2fdbc5fc54f17b9a8012a964999ae5ab37d90be5 Mon Sep 17 00:00:00 2001 From: matts <22215332+devopsmatt@users.noreply.github.com> Date: Thu, 3 Oct 2024 14:08:37 -0700 Subject: [PATCH 07/14] Update DevSecOps-WoW.md s/Patrick/The project engineering lead/ Co-authored-by: Arnej <118766341+arnejduranovic@users.noreply.github.com> --- DevSecOps-WoW.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DevSecOps-WoW.md b/DevSecOps-WoW.md index 25369412973..0d8a026b09c 100644 --- a/DevSecOps-WoW.md +++ b/DevSecOps-WoW.md @@ -18,7 +18,7 @@ - The DevSecOps Team will operate using a Kanban approach and stay within established Work in Progress (WIP) limits. - While the DevSecOps Team won't be embedded with specific Apps Teams, they will still operate within the same sprint cadence as the Apps Teams. - DevSecOps will review new tickets as they come in and prioritize them. -- Every 2 weeks on Wednesday, a DevSecOps Team representative will attend a Scrum of Scrums with the Apps Tech Leads (and TBD) to review the status of in progress items, flag blockers, and discuss prioritization of new issues. Patrick will give ultimate guidance on what issues need to take priority over others in scenarios, wherever it is unclear. +- Every 2 weeks on Wednesday, a DevSecOps Team representative will attend a Scrum of Scrums with the Apps Tech Leads (and TBD) to review the status of in progress items, flag blockers, and discuss prioritization of new issues. The project engineering lead will give ultimate guidance on what issues need to take priority over others in scenarios, wherever it is unclear. - DevSecOps Team members are not required to attend Apps Team stand ups, sprint planning, refinement, or retro meetings given that they may be doing work across multiple apps at a given time. That said, they will join Apps Team stand ups at their discretion if they need to communicate with the Team, and/or seek clarifications or are requested to join to provide input. They will also pair with Tech Leads to flesh out draft issues, when their input is needed. - DevSecOps Team members will not factor into app Team velocity calculations. - The DevSecOps Team will conduct their own internal stand ups for 10-15 minutes and other recurring Agile ceremonies. From 30702c2f2c87be403140ddebf0d3f0a408b27c1a Mon Sep 17 00:00:00 2001 From: matts <22215332+devopsmatt@users.noreply.github.com> Date: Thu, 3 Oct 2024 14:19:44 -0700 Subject: [PATCH 08/14] Update links to absolute URLs in DevSecOps-WoW.md Changed relative URL links to the DevSecOps-issue.md issue template to absolute, so they work from outside of the repo. --- DevSecOps-WoW.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DevSecOps-WoW.md b/DevSecOps-WoW.md index 0d8a026b09c..5127c0d47a0 100644 --- a/DevSecOps-WoW.md +++ b/DevSecOps-WoW.md @@ -7,9 +7,9 @@ ## Requesting DevSecOps Support -- Quick questions that take <5 min to answer can be asked in Slack and answered by anyone with knowledge, but any issues that need investigation or that may take longer than 5 minutes to gather context and resolve should be opened in ticket form using the [DevSecOps-issue.md](../ISSUE_TEMPLATE/DevSecOps-issue.md) ticket template. +- Quick questions that take <5 min to answer can be asked in Slack and answered by anyone with knowledge, but any issues that need investigation or that may take longer than 5 minutes to gather context and resolve should be opened in ticket form using the [DevSecOps-issue.md](https://github.com/CDCgov/prime-reportstream/blob/master/.github/ISSUE_TEMPLATE/DevSecOps-issue.md) ticket template. - Please avoid using DMs to communicate directly with the DevSecOps Team - instead post communication in the shared #prime-devops channel and tag the appropriate individual(s). -- Apps Teams will request DevSecOps support via the [DevSecOps-issue.md](../ISSUE_TEMPLATE/DevSecOps-issue.md) Github ticket template. The template will automatically add appropriate labels and assign the ticket to the DevSecOps Team. +- Apps Teams will request DevSecOps support via the [DevSecOps-issue.md](https://github.com/CDCgov/prime-reportstream/blob/master/.github/ISSUE_TEMPLATE/DevSecOps-issue.md) Github ticket template. The template will automatically add appropriate labels and assign the ticket to the DevSecOps Team. - The DevSecOps Team will track all DevSecOps issues on the ReportStream board that are tagged with the `DevSecOps` and `reportstream` labels. - When new issues are created using the DevSecOps issue template, the DevSecOps Team will receive an automatic Slack notification in a separate DevSecOps alert channel (TBD). From 004fc4938e8feb016c42e9185ff1e47aa1a5c765 Mon Sep 17 00:00:00 2001 From: devopsmatt Date: Fri, 4 Oct 2024 12:53:59 -0700 Subject: [PATCH 09/14] moving DevSecOps-WoW.md to SharePoint --- DevSecOps-WoW.md | 26 -------------------------- 1 file changed, 26 deletions(-) delete mode 100644 DevSecOps-WoW.md diff --git a/DevSecOps-WoW.md b/DevSecOps-WoW.md deleted file mode 100644 index 5127c0d47a0..00000000000 --- a/DevSecOps-WoW.md +++ /dev/null @@ -1,26 +0,0 @@ -# CDC ReportStream DevSecOps Ways of Working - -## Team Structure - -- The CDC DevSecOps Team includes multiple members, although no person is assigned to a specific Application Team. -- The CDC DevSecOps Team operates on more of a 'help desk' model where they receive tickets and internally triage/prioritize their work. - -## Requesting DevSecOps Support - -- Quick questions that take <5 min to answer can be asked in Slack and answered by anyone with knowledge, but any issues that need investigation or that may take longer than 5 minutes to gather context and resolve should be opened in ticket form using the [DevSecOps-issue.md](https://github.com/CDCgov/prime-reportstream/blob/master/.github/ISSUE_TEMPLATE/DevSecOps-issue.md) ticket template. -- Please avoid using DMs to communicate directly with the DevSecOps Team - instead post communication in the shared #prime-devops channel and tag the appropriate individual(s). -- Apps Teams will request DevSecOps support via the [DevSecOps-issue.md](https://github.com/CDCgov/prime-reportstream/blob/master/.github/ISSUE_TEMPLATE/DevSecOps-issue.md) Github ticket template. The template will automatically add appropriate labels and assign the ticket to the DevSecOps Team. -- The DevSecOps Team will track all DevSecOps issues on the ReportStream board that are tagged with the `DevSecOps` and `reportstream` labels. -- When new issues are created using the DevSecOps issue template, the DevSecOps Team will receive an automatic Slack notification in a separate DevSecOps alert channel (TBD). - -## Workflow Management/Task Prioritization - -- The DevSecOps Team will operate using a Kanban approach and stay within established Work in Progress (WIP) limits. -- While the DevSecOps Team won't be embedded with specific Apps Teams, they will still operate within the same sprint cadence as the Apps Teams. -- DevSecOps will review new tickets as they come in and prioritize them. -- Every 2 weeks on Wednesday, a DevSecOps Team representative will attend a Scrum of Scrums with the Apps Tech Leads (and TBD) to review the status of in progress items, flag blockers, and discuss prioritization of new issues. The project engineering lead will give ultimate guidance on what issues need to take priority over others in scenarios, wherever it is unclear. -- DevSecOps Team members are not required to attend Apps Team stand ups, sprint planning, refinement, or retro meetings given that they may be doing work across multiple apps at a given time. That said, they will join Apps Team stand ups at their discretion if they need to communicate with the Team, and/or seek clarifications or are requested to join to provide input. They will also pair with Tech Leads to flesh out draft issues, when their input is needed. -- DevSecOps Team members will not factor into app Team velocity calculations. -- The DevSecOps Team will conduct their own internal stand ups for 10-15 minutes and other recurring Agile ceremonies. -- DevSecOps Team members will communicate status of their tickets with the respective App Tech Leads. -- DevSec0ps team shall participate in, and help inform, quarterly program planning exercises From d8b3d3e422c6025f060c41dfe0c2d27fb9ee9b37 Mon Sep 17 00:00:00 2001 From: Matt Date: Mon, 7 Oct 2024 16:56:24 -0700 Subject: [PATCH 10/14] Test GPG signing From ecc6d505738c88a0cace7c69fa0604c2cd2e2f96 Mon Sep 17 00:00:00 2001 From: Gabriel Dorsch Date: Mon, 7 Oct 2024 12:15:54 -0400 Subject: [PATCH 11/14] Rework theNextAction members --- .../test/kotlin/common/ReportNodeBuilder.kt | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/prime-router/src/test/kotlin/common/ReportNodeBuilder.kt b/prime-router/src/test/kotlin/common/ReportNodeBuilder.kt index 340261bce85..616773f1e6d 100644 --- a/prime-router/src/test/kotlin/common/ReportNodeBuilder.kt +++ b/prime-router/src/test/kotlin/common/ReportNodeBuilder.kt @@ -25,7 +25,7 @@ class ReportGraphBuilder { private lateinit var theTopic: Topic private lateinit var theFormat: MimeFormat private lateinit var theSender: Sender - private lateinit var theNextAction: TaskAction + private var theNextAction: TaskAction? = null fun topic(topic: Topic) { this.theTopic = topic @@ -85,13 +85,7 @@ class ReportGraphBuilder { .setItemCount(theSubmission.theItemCount) .setExternalName("test-external-name") .setBodyUrl(theSubmission.theReportBlobUrl) - .setNextAction( - if (::theNextAction.isInitialized) { - theNextAction - } else { - theSubmission.reportGraphNodes.firstOrNull()?.theAction - } - ) + .setNextAction(theNextAction ?: theSubmission.reportGraphNodes.firstOrNull()?.theAction) .setCreatedAt(OffsetDateTime.now()) dbAccess.insertReportFile( reportFile, txn, action @@ -142,13 +136,7 @@ class ReportGraphBuilder { .setExternalName("test-external-name") .setBodyUrl(node.theReportBlobUrl) .setTransportResult(node.theTransportResult) - .setNextAction( - if (node.theNextAction != null) { - node.theNextAction - } else { - node.reportGraphNodes.firstOrNull()?.theAction - } - ) + .setNextAction(node.theNextAction ?: node.reportGraphNodes.firstOrNull()?.theAction) .setCreatedAt(graph.node.createdAt.plusMinutes(1)) if (node.receiver != null) { From 5025bf5dc5269757cbe82711aaf80c4a4b88b99f Mon Sep 17 00:00:00 2001 From: Michael Kalish Date: Tue, 8 Oct 2024 11:18:24 -0400 Subject: [PATCH 12/14] 15223: generate empty in convert (#16137) * 15223: generate empty report for garbled convert data * fixup! 15223: generate empty report for garbled convert data * fixup! 15223: generate empty report for garbled convert data * fixup! 15223: generate empty report for garbled convert data * fixup! 15223: generate empty report for garbled convert data * fixup! 15223: generate empty report for garbled convert data --- .../src/main/kotlin/azure/ActionHistory.kt | 16 +++++ .../kotlin/fhirengine/engine/FHIRConverter.kt | 33 +-------- .../common/UniversalPipelineTestUtils.kt | 47 +++++++++---- .../azure/FHIRConverterIntegrationTests.kt | 67 +++++++++++++++++-- .../FHIRDestinationFilterIntegrationTests.kt | 2 +- 5 files changed, 116 insertions(+), 49 deletions(-) diff --git a/prime-router/src/main/kotlin/azure/ActionHistory.kt b/prime-router/src/main/kotlin/azure/ActionHistory.kt index 53c139c788b..3f620279a36 100644 --- a/prime-router/src/main/kotlin/azure/ActionHistory.kt +++ b/prime-router/src/main/kotlin/azure/ActionHistory.kt @@ -448,6 +448,22 @@ class ActionHistory( } } + /** + * Allows tracking of an empty report regardless of where it is generated in the pipeline + * @param report the details of the report + */ + fun trackEmptyReport(report: Report) { + generatingEmptyReport = true + val reportFile = ReportFile() + reportFile.reportId = report.id + reportFile.schemaTopic = report.schema.topic + reportFile.schemaName = "None" + reportFile.itemCount = 0 + reportFile.bodyFormat = report.bodyFormat.toString() + reportFile.nextAction = TaskAction.none + reportsOut[reportFile.reportId] = reportFile + } + /** * Use this to record history info about a newly generated empty [report] for sending to [receiver] that * has requested an empty batch. The [event] will be batch or send. diff --git a/prime-router/src/main/kotlin/fhirengine/engine/FHIRConverter.kt b/prime-router/src/main/kotlin/fhirengine/engine/FHIRConverter.kt index a831daa3b7d..46565172885 100644 --- a/prime-router/src/main/kotlin/fhirengine/engine/FHIRConverter.kt +++ b/prime-router/src/main/kotlin/fhirengine/engine/FHIRConverter.kt @@ -33,7 +33,6 @@ import gov.cdc.prime.router.azure.LookupTableConditionMapper import gov.cdc.prime.router.azure.ProcessEvent import gov.cdc.prime.router.azure.db.Tables import gov.cdc.prime.router.azure.db.enums.TaskAction -import gov.cdc.prime.router.azure.db.tables.pojos.ItemLineage import gov.cdc.prime.router.azure.observability.bundleDigest.BundleDigestExtractor import gov.cdc.prime.router.azure.observability.bundleDigest.FhirPathBundleDigestLabResultExtractorStrategy import gov.cdc.prime.router.azure.observability.context.MDCUtils @@ -277,41 +276,15 @@ class FHIRConverter( }.collect(Collectors.toList()).filterNotNull() } } else { - val nextEvent = ProcessEvent( - Event.EventAction.NONE, - queueMessage.reportId, - Options.None, - emptyMap(), - emptyList() - ) - - // TODO: https://github.com/CDCgov/prime-reportstream/issues/15223 val report = Report( - MimeFormat.FHIR, + format, emptyList(), - 1, + 0, metadata = this.metadata, topic = queueMessage.topic, nextAction = TaskAction.none ) - - // create item lineage - report.itemLineages = listOf( - ItemLineage( - null, - queueMessage.reportId, - 1, - report.id, - 1, - null, - null, - null, - report.getItemHashForRow(1) - ) - ) - - // ensure tracking is set - actionHistory.trackCreatedReport(nextEvent, report) + actionHistory.trackEmptyReport(report) reportEventService.sendReportProcessingError( ReportStreamEventName.REPORT_NOT_PROCESSABLE, report, diff --git a/prime-router/src/test/kotlin/common/UniversalPipelineTestUtils.kt b/prime-router/src/test/kotlin/common/UniversalPipelineTestUtils.kt index 7a40dd6272b..f158a08993b 100644 --- a/prime-router/src/test/kotlin/common/UniversalPipelineTestUtils.kt +++ b/prime-router/src/test/kotlin/common/UniversalPipelineTestUtils.kt @@ -127,6 +127,24 @@ OBX|4|CWE|95421-4^Resides in a congregate care setting^LN^^^^2.69||N^No^HL70136| OBX|5|CWE|95419-8^Has symptoms related to condition of interest^LN^^^^2.69||N^No^HL70136||||||F|||202102090000-0600|||||||||||||||QST SPM|1|0cba76f5-35e0-4a28-803a-2f31308aae9b||258500001^Nasopharyngeal swab^SCT||||71836000^Nasopharyngeal structure (body structure)^SCT^^^^2020-09-01|||||||||202102090000-0600|202102090000-0600""" +// This report is trying to contain two items, but the HL7 is garbled, the first is missing an MSH segment and the second +// has a typo in its MSH segment +@Suppress("ktlint:standard:max-line-length") +const val garbledHL7Record = + """FT|Centers for Disease Control and Prevention|0.1-SNAPSHOT|PRIME ReportStream|0.1-SNAPSHOT||20210210 +PID|1||2a14112c-ece1-4f82-915c-7b3a8d152eda^^^Avante at Ormond Beach^PI||Buckridge^Kareem^Millie^^^^L||19580810|F||2106-3^White^HL70005^^^^2.5.1|688 Leighann Inlet^^South Rodneychester^TX^67071^^^^48077||7275555555:1:^PRN^^roscoe.wilkinson@email.com^1^211^2240784|||||||||U^Unknown^HL70189||||||||N +ORC|RE|73a6e9bd-aaec-418e-813a-0ad33366ca85^6^7^8&F^9|73a6e9bd-aaec-418e-813a-0ad33366ca85|||||||||1629082607^Eddin^Husam^^^^^^CMS&2.16.840.1.113883.3.249&ISO^^^^NPI||^WPN^^^1^386^6825220|20210209||||||Avante at Ormond Beach|170 North King Road^^Ormond Beach^FL^32174^^^^12127|^WPN^^jbrush@avantecenters.com^1^407^7397506|^^^^32174 +OBR|1|73a6e9bd-aaec-418e-813a-0ad33366ca85|0cba76f5-35e0-4a28-803a-2f31308aae9b|94558-4^SARS-CoV-2 (COVID-19) Ag [Presence] in Respiratory specimen by Rapid immunoassay^LN|||202102090000-0600|202102090000-0600||||||||1629082607^Eddin^Husam^^^^^^CMS&2.16.840.1.113883.3.249&ISO^^^^NPI|^WPN^^^1^386^6825220|||||202102090000-0600|||F +OBX|1|CWE|94558-4^SARS-CoV-2 (COVID-19) Ag [Presence] in Respiratory specimen by Rapid immunoassay^LN||260415000^Not detected^SCT|||N^Normal (applies to non-numeric results)^HL70078|||F|||202102090000-0600|||CareStart COVID-19 Antigen test_Access Bio, Inc._EUA^^99ELR||202102090000-0600||||Avante at Ormond Beach^^^^^CLIA&2.16.840.1.113883.4.7&ISO^^^^10D0876999^CLIA|170 North King Road^^Ormond Beach^FL^32174^^^^12127 +OBX|2|CWE|95418-0^Whether patient is employed in a healthcare setting^LN^^^^2.69||Y^Yes^HL70136||||||F|||202102090000-0600|||||||||||||||QST +OBX|3|CWE|95417-2^First test for condition of interest^LN^^^^2.69||Y^Yes^HL70136||||||F|||202102090000-0600|||||||||||||||QST +OBX|4|CWE|95421-4^Resides in a congregate care setting^LN^^^^2.69||N^No^HL70136||||||F|||202102090000-0600|||||||||||||||QST +OBX|5|CWE|95419-8^Has symptoms related to condition of interest^LN^^^^2.69||N^No^HL70136||||||F|||202102090000-0600|||||||||||||||QST +SPM|1|0cba76f5-35e0-4a28-803a-2f31308aae9b||258500001^Nasopharyngeal swab^SCT||||71836000^Nasopharyngeal structure (body structure)^SCT^^^^2020-09-01|||||||||202102090000-0600|202102090000-0600 +SH|^~\&#!|CDC PRIME - Atlanta, Georgia (Dekalb)^2.16.840.1.114222.4.1.237821^ISO|Avante at Ormond Beach^10D0876999^CLIA|PRIME_DOH|Prime ReportStream|20210210170737||ORU^R01^ORU_R01|371784|P|2.5.1|||NE|NE|USA||||PHLabReportNoAck^ELR_Receiver^2.16.840.1.113883.9.99^ISO +SFT|Centers for Disease Control and Prevention|0.1-SNAPSHOT|PRIME ReportStream|0.1-SNAPSHOT||20210210 +PID|1||2a14112c-ece1-4f82-915c-7b3a8d152eda^^^Avante at Ormond Beach^PI||Buckridge^Kareem^Millie^^^^L||19580810|F||2106-3^White^HL70005^^^^2.5.1|688 Leighann Inlet^^South Rodneychester^TX^67071^^^^48077||7275555555:1:^PRN^^roscoe.wilkinson@email.com^1^211^2240784|||||||||U^Unknown^HL70189||||||||N""" + @Suppress("ktlint:standard:max-line-length") const val validRadxMarsHL7Message = """MSH|^~\&|MMTC.PROD^2.16.840.1.113883.3.8589.4.2.106.1^ISO|CAREEVOLUTION^00Z0000024^CLIA|AIMS.INTEGRATION.PRD^2.16.840.1.114222.4.3.15.1^ISO|AIMS.PLATFORM^2.16.840.1.114222.4.1.217446^ISO|20240403205305+0000||ORU^R01^ORU_R01|20240403205305_dba7572cc6334f1ea0744c5f235c823e|P|2.5.1|||NE|NE|||||PHLabReport-NoAck^ELR251R1_Rcvr_Prof^2.16.840.1.113883.9.11^ISO @@ -257,12 +275,13 @@ object UniversalPipelineTestUtils { ) /** - * fetch child reports associated with a [parent] report and ensure we find an [expected] number of children + * fetch child reports associated with a [parent] report and ensure we find an [expectedItems] number of children */ fun fetchChildReports( parent: Report, txn: DataAccessTransaction, - expected: Int? = null, + expectedItems: Int? = null, + expectedReports: Int = 1, ): List { val itemLineages = DSL .using(txn) @@ -271,15 +290,15 @@ object UniversalPipelineTestUtils { .where(ItemLineage.ITEM_LINEAGE.PARENT_REPORT_ID.eq(parent.id)) .fetchInto(gov.cdc.prime.router.azure.db.tables.pojos.ItemLineage::class.java) - if (expected != null) { - assertThat(itemLineages).hasSize(expected) - assertThat(itemLineages.map { it.childIndex }).isEqualTo(MutableList(expected) { 1 }) + if (expectedItems != null) { + assertThat(itemLineages).hasSize(expectedItems) + assertThat(itemLineages.map { it.childIndex }).isEqualTo(MutableList(expectedItems) { 1 }) // itemCount is on the report created by the test. It will not be null. if (parent.itemCount > 1) { - assertThat(itemLineages.map { it.parentIndex }).isEqualTo((1..expected).toList()) + assertThat(itemLineages.map { it.parentIndex }).isEqualTo((1..expectedItems).toList()) } else { - assertThat(itemLineages.map { it.parentIndex }).isEqualTo(MutableList(expected) { 1 }) + assertThat(itemLineages.map { it.parentIndex }).isEqualTo(MutableList(expectedItems) { 1 }) } } @@ -290,9 +309,7 @@ object UniversalPipelineTestUtils { .where(ReportLineage.REPORT_LINEAGE.PARENT_REPORT_ID.eq(parent.id)) .fetchInto(gov.cdc.prime.router.azure.db.tables.pojos.ReportLineage::class.java) - if (expected != null) { - assertThat(reportLineages).hasSize(expected) - } + assertThat(reportLineages).hasSize(expectedReports) val childReportIds = reportLineages.map { it.childReportId @@ -307,11 +324,13 @@ object UniversalPipelineTestUtils { ) ) .fetchInto(ReportFile::class.java) - if (expected != null) { - assertThat(reportFiles).hasSize(expected) + + assertThat(reportFiles).hasSize(expectedReports) + + if (expectedItems != 0) { + assertThat(itemLineages).transform { lineages -> lineages.map { it.childReportId }.sorted() } + .isEqualTo(reportFiles.map { it.reportId }.sorted()) } - assertThat(itemLineages).transform { lineages -> lineages.map { it.childReportId }.sorted() } - .isEqualTo(reportFiles.map { it.reportId }.sorted()) return reportFiles } diff --git a/prime-router/src/test/kotlin/fhirengine/azure/FHIRConverterIntegrationTests.kt b/prime-router/src/test/kotlin/fhirengine/azure/FHIRConverterIntegrationTests.kt index 69b6dbe7182..772dcec6796 100644 --- a/prime-router/src/test/kotlin/fhirengine/azure/FHIRConverterIntegrationTests.kt +++ b/prime-router/src/test/kotlin/fhirengine/azure/FHIRConverterIntegrationTests.kt @@ -49,6 +49,7 @@ import gov.cdc.prime.router.common.cleanHL7Record import gov.cdc.prime.router.common.cleanHL7RecordConverted import gov.cdc.prime.router.common.cleanHL7RecordConvertedAndTransformed import gov.cdc.prime.router.common.conditionCodedValidFHIRRecord1 +import gov.cdc.prime.router.common.garbledHL7Record import gov.cdc.prime.router.common.invalidEmptyFHIRRecord import gov.cdc.prime.router.common.invalidHL7Record import gov.cdc.prime.router.common.invalidHL7RecordConverted @@ -257,7 +258,7 @@ class FHIRConverterIntegrationTests { ReportStreamTestDatabaseContainer.testDatabaseAccess.transact { txn -> val (routedReports, unroutedReports) = fetchChildReports( - receiveReport, txn, 4 + receiveReport, txn, 4, 4 ).partition { it.nextAction != TaskAction.none } assertThat(routedReports).hasSize(2) routedReports.forEach { @@ -441,7 +442,7 @@ class FHIRConverterIntegrationTests { ReportStreamTestDatabaseContainer.testDatabaseAccess.transact { txn -> val (routedReports, unroutedReports) = fetchChildReports( - receiveReport, txn, 4 + receiveReport, txn, 4, 4 ).partition { it.nextAction != TaskAction.none } assertThat(routedReports).hasSize(2) routedReports.forEach { @@ -584,7 +585,7 @@ class FHIRConverterIntegrationTests { ReportStreamTestDatabaseContainer.testDatabaseAccess.transact { txn -> val (routedReports, notRouted) = fetchChildReports( - receiveReport, txn, 2 + receiveReport, txn, 2, 2 ).partition { it.nextAction != TaskAction.none } with(routedReports.single()) { @@ -711,7 +712,7 @@ class FHIRConverterIntegrationTests { fhirFunctions.process(queueMessage, 1, createFHIRConverter(), ActionHistory(TaskAction.convert)) ReportStreamTestDatabaseContainer.testDatabaseAccess.transact { txn -> - val routedReports = fetchChildReports(receiveReport, txn, 2) + val routedReports = fetchChildReports(receiveReport, txn, 2, 2) routedReports.forEach { assertThat(it.nextAction).isEqualTo(TaskAction.destination_filter) assertThat(it.receivingOrg).isEqualTo(null) @@ -804,4 +805,62 @@ class FHIRConverterIntegrationTests { assertThat(report.bodyFormat).isEqualTo("FHIR") } } + + @Test + fun `test should gracefully handle a case where number of items is unknown`() { + val receivedReportContents = garbledHL7Record + val receiveBlobUrl = BlobAccess.uploadBlob( + "receive/happy-path.hl7", + receivedReportContents.toByteArray(), + getBlobContainerMetadata() + ) + + val receiveReport = setupConvertStep(MimeFormat.HL7, hl7Sender, receiveBlobUrl, 1) + val queueMessage = generateQueueMessage(receiveReport, receivedReportContents, hl7Sender) + val fhirFunctions = createFHIRFunctionsInstance() + + fhirFunctions.process(queueMessage, 1, createFHIRConverter(), ActionHistory(TaskAction.convert)) + + verify(exactly = 0) { + QueueAccess.sendMessage(any(), any()) + } + ReportStreamTestDatabaseContainer.testDatabaseAccess.transact { txn -> + val report = fetchChildReports(receiveReport, txn, 0).single() + assertThat(report.nextAction).isEqualTo(TaskAction.none) + assertThat(report.receivingOrg).isEqualTo(null) + assertThat(report.receivingOrgSvc).isEqualTo(null) + assertThat(report.schemaName).isEqualTo("None") + assertThat(report.schemaTopic).isEqualTo(Topic.FULL_ELR) + assertThat(report.bodyFormat).isEqualTo("HL7") + } + } + + @Test + fun `test should gracefully handle a case with an empty contents`() { + val receivedReportContents = " " + val receiveBlobUrl = BlobAccess.uploadBlob( + "receive/happy-path.hl7", + receivedReportContents.toByteArray(), + getBlobContainerMetadata() + ) + + val receiveReport = setupConvertStep(MimeFormat.HL7, hl7Sender, receiveBlobUrl, 1) + val queueMessage = generateQueueMessage(receiveReport, receivedReportContents, hl7Sender) + val fhirFunctions = createFHIRFunctionsInstance() + + fhirFunctions.process(queueMessage, 1, createFHIRConverter(), ActionHistory(TaskAction.convert)) + + verify(exactly = 0) { + QueueAccess.sendMessage(any(), any()) + } + ReportStreamTestDatabaseContainer.testDatabaseAccess.transact { txn -> + val report = fetchChildReports(receiveReport, txn, 0, 1).single() + assertThat(report.nextAction).isEqualTo(TaskAction.none) + assertThat(report.receivingOrg).isEqualTo(null) + assertThat(report.receivingOrgSvc).isEqualTo(null) + assertThat(report.schemaName).isEqualTo("None") + assertThat(report.schemaTopic).isEqualTo(Topic.FULL_ELR) + assertThat(report.bodyFormat).isEqualTo("HL7") + } + } } \ No newline at end of file diff --git a/prime-router/src/test/kotlin/fhirengine/azure/FHIRDestinationFilterIntegrationTests.kt b/prime-router/src/test/kotlin/fhirengine/azure/FHIRDestinationFilterIntegrationTests.kt index 76d52f0b353..f86cc54a2bf 100644 --- a/prime-router/src/test/kotlin/fhirengine/azure/FHIRDestinationFilterIntegrationTests.kt +++ b/prime-router/src/test/kotlin/fhirengine/azure/FHIRDestinationFilterIntegrationTests.kt @@ -194,7 +194,7 @@ class FHIRDestinationFilterIntegrationTests : Logging { // check results ReportStreamTestDatabaseContainer.testDatabaseAccess.transact { txn -> - val routedReports = fetchChildReports(report, txn, 2) + val routedReports = fetchChildReports(report, txn, 2, 2) with(routedReports.first()) { assertThat(this.nextAction).isEqualTo(TaskAction.receiver_filter) assertThat(this.receivingOrg).isEqualTo("phd") From 0de971dca20d004f9283548ea8e98e8b3abbb263 Mon Sep 17 00:00:00 2001 From: etanb Date: Wed, 9 Oct 2024 16:28:04 -0700 Subject: [PATCH 13/14] remove .skips --- .../authenticated/daily-data-page-user-flow.spec.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/frontend-react/e2e/spec/chromium-only/authenticated/daily-data-page-user-flow.spec.ts b/frontend-react/e2e/spec/chromium-only/authenticated/daily-data-page-user-flow.spec.ts index 1a7001cadde..c9aff20e410 100644 --- a/frontend-react/e2e/spec/chromium-only/authenticated/daily-data-page-user-flow.spec.ts +++ b/frontend-react/e2e/spec/chromium-only/authenticated/daily-data-page-user-flow.spec.ts @@ -206,7 +206,7 @@ test.describe( ); }); - test.skip("clears 'Report ID'", async ({ dailyDataPage }) => { + test("clears 'Report ID'", async ({ dailyDataPage }) => { // Search by Report ID const reportId = await tableDataCellValue(dailyDataPage.page, 0, 0); await searchInput(dailyDataPage.page).fill(reportId); @@ -289,7 +289,7 @@ test.describe( expect(await tableDataCellValue(dailyDataPage.page, 0, 0)).toEqual(reportId); }); - test.skip("returns match for Filename", async ({ dailyDataPage }) => { + test("returns match for Filename", async ({ dailyDataPage }) => { // Filename search is currently broken and being tracked // in ticket #15644 const fileName = await tableDataCellValue(dailyDataPage.page, 0, 4); @@ -302,7 +302,8 @@ test.describe( // Check filter status lists receiver value const filterStatusText = filterStatus([fileName]); - await expect(dailyDataPage.page.getByTestId("filter-status")).toContainText(filterStatusText); + const actualText = await dailyDataPage.page.getByTestId("filter-status").textContent(); + expect(filterStatusText).toContain(actualText); //Check table data matches search expect(await tableDataCellValue(dailyDataPage.page, 0, 4)).toEqual(fileName); From 031c752af9cd8d1c62d466ea8031d335fb6fad17 Mon Sep 17 00:00:00 2001 From: Michael Kalish Date: Fri, 11 Oct 2024 10:48:53 -0400 Subject: [PATCH 14/14] 14609: findDetailedDeliveryHistory handles actions that produce multiple reports (#16188) * 14609: findDetailedDeliveryHistory handles actions that produce multiple reports * fixup! 14609: findDetailedDeliveryHistory handles actions that produce multiple reports * fixup! 14609: findDetailedDeliveryHistory handles actions that produce multiple reports --- .../main/kotlin/history/DeliveryHistory.kt | 24 ++++++++++ .../kotlin/history/azure/DeliveryFacade.kt | 29 ++++++++--- .../kotlin/history/azure/DeliveryFunction.kt | 2 +- .../history/azure/DeliveryFacadeTests.kt | 48 +++++++++++++++++++ .../history/azure/DeliveryFunctionTests.kt | 8 ++-- 5 files changed, 100 insertions(+), 11 deletions(-) diff --git a/prime-router/src/main/kotlin/history/DeliveryHistory.kt b/prime-router/src/main/kotlin/history/DeliveryHistory.kt index f5778d5f5b0..d90227fc436 100644 --- a/prime-router/src/main/kotlin/history/DeliveryHistory.kt +++ b/prime-router/src/main/kotlin/history/DeliveryHistory.kt @@ -7,6 +7,8 @@ import com.fasterxml.jackson.annotation.JsonProperty import com.fasterxml.jackson.annotation.JsonPropertyOrder import gov.cdc.prime.router.Receiver import gov.cdc.prime.router.Topic +import gov.cdc.prime.router.azure.db.tables.pojos.Action +import gov.cdc.prime.router.azure.db.tables.pojos.ReportFile import java.time.OffsetDateTime /** @@ -65,6 +67,28 @@ class DeliveryHistory( schema_topic, itemCount ) { + + companion object { + fun createDeliveryHistoryFromReportAndAction( + reportFile: ReportFile, + action: Action, + ): DeliveryHistory { + return DeliveryHistory( + actionId = action.actionId, + createdAt = action.createdAt, + receivingOrg = reportFile.receivingOrg, + receivingOrgSvc = reportFile.receivingOrgSvc, + externalName = action.externalName, + reportId = reportFile.reportId.toString(), + schema_topic = reportFile.schemaTopic, + itemCount = reportFile.itemCount, + bodyUrl = reportFile.bodyUrl, + schemaName = reportFile.schemaName, + bodyFormat = reportFile.bodyFormat + ) + } + } + @JsonIgnore private val DAYS_TO_SHOW = 30L diff --git a/prime-router/src/main/kotlin/history/azure/DeliveryFacade.kt b/prime-router/src/main/kotlin/history/azure/DeliveryFacade.kt index 3b8e11bbf94..17421529b50 100644 --- a/prime-router/src/main/kotlin/history/azure/DeliveryFacade.kt +++ b/prime-router/src/main/kotlin/history/azure/DeliveryFacade.kt @@ -19,7 +19,7 @@ import java.util.* */ class DeliveryFacade( private val dbDeliveryAccess: DatabaseDeliveryAccess = DatabaseDeliveryAccess(), - dbAccess: DatabaseAccess = BaseEngine.databaseAccessSingleton, + private val dbAccess: DatabaseAccess = BaseEngine.databaseAccessSingleton, val reportService: ReportService = ReportService(), ) : ReportFileFacade( dbAccess @@ -91,17 +91,34 @@ class DeliveryFacade( /** * Get expanded details for a single report * + * @param id either a report id (UUID) or action id (Long) * @param deliveryId id for the delivery being used * @return Report details */ fun findDetailedDeliveryHistory( + id: String, deliveryId: Long, ): DeliveryHistory? { - val deliveryHistory = dbDeliveryAccess.fetchAction( - deliveryId, - orgName = null, - DeliveryHistory::class.java - ) + // This functionality is handling the fact that the calling function supports loading the history either + // by the action id or report id + val reportFileId = try { + UUID.fromString(id) + } catch (ex: IllegalArgumentException) { + null + } + + val deliveryHistory = if (reportFileId != null) { + val action = dbAccess.fetchAction(deliveryId) + val reportFile = dbAccess.fetchReportFile(reportFileId) + DeliveryHistory.createDeliveryHistoryFromReportAndAction(reportFile, action!!) + } else { + dbDeliveryAccess.fetchAction( + deliveryId, + orgName = null, + DeliveryHistory::class.java + ) + } + val reportId = deliveryHistory?.reportId if (reportId != null) { val roots = reportService.getRootReports(UUID.fromString(reportId)) diff --git a/prime-router/src/main/kotlin/history/azure/DeliveryFunction.kt b/prime-router/src/main/kotlin/history/azure/DeliveryFunction.kt index 71fd3deaa46..3a8c9c7065e 100644 --- a/prime-router/src/main/kotlin/history/azure/DeliveryFunction.kt +++ b/prime-router/src/main/kotlin/history/azure/DeliveryFunction.kt @@ -167,7 +167,7 @@ class DeliveryFunction( * @return */ override fun singleDetailedHistory(id: String, txn: DataAccessTransaction, action: Action): DeliveryHistory? { - return deliveryFacade.findDetailedDeliveryHistory(action.actionId) + return deliveryFacade.findDetailedDeliveryHistory(id, action.actionId) } @FunctionName("getDeliveriesV1") diff --git a/prime-router/src/test/kotlin/history/azure/DeliveryFacadeTests.kt b/prime-router/src/test/kotlin/history/azure/DeliveryFacadeTests.kt index 7e71b827185..1f7b45bd440 100644 --- a/prime-router/src/test/kotlin/history/azure/DeliveryFacadeTests.kt +++ b/prime-router/src/test/kotlin/history/azure/DeliveryFacadeTests.kt @@ -218,6 +218,53 @@ class DeliveryFacadeTests { ) } + @Test + fun `test findDetailedDeliveryHistory with reportId`() { + val mockDeliveryAccess = mockk() + val mockReportService = mockk() + val mockDbAccess = mockk() + val facade = DeliveryFacade(mockDeliveryAccess, mockDbAccess, mockReportService) + + val reportFile = ReportFile() + reportFile.createdAt = OffsetDateTime.parse("2022-04-13T17:06:10.534Z") + reportFile.reportId = UUID.fromString("b3c8e304-8eff-4882-9000-3645054a30b7") + reportFile.sendingOrg = "DogCow Associates" + reportFile.schemaTopic = Topic.FULL_ELR + reportFile.itemCount = 1 + reportFile.bodyUrl = "body-url" + reportFile.schemaName = "" + reportFile.bodyFormat = "HL7" + reportFile.receivingOrg = "ignore" + reportFile.receivingOrgSvc = "FULL-ELR" + + val action = Action() + action.actionId = 1L + action.createdAt = reportFile.createdAt + action.externalName = "external" + + every { mockDbAccess.fetchAction(any()) } returns action + every { mockDbAccess.fetchReportFile(any()) } returns reportFile + + every { + mockReportService.getRootReports( + any(), + ) + } returns listOf(reportFile) + + val result = facade.findDetailedDeliveryHistory( + reportFile.reportId.toString(), + action.actionId, + ) + assertThat(result?.actionId).isEqualTo(1L) + assertThat(result?.topic).isEqualTo(Topic.FULL_ELR) + assertThat(result?.reportId.toString()).isEqualTo(reportFile.reportId.toString()) + assertThat(result?.reportItemCount).isEqualTo(1) + assertThat(result?.fileName).isEqualTo("body-url") + assertThat(result?.originalIngestion?.first()?.get("ingestionTime")).isEqualTo(reportFile.createdAt) + assertThat(result?.originalIngestion?.first()?.get("reportId")).isEqualTo(reportFile.reportId) + assertThat(result?.originalIngestion?.first()?.get("sendingOrg")).isEqualTo(reportFile.sendingOrg) + } + @Test fun `test findDetailedDeliveryHistory`() { val mockDeliveryAccess = mockk() @@ -258,6 +305,7 @@ class DeliveryFacadeTests { } returns listOf(reportFile) val result = facade.findDetailedDeliveryHistory( + delivery.actionId.toString(), delivery.actionId, ) diff --git a/prime-router/src/test/kotlin/history/azure/DeliveryFunctionTests.kt b/prime-router/src/test/kotlin/history/azure/DeliveryFunctionTests.kt index bcef66ebe44..ec054de8d1c 100644 --- a/prime-router/src/test/kotlin/history/azure/DeliveryFunctionTests.kt +++ b/prime-router/src/test/kotlin/history/azure/DeliveryFunctionTests.kt @@ -497,7 +497,7 @@ class DeliveryFunctionTests : Logging { action.actionName = TaskAction.batch every { mockDeliveryFacade.fetchActionForReportId(any()) } returns action every { mockDeliveryFacade.fetchAction(any()) } returns null // not used for a UUID - every { mockDeliveryFacade.findDetailedDeliveryHistory(any()) } returns returnBody + every { mockDeliveryFacade.findDetailedDeliveryHistory(any(), any()) } returns returnBody every { mockDeliveryFacade.checkAccessAuthorizationForAction(any(), any(), any()) } returns true response = function.getDeliveryDetails(mockRequest, goodUuid) assertThat(response.status).isEqualTo(HttpStatus.OK) @@ -536,7 +536,7 @@ class DeliveryFunctionTests : Logging { // Happy path with a good actionId every { mockDeliveryFacade.fetchActionForReportId(any()) } returns null // not used for an actionId every { mockDeliveryFacade.fetchAction(any()) } returns action - every { mockDeliveryFacade.findDetailedDeliveryHistory(any()) } returns returnBody + every { mockDeliveryFacade.findDetailedDeliveryHistory(any(), any()) } returns returnBody every { mockDeliveryFacade.checkAccessAuthorizationForAction(any(), any(), any()) } returns true response = function.getDeliveryDetails(mockRequest, goodActionId) assertThat(response.status).isEqualTo(HttpStatus.OK) @@ -713,7 +713,7 @@ class DeliveryFunctionTests : Logging { every { mockDeliveryFacade.fetchActionForReportId(any()) } returns action every { mockDeliveryFacade.fetchAction(any()) } returns null // not used for a UUID - every { mockDeliveryFacade.findDetailedDeliveryHistory(any()) } returns returnBody + every { mockDeliveryFacade.findDetailedDeliveryHistory(any(), any()) } returns returnBody every { mockDeliveryFacade.checkAccessAuthorizationForAction(any(), any(), any()) } returns true val restCreds = mockk() @@ -814,7 +814,7 @@ class DeliveryFunctionTests : Logging { every { mockDeliveryFacade.fetchActionForReportId(any()) } returns action every { mockDeliveryFacade.fetchAction(any()) } returns null // not used for a UUID - every { mockDeliveryFacade.findDetailedDeliveryHistory(any()) } returns null + every { mockDeliveryFacade.findDetailedDeliveryHistory(any(), any()) } returns null every { mockDeliveryFacade.checkAccessAuthorizationForAction(any(), any(), any()) } returns true val restCreds = mockk()