Skip to content

Commit

Permalink
AB#1011 Clean up after MartReportImporterWorkerTest
Browse files Browse the repository at this point in the history 
Delete imported reports, thus fixing MartImportedReportsResourceTest.
Properly report MART import errors as a list.
  • Loading branch information
@gjvoosten
gjvoosten committed Jan 15, 2025
1 parent 857f044 commit 50c3553
Show file tree
Hide file tree
Showing 5 changed files with 155 additions and 64 deletions.
2 changes: 1 addition & 1 deletion insertBaseData-psql.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1402,7 +1402,7 @@ INSERT INTO "entityAvatars" ("relatedObjectType", "relatedObjectUuid", "attachme

-- Add mart imported reports for testing
INSERT INTO "martImportedReports" ("personUuid", "reportUuid", "success", "createdAt", "errors") VALUES
('87fdbc6a-3109-4e11-9702-a894d6ca31ef', '59be259b-30b9-4d04-9e21-e8ceb58cbe9c', TRUE, CURRENT_TIMESTAMP, 'some error');
('87fdbc6a-3109-4e11-9702-a894d6ca31ef', '59be259b-30b9-4d04-9e21-e8ceb58cbe9c', TRUE, CURRENT_TIMESTAMP, NULL);

-- Update the link-text indexes
REFRESH MATERIALIZED VIEW CONCURRENTLY "mv_lts_attachments";
Expand Down
20 changes: 20 additions & 0 deletions src/main/java/mil/dds/anet/database/MartImportedReportDao.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,4 +69,24 @@ public int insert(final MartImportedReport martImportedReport) {
}
}

@Transactional
public int delete(final MartImportedReport martImportedReport) {
final Handle handle = getDbHandle();
try {
final StringBuilder sql =
new StringBuilder("/* deleteMartImportedReport */ DELETE FROM \"martImportedReports\" "
+ "WHERE success = :success AND \"createdAt\" = :createdAt");
sql.append(martImportedReport.getPersonUuid() == null ? " AND \"personUuid\" IS NULL"
: " AND \"personUuid\" = :personUuid");
sql.append(martImportedReport.getReportUuid() == null ? " AND \"reportUuid\" IS NULL"
: " AND \"reportUuid\" = :reportUuid");
sql.append(
martImportedReport.getErrors() == null ? " AND errors IS NULL" : " AND errors = :errors");
return handle.createUpdate(sql).bindBean(martImportedReport)
.bind("createdAt", DaoUtils.asLocalDateTime(martImportedReport.getCreatedAt())).execute();
} finally {
closeDbHandle(handle);
}
}

}
70 changes: 39 additions & 31 deletions src/main/java/mil/dds/anet/threads/mart/MartReportImporterWorker.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,28 +108,36 @@ protected void runInternal(Instant now, JobHistory jobHistory, GraphQLContext co

private void processEmailMessage(EmailMessage email) {
final MartImportedReport martImportedReport = new MartImportedReport();
final StringBuilder errors = new StringBuilder();
ReportDto reportDto = null;
final List<String> errors = new ArrayList<>();
try {
email.load();
logger.debug("Processing e-mail sent on: {}", email.getDateTimeCreated());
// Get the report JSON
final Report anetReport =
processReportInfo(getReportInfo(email.getBody().toString()), errors);
reportDto = getReportInfo(email.getBody().toString());
final Report anetReport = processReportInfo(reportDto, errors);
if (anetReport != null) {
processAttachments(email, anetReport, errors);
martImportedReport.setSuccess(true);
martImportedReport.setReport(anetReport);
martImportedReport.setPerson(anetReport.getReportPeople().get(0));
}
} catch (Exception e) {
errors.append("Could not process email").append(email.toString());
errors.add(String.format("Could not process email %s", email));
}
if (!errors.isEmpty()) {
final StringBuilder errorMsg = new StringBuilder();
if (reportDto != null) {
errorMsg.append(String.format("While importing report %s:", reportDto.getUuid()));
}
errorMsg.append(String.format("<ul><li>%s</li></ul>", String.join("</li><li>", errors)));
martImportedReport.setErrors(errorMsg.toString());
}
martImportedReport.setErrors(errors.toString());
martImportedReport.setCreatedAt(Instant.now());
martImportedReportDao.insert(martImportedReport);
}

private void processAttachments(EmailMessage email, Report anetReport, StringBuilder errors) {
private void processAttachments(EmailMessage email, Report anetReport, List<String> errors) {
try {
for (final microsoft.exchange.webservices.data.property.complex.Attachment attachment : email
.getAttachments()) {
Expand All @@ -152,26 +160,27 @@ private void processAttachments(EmailMessage email, Report anetReport, StringBui
attachmentDao.saveContentBlob(anetAttachment.getUuid(),
TikaInputStream.get(fileAttachment.getContent()));
} else {
errors.append("Attachment found in e-mail is not valid: ")
.append(fileAttachment.getName());
errors.add(String.format("Attachment found in e-mail is not valid: %s",
fileAttachment.getName()));
}
}
}
} catch (Exception e) {
errors.append("Could not process report attachments");
errors.add("Could not process report attachments");
}
}

private void getTasks(Map<String, String> martTasks, List<Task> tasks, StringBuilder errors) {
martTasks.keySet().forEach(martTask -> {
final Task task = taskDao.getByUuid(martTask);
private List<Task> getTasks(Map<String, String> martTasks, List<String> errors) {
final List<Task> tasks = new ArrayList<>();
martTasks.forEach((key, value) -> {
final Task task = taskDao.getByUuid(key);
if (task != null) {
tasks.add(task);
} else {
errors.append("Can not find task: '").append(martTasks.get(martTask))
.append("' with uuid: ").append(martTask).append("<br>");
errors.add(String.format("Can not find task: '%s' with uuid: %s", value, key));
}
});
return tasks;
}

private boolean assertAllowedMimeType(final String mimeType) {
Expand Down Expand Up @@ -199,33 +208,34 @@ private ReportDto getReportInfo(String reportJson) {
return report;
}

private Report processReportInfo(ReportDto martReport, StringBuilder errors) {
private Report processReportInfo(ReportDto martReport, List<String> errors) {
// Do we have this report already?
if (reportDao.getByUuid(martReport.getUuid()) != null) {
logger.info("Report with UUID={} already exists", martReport.getUuid());
errors.append("Report with UUID already exists: ").append(martReport.getUuid());
errors.add(String.format("Report with UUID already exists: %s", martReport.getUuid()));
return null;
}

// Try to find the person/s with the email
final List<Person> matchingPersons = personDao.findByEmailAddress(martReport.getEmail());
final List<ReportPerson> reportPeople = new ArrayList<>();

// Validate author organization, if not valid finish
// Validate author organization
final Organization organization = organizationDao.getByUuid(martReport.getOrganizationUuid());
if (organization == null) {
errors.append("Can not find submitter organization: '")
.append(martReport.getOrganizationName()).append("' with uuid: ")
.append(martReport.getOrganizationUuid());
return null;
errors.add(String.format("Can not find submitter organization: '%s' with uuid: %s",
martReport.getOrganizationName(), martReport.getOrganizationUuid()));
}

// Validate report location, it not valid finish
// Validate report location
final Location location = locationDao.getByUuid(martReport.getLocationUuid());
if (location == null) {
errors.append("Can not find report location: ").append(martReport.getLocationName());
.append("' with uuid: ")
.append(martReport.getLocationUuid());
errors.add(String.format("Can not find report location: '%s' with uuid: %s",
martReport.getLocationName(), martReport.getLocationUuid()));
}

// Return early if there are errors
if (!errors.isEmpty()) {
return null;
}

Expand Down Expand Up @@ -285,8 +295,7 @@ private Report processReportInfo(ReportDto martReport, StringBuilder errors) {
anetReport.setAdvisorOrg(organization);

// Report tasks
final List<Task> tasks = new ArrayList<>();
getTasks(martReport.getTasks(), tasks, errors);
final List<Task> tasks = getTasks(martReport.getTasks(), errors);
anetReport.setTasks(tasks);
// Custom fields
anetReport.setCustomFields(martReport.getCustomFields());
Expand All @@ -298,7 +307,7 @@ private Report processReportInfo(ReportDto martReport, StringBuilder errors) {
anetReport = reportDao.insertWithExistingUuid(anetReport);
} catch (Exception e) {
logger.info("Report with UUID={} already exists", martReport.getUuid());
errors.append("Report with UUID already exists: ").append(martReport.getUuid());
errors.add(String.format("Report with UUID already exists: %s", martReport.getUuid()));
return null;
}

Expand All @@ -308,13 +317,12 @@ private Report processReportInfo(ReportDto martReport, StringBuilder errors) {
return anetReport;
}

private void getPersonCountry(Person person, ReportDto martReport, StringBuilder errors) {
private void getPersonCountry(Person person, ReportDto martReport, List<String> errors) {
final LocationSearchQuery searchQuery = new LocationSearchQuery();
searchQuery.setText(martReport.getCountry());
final List<Location> countries = locationDao.search(searchQuery).getList();
if (countries.isEmpty()) {
errors.append("Can not find submitter country '").append(martReport.getCountry())
.append("<br>");
errors.add(String.format("Can not find submitter country '%s'", martReport.getCountry()));
} else {
person.setCountry(countries.get(0));
}
Expand Down
30 changes: 28 additions & 2 deletions src/test/java/mil/dds/anet/test/TestData.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ public static TaskInput createTaskInput(String shortName, String longName, Strin
public static ReportDto createGoodMartReport() {
final ReportDto reportDto = new ReportDto();
// User Info
reportDto.setUuid(UUID.randomUUID().toString());
reportDto.setUuid("231196f5-3b13-45ea-9d73-524d042b16e7");
reportDto.setOrganizationUuid("9a35caa7-a095-4963-ac7b-b784fde4d583");
reportDto.setOrganizationName("Planning Programming, Budgeting and Execution");
reportDto.setRank("OF-6");
Expand All @@ -111,6 +111,7 @@ public static ReportDto createGoodMartReport() {
reportDto.setReportText("Report Text");
reportDto.setEngagementDate(Instant.now());
reportDto.setLocationUuid("0855fb0a-995e-4a79-a132-4024ee2983ff");
reportDto.setLocationName("General Hospital");
reportDto.setCountry("British");
reportDto.setPositionName("MART Team Member");
reportDto.setSubmittedAt(Instant.now());
Expand All @@ -122,24 +123,49 @@ public static ReportDto createGoodMartReport() {
// Tasks
final Map<String, String> tasks = new HashMap<>();
tasks.put("19364d81-3203-483d-a6bf-461d58888c76", "Intelligence");
tasks.put("does not exist", "does not exist");
reportDto.setTasks(tasks);

return reportDto;
}

public static ReportDto createGoodMartReportWithUnknownTask() {
final ReportDto reportDto = createGoodMartReport();
reportDto.setUuid("34faac7c-8c85-4dec-8e9f-57d9254b5ae2");
reportDto.getTasks().put("does not exist", "does not exist");
return reportDto;
}

public static ReportDto createMartReportWrongOrganization() {
final ReportDto reportDto = new ReportDto();
reportDto.setUuid("fb875171-2501-46c9-9246-60dafabb656d");
reportDto.setOrganizationUuid("does not exist");
reportDto.setOrganizationName("does not exist");
reportDto.setLocationUuid("0855fb0a-995e-4a79-a132-4024ee2983ff");
reportDto.setLocationName("General Hospital");
return reportDto;
}

public static ReportDto createMartReportWrongLocation() {
final ReportDto reportDto = new ReportDto();
reportDto.setUuid("2d6c7a19-d878-4792-bdaf-7a73dc3bfc83");
reportDto.setOrganizationUuid("9a35caa7-a095-4963-ac7b-b784fde4d583");
reportDto.setOrganizationName("Planning Programming, Budgeting and Execution");
reportDto.setLocationUuid("does not exist");
reportDto.setLocationName("does not exist");
return reportDto;
}

public static ReportDto createMartReportCompletelyWrong() {
final ReportDto reportDto = new ReportDto();
reportDto.setUuid("68077002-b766-4a79-bcf2-40b7dbffe6e6");
reportDto.setOrganizationUuid("does not exist");
reportDto.setOrganizationName("does not exist");
reportDto.setLocationUuid("does not exist");
reportDto.setLocationName("does not exist");
final Map<String, String> tasks = new HashMap<>();
tasks.put("19364d81-3203-483d-a6bf-461d58888c76", "Intelligence");
tasks.put("does not exist", "does not exist");
reportDto.setTasks(tasks);
return reportDto;
}
}
97 changes: 67 additions & 30 deletions src/test/java/mil/dds/anet/test/integration/mart/MartReportImporterWorkerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,11 +91,14 @@ void setUp() throws Exception {
EmailMessage emailMessage3 =
createMockEmail(TestData.createMartReportWrongOrganization(), false);
EmailMessage emailMessage4 = createMockEmail(TestData.createMartReportWrongLocation(), false);
EmailMessage emailMessage5 = createMockEmail(TestData.createMartReportCompletelyWrong(), false);
EmailMessage emailMessage6 =
createMockEmail(TestData.createGoodMartReportWithUnknownTask(), true);

// Mock the mail exchange server
IMailReceiver iMailReceiverMock = Mockito.mock();
when(iMailReceiverMock.downloadEmails())
.thenReturn(List.of(emailMessage1, emailMessage2, emailMessage3, emailMessage4));
when(iMailReceiverMock.downloadEmails()).thenReturn(List.of(emailMessage1, emailMessage2,
emailMessage3, emailMessage4, emailMessage5, emailMessage6));

martReportImporterWorker = new MartReportImporterWorker(dict, jobHistoryDao, reportDao,
personDao, positionDao, taskDao, organizationDao, locationDao, martImportedReportDao,
Expand All @@ -113,7 +116,7 @@ void testWorker() {
final PersonSearchQueryInput queryPerson =
PersonSearchQueryInput.builder().withOrgUuid(List.of(reportDto.getOrganizationUuid()))
.withEmailNetwork("Internet").withHasBiography(false).withRank("OF-6").build();
AnetBeanList_Person searchResults = withCredentials("arthur",
final AnetBeanList_Person searchResults = withCredentials("arthur",
t -> queryExecutor.personList(getListFields(PersonResourceTest.FIELDS), queryPerson));
assertThat(searchResults.getTotalCount()).isPositive();
Person person = searchResults.getList().get(0);
Expand All @@ -134,44 +137,78 @@ void testWorker() {
assertThat(createdReport.getAttachments().get(0).getFileName()).isEqualTo(ATTACHMENT_NAME);
assertThat(createdReport.getTasks().get(0).getLongName()).isEqualTo("Intelligence");

// Six new records in MartImportedReports, verify them
final List<MartImportedReport> martImportedReports = martImportedReportDao.getAll();

// Four new records in MartImportedReports, verify then
List<MartImportedReport> martImportedReports = martImportedReportDao.getAll();

assertThat(martImportedReports.stream()
.filter(
martImportedReport -> !martImportedReport.isSuccess() && martImportedReport.getErrors()
.equals("Can not find report location: 'does not exist' with uuid: does not exist"))
.hasSize(1);

assertThat(martImportedReports.stream()
.filter(martImportedReport -> !martImportedReport.isSuccess()
&& martImportedReport.getErrors().equals(
"Can not find submitter organization: 'does not exist' with uuid: does not exist")))
.hasSize(1);

assertThat(martImportedReports.stream()
.filter(martImportedReport -> !martImportedReport.isSuccess() && martImportedReport
.getErrors().equals("Report with UUID already exists: " + reportDto.getUuid())))
.hasSize(1);

assertThat(martImportedReports.stream()
List<MartImportedReport> reportList = martImportedReports.stream()
.filter(martImportedReport -> martImportedReport.isSuccess()
&& martImportedReport.getReportUuid().equals(reportDto.getUuid())
&& martImportedReport.getPersonUuid().equals(person.getUuid())
&& martImportedReport.getErrors()
.equals("Can not find task: 'does not exist' with uuid: does not exist<br>")))
.hasSize(1);
&& martImportedReport.getErrors() == null)
.toList();
assertThat(reportList).hasSize(1);
assertThat(martImportedReportDao.delete(reportList.get(0))).isOne();

reportList = martImportedReports.stream().filter(martImportedReport -> !martImportedReport
.isSuccess()
&& martImportedReport.getErrors() != null
&& martImportedReport.getErrors()
.equals("While importing report 231196f5-3b13-45ea-9d73-524d042b16e7:"
+ "<ul><li>Report with UUID already exists: 231196f5-3b13-45ea-9d73-524d042b16e7</li></ul>"))
.toList();
assertThat(reportList).hasSize(1);
assertThat(martImportedReportDao.delete(reportList.get(0))).isOne();

reportList = martImportedReports.stream().filter(martImportedReport -> !martImportedReport
.isSuccess()
&& martImportedReport.getErrors() != null
&& martImportedReport.getErrors()
.equals("While importing report fb875171-2501-46c9-9246-60dafabb656d:"
+ "<ul><li>Can not find submitter organization: 'does not exist' with uuid: does not exist</li></ul>"))
.toList();
assertThat(reportList).hasSize(1);
assertThat(martImportedReportDao.delete(reportList.get(0))).isOne();

reportList = martImportedReports.stream().filter(martImportedReport -> !martImportedReport
.isSuccess()
&& martImportedReport.getErrors() != null
&& martImportedReport.getErrors()
.equals("While importing report 2d6c7a19-d878-4792-bdaf-7a73dc3bfc83:"
+ "<ul><li>Can not find report location: 'does not exist' with uuid: does not exist</li></ul>"))
.toList();
assertThat(reportList).hasSize(1);
assertThat(martImportedReportDao.delete(reportList.get(0))).isOne();

reportList = martImportedReports.stream().filter(martImportedReport -> !martImportedReport
.isSuccess()
&& martImportedReport.getErrors() != null
&& martImportedReport.getErrors()
.equals("While importing report 68077002-b766-4a79-bcf2-40b7dbffe6e6:"
+ "<ul><li>Can not find submitter organization: 'does not exist' with uuid: does not exist</li>"
+ "<li>Can not find report location: 'does not exist' with uuid: does not exist</li></ul>"))
.toList();
assertThat(reportList).hasSize(1);
assertThat(martImportedReportDao.delete(reportList.get(0))).isOne();

reportList = martImportedReports.stream().filter(martImportedReport -> martImportedReport
.isSuccess()
&& martImportedReport.getErrors() != null
&& martImportedReport.getErrors()
.equals("While importing report 34faac7c-8c85-4dec-8e9f-57d9254b5ae2:"
+ "<ul><li>Can not find task: 'does not exist' with uuid: does not exist</li></ul>"))
.toList();
assertThat(reportList).hasSize(1);
assertThat(martImportedReportDao.delete(reportList.get(0))).isOne();
}

private EmailMessage createMockEmail(ReportDto reportDto, boolean withAttachment)
throws ServiceLocalException, IOException {
EmailMessage emailMessageMock = Mockito.mock();
MessageBody messageBody = new MessageBody();
final EmailMessage emailMessageMock = Mockito.mock();
final MessageBody messageBody = new MessageBody();
messageBody.setText(ignoringMapper.writeValueAsString(reportDto));
when(emailMessageMock.getBody()).thenReturn(messageBody);
if (withAttachment) {
AttachmentCollection attachmentCollection = new AttachmentCollection();
final AttachmentCollection attachmentCollection = new AttachmentCollection();
attachmentCollection.addFileAttachment(ATTACHMENT_NAME,
IOUtils.toByteArray(Objects.requireNonNull(
this.getClass().getClassLoader().getResourceAsStream("assets/default_avatar.png"))));
Expand Down

0 comments on commit 50c3553

Please sign in to comment.