Rework exported data (#365)

* Rework exported data

* Add export note functionality

* fix condition

* updated queries to always show polling station and county info

* Revert ngo changes

Co-authored-by: Irina Borozan <[email protected]>

src/api/VoteMonitor.Api.DataExport/Controllers/DataExportController.cs

@@ -5,12 +5,13 @@
 using System;
 using System.Threading.Tasks;
 using VoteMonitor.Api.DataExport.FileGenerator;
+using VoteMonitor.Api.DataExport.Models;
 using VoteMonitor.Api.DataExport.Queries;
 
 namespace VoteMonitor.Api.DataExport.Controllers
 {
     [Route("api/v1/export")]
-    public class DataExportController : Microsoft.AspNetCore.Mvc.Controller
+    public class DataExportController : Controller
     {
         private readonly IMediator _mediator;
         private readonly ILogger<DataExportController> _logger;
@@ -27,7 +28,7 @@ public DataExportController(IMediator mediator, ILogger<DataExportController> lo
         /// <returns></returns>
         [HttpGet("all")]
         [Authorize("Organizer")]
-        public async Task<IActionResult> GetMyData(int? idNgo, int? idObserver, int? pollingStationNumber, string county, DateTime? from, DateTime? to)
+        public async Task<IActionResult> GetAllData(int? idNgo, int? idObserver, int? pollingStationNumber, string county, DateTime? from, DateTime? to)
         {
             var filter = new GetDataForExport
             {
@@ -47,7 +48,7 @@ public async Task<IActionResult> GetMyData(int? idNgo, int? idObserver, int? pol
             }
             catch (Exception e)
             {
-                _logger.LogError(e, nameof(GetMyData));
+                _logger.LogError(e, nameof(GetAllData));
             }
 
 
@@ -62,5 +63,43 @@ public async Task<IActionResult> GetMyData(int? idNgo, int? idObserver, int? pol
                 fileDownloadName: "data.csv"
             );
         }
+
+        [HttpGet("all/notes")]
+        [Authorize("Organizer")]
+        public async Task<IActionResult> GetAllNotes(int? idNgo, int? idObserver, int? pollingStationNumber, string county, DateTime? from, DateTime? to)
+        {
+            var filter = new GetNotesForExport
+            {
+                NgoId = idNgo,
+                ObserverId = idObserver,
+                PollingStationNumber = pollingStationNumber,
+                County = county,
+                From = from,
+                To = to
+            };
+
+            var csvFileBytes = default(byte[]);
+            try
+            {
+                var data = await _mediator.Send(filter);
+                csvFileBytes = await _mediator.Send(new GenerateNotesCSVFile(data));
+            }
+            catch (Exception e)
+            {
+                _logger.LogError(e, nameof(GetAllNotes));
+            }
+
+
+            if (csvFileBytes == null || csvFileBytes.Length == 0)
+            {
+                return NotFound();
+            }
+
+            return File(
+                fileContents: csvFileBytes,
+                contentType: CsvUtility.CSV_MEDIA_TYPE,
+                fileDownloadName: "notes-data.csv"
+            );
+        }
     }
 }

src/api/VoteMonitor.Api.DataExport/Handlers/CsvGeneratorQueryHandler.cs

@@ -7,7 +7,8 @@
 
 namespace VoteMonitor.Api.DataExport.Handlers
 {
-    public class CsvGeneratorQueryHandler : IRequestHandler<GenerateCSVFile, byte[]>
+    public class CsvGeneratorQueryHandler : IRequestHandler<GenerateCSVFile, byte[]>,
+         IRequestHandler<GenerateNotesCSVFile, byte[]>
     {
         private readonly ICsvGenerator _csvGenerator;
         private readonly ILogger _logger;
@@ -24,5 +25,12 @@ public Task<byte[]> Handle(GenerateCSVFile request, CancellationToken cancellati
 
             return Task.FromResult(fileContents);
         }
+
+        public Task<byte[]> Handle(GenerateNotesCSVFile request, CancellationToken cancellationToken)
+        {
+            var fileContents = _csvGenerator.Export(request.Data, "notes");
+
+            return Task.FromResult(fileContents);
+        }
     }
 }

src/api/VoteMonitor.Api.DataExport/Handlers/DataExportQueryHandler.cs

@@ -13,7 +13,8 @@
 
 namespace VoteMonitor.Api.DataExport.Handlers
 {
-    public class DataExportQueryHandler : IRequestHandler<GetDataForExport, IEnumerable<ExportModelDto>>
+    public class DataExportQueryHandler : IRequestHandler<GetDataForExport, IEnumerable<ExportModelDto>>,
+         IRequestHandler<GetNotesForExport, IEnumerable<NotesExportModel>>
     {
         private readonly VoteMonitorContext _context;
         private readonly ILogger _logger;
@@ -33,11 +34,11 @@ public Task<IEnumerable<ExportModelDto>> Handle(GetDataForExport request, Cancel
 			q.Text as QuestionText,
 			o.Text as [OptionText],
 			a.[Value] as [AnswerFreeText],
-			n.Text as NoteText,
-			na.Path as [NoteAttachmentPath],
 			a.LastModified,
 			a.CountyCode,
-			a.PollingStationNumber
+			a.PollingStationNumber,
+			count(n.Text) as NumberOfNotes,
+			count(na.Path) as NumberOfAttachments
 		FROM 
 			(Answers a 
 			INNER JOIN Observers obs
@@ -97,13 +98,173 @@ LEFT JOIN NotesAttachments na
                 }
             }
 
+            query = query + @"
+                group by obs.Phone ,
+                        obs.IdNgo,
+                        f.Code,
+                        q.Text,
+                        o.Text,
+                        a.[Value],
+                        a.LastModified,
+                        a.CountyCode,
+                        a.PollingStationNumber";
+
             IEnumerable<ExportModelDto> data = Enumerable.Empty<ExportModelDto>();
             using (var db = _context.Database.GetDbConnection())
             {
                 db.Open();
                 data = db.Query<ExportModelDto>(sql: query.ToString(), param: parameters, commandTimeout: 60);
             }
 
+            return Task.FromResult(data);
+        } 
+
+        public Task<IEnumerable<NotesExportModel>> Handle(GetNotesForExport request, CancellationToken cancellationToken)
+        {
+            var queryNotesNotAttachedToQuestions = @"
+            SELECT obs.Phone AS [ObserverPhone],
+                obs.IdNgo,
+                '' AS FormCode,
+                '' AS QuestionText,
+                '' AS [OptionText],
+                '' AS [AnswerFreeText],
+                n.Text AS NoteText,
+                na.Path AS [NoteAttachmentPath],
+                NULL AS LastModified,
+                cc.Code AS CountyCode,
+                ps.Number AS PollingStationNumber
+            FROM Notes n
+            INNER JOIN Observers obs ON n.IdObserver = obs.Id
+            LEFT JOIN NotesAttachments na ON na.NoteId = n.Id
+            INNER JOIN PollingStations ps ON n.IdPollingStation = ps.Id
+            INNER JOIN Counties cc ON ps.IdCounty = cc.Id
+            WHERE obs.IsTestObserver = 0
+            AND n.IdQuestion IS NULL
+            AND n.LastModified >= @from
+            AND obs.IsTestObserver = 0";
+
+            var queryNotesForAnsweredQuestions = @"
+            SELECT obs.Phone AS [ObserverPhone],
+                   obs.IdNgo,
+                   f.Code AS FormCode,
+                   q.Text AS QuestionText,
+                   o.Text AS [OptionText],
+                   a.[Value] AS [AnswerFreeText],
+                   n.Text AS NoteText,
+                   na.Path AS [NoteAttachmentPath],
+                   a.LastModified,
+                   a.CountyCode,
+                   a.PollingStationNumber
+            FROM Notes n
+            INNER JOIN Observers obs ON n.IdObserver = obs.Id
+            INNER JOIN Questions q ON n.IdQuestion = q.Id
+            INNER JOIN OptionsToQuestions oq ON oq.IdQuestion = q.Id
+            INNER JOIN OPTIONS o ON oq.IdOption = o.Id
+            INNER JOIN FormSections fs ON q.IdSection = fs.Id
+            INNER JOIN Forms f ON fs.IdForm = f.Id
+            INNER JOIN Answers a ON a.IdOptionToQuestion = oq.Id
+            AND a.IdObserver = obs.Id
+            AND n.IdPollingStation = a.IdPollingStation
+            LEFT JOIN NotesAttachments na ON na.NoteId = n.Id
+            WHERE obs.IsTestObserver = 0
+            AND n.IdQuestion IS NOT NULL
+            AND n.LastModified >= @from";
+
+            var queryForNotesAttachedToQuestionsButNoAnswers = @"
+            SELECT obs.Phone AS [ObserverPhone],
+                   obs.IdNgo,
+                   f.Code AS FormCode,
+                   q.Text AS QuestionText,
+                   '' AS [OptionText],
+                   '' AS [AnswerFreeText],
+                   n.Text AS NoteText,
+                   na.Path AS [NoteAttachmentPath],
+                   '' AS LastModified,
+                   cc.Code AS CountyCode,
+                   ps.Number AS PollingStationNumber
+            FROM Notes n
+            INNER JOIN Observers obs ON n.IdObserver = obs.Id
+            INNER JOIN Questions q ON n.IdQuestion = q.Id
+            INNER JOIN FormSections fs ON q.IdSection = fs.Id
+            INNER JOIN Forms f ON fs.IdForm = f.Id
+            LEFT JOIN NotesAttachments na ON na.NoteId = n.Id
+            INNER JOIN PollingStations ps ON n.IdPollingStation = ps.Id
+            INNER JOIN Counties cc ON ps.IdCounty = cc.Id
+            WHERE NOT EXISTS
+                (SELECT *
+                 FROM Answers a
+                 INNER JOIN OptionsToQuestions oq ON oq.Id = a.IdOptionToQuestion
+                 AND oq.IdQuestion = q.Id
+                 WHERE a.IdObserver = obs.id )
+                    AND obs.IsTestObserver = 0
+                 AND n.LastModified >= @from";
+
+            var parameters = new DynamicParameters();
+            parameters.Add("from", request.From ?? new DateTime(2019, 11, 08, 6, 0, 0));
+
+            if (request.ApplyFilters)
+            {
+                if (request.To.HasValue)
+                {
+                    queryNotesNotAttachedToQuestions += " AND n.LastModified <= @to ";
+                    queryNotesForAnsweredQuestions += " AND n.LastModified <= @to ";
+                    queryForNotesAttachedToQuestionsButNoAnswers += " AND n.LastModified <= @to ";
+                    parameters.Add("to", request.To ?? DateTime.Now.AddDays(2));
+                }
+
+                if (request.ObserverId.HasValue)
+                {
+                    queryNotesNotAttachedToQuestions += " AND obs.Id = @ObserverId ";
+                    queryNotesForAnsweredQuestions += " AND obs.Id = @ObserverId ";
+                    queryForNotesAttachedToQuestionsButNoAnswers += " AND obs.Id = @ObserverId ";
+                    parameters.Add("ObserverId", request.ObserverId);
+                }
+
+                if (request.NgoId.HasValue)
+                {
+                    queryNotesNotAttachedToQuestions += " AND obs.IdNgo = @IdNgo ";
+                    queryNotesForAnsweredQuestions += " AND obs.IdNgo = @IdNgo ";
+                    queryForNotesAttachedToQuestionsButNoAnswers += " AND obs.IdNgo = @IdNgo ";
+                    parameters.Add("IdNgo", request.NgoId);
+                }
+
+                if (!string.IsNullOrEmpty(request.County))
+                {
+                    queryNotesNotAttachedToQuestions += " AND obs.IdNgo = @IdNgo ";
+                    queryNotesForAnsweredQuestions += " AND obs.IdNgo = @IdNgo ";
+                    queryForNotesAttachedToQuestionsButNoAnswers += " AND obs.IdNgo = @IdNgo ";
+                    parameters.Add("County", request.County);
+                }
+
+                if (request.PollingStationNumber.HasValue)
+                {
+                    // PollingStationNumber is for answers only if note does not lead to an answer do not use it
+                    queryNotesNotAttachedToQuestions += " AND 1=2 ";
+
+                    queryNotesForAnsweredQuestions += " AND a.PollingStationNumber = @PollingStationNumber ";
+
+                    // PollingStationNumber is for answers only if note does not lead to an answer do not use it
+                    queryForNotesAttachedToQuestionsButNoAnswers += " AND 1=2 ";
+
+                    parameters.Add("PollingStationNumber", request.PollingStationNumber);
+                }
+            }
+
+            var query = @$"
+                {queryNotesNotAttachedToQuestions}
+                UNION
+                {queryNotesForAnsweredQuestions}
+                UNION
+                {queryForNotesAttachedToQuestionsButNoAnswers}
+                ";
+
+            IEnumerable<NotesExportModel> data = Enumerable.Empty<NotesExportModel>();
+            using (var db = _context.Database.GetDbConnection())
+            {
+                db.Open();
+                data = db.Query<NotesExportModel>(sql: query.ToString(), param: parameters, commandTimeout: 60);
+            }
+
             return Task.FromResult(data);
         }
     }

src/api/VoteMonitor.Api.DataExport/Models/ExportModelDto.cs

@@ -14,10 +14,12 @@ public class ExportModelDto
         public string QuestionText { get; set; }
         public string OptionText { get; set; }
         public string AnswerFreeText { get; set; }
-        public string NoteText { get; set; }
-        public string NoteAttachmentPath { get; set; }
         public DateTime LastModified { get; set; }
         public string CountyCode { get; set; }
         public int PollingStationNumber { get; set; }
-    }
+        public bool HasNotes => NumberOfNotes > 0;
+        public int NumberOfNotes { get; set; }
+        public bool HasAttachments => NumberOfAttachments > 0;
+        public int NumberOfAttachments { get; set; }
+    }    
 }

src/api/VoteMonitor.Api.DataExport/Models/NotesExportModel.cs

@@ -0,0 +1,19 @@
+using System;
+
+namespace VoteMonitor.Api.DataExport.Models
+{
+    public class NotesExportModel
+    {
+        public string ObserverPhone { get; set; }
+        public int IdNgo { get; set; }
+        public string FormCode { get; set; }
+        public string QuestionText { get; set; }
+        public string OptionText { get; set; }
+        public string AnswerFreeText { get; set; }
+        public string NoteText { get; set; }
+        public string NoteAttachmentPath { get; set; }
+        public DateTime LastModified { get; set; }
+        public string CountyCode { get; set; }
+        public int PollingStationNumber { get; set; }
+    }
+}

src/api/VoteMonitor.Api.DataExport/Queries/GenerateCSVFile.cs

@@ -13,4 +13,5 @@ public GenerateCSVFile(IEnumerable<ExportModelDto> data)
             Data = data;
         }
     }
+
 }

src/api/VoteMonitor.Api.DataExport/Queries/GenerateNotesCSVFile.cs

@@ -0,0 +1,17 @@
+using MediatR;
+using System.Collections.Generic;
+using VoteMonitor.Api.DataExport.Models;
+
+namespace VoteMonitor.Api.DataExport.Queries
+{
+    public class GenerateNotesCSVFile : IRequest<byte[]>
+    {
+        public IEnumerable<NotesExportModel> Data { get; }
+
+        public GenerateNotesCSVFile(IEnumerable<NotesExportModel> data)
+        {
+            Data = data;
+        }
+    }
+
+}

src/api/VoteMonitor.Api.DataExport/Queries/GetDataForExport.cs

@@ -16,5 +16,5 @@ public class GetDataForExport : IRequest<IEnumerable<ExportModelDto>>
 
         public bool ApplyFilters => NgoId.HasValue || ObserverId.HasValue || PollingStationNumber.HasValue ||
                                     From.HasValue || To.HasValue || !string.IsNullOrEmpty(County);
-    }
+    } 
 }

src/api/VoteMonitor.Api.DataExport/Queries/GetNotesForExport.cs

@@ -0,0 +1,20 @@
+using MediatR;
+using System;
+using System.Collections.Generic;
+using VoteMonitor.Api.DataExport.Models;
+
+namespace VoteMonitor.Api.DataExport.Queries
+{
+    public class GetNotesForExport : IRequest<IEnumerable<NotesExportModel>>
+    {
+        public int? NgoId { get; set; }
+        public int? ObserverId { get; set; }
+        public int? PollingStationNumber { get; set; }
+        public string County { get; set; }
+        public DateTime? From { get; set; }
+        public DateTime? To { get; set; }
+
+        public bool ApplyFilters => NgoId.HasValue || ObserverId.HasValue || PollingStationNumber.HasValue ||
+                                    From.HasValue || To.HasValue || !string.IsNullOrEmpty(County);
+    }
+}