From 29262730b2360da41a1d31f0f99c7e29f4a9a1b3 Mon Sep 17 00:00:00 2001 From: "D. Ror" Date: Fri, 5 Mar 2021 10:08:35 -0500 Subject: [PATCH] Use InsertManyAsync() on import to reduce connections to mongo. (#1054) --- Backend.Tests/Mocks/WordRepositoryMock.cs | 19 ++++++++++++++- Backend/Controllers/LiftController.cs | 1 + Backend/Interfaces/ILiftService.cs | 12 +++++++--- Backend/Interfaces/IWordRepository.cs | 2 ++ Backend/Services/LiftApiServices.cs | 29 +++++++++++++++++------ Backend/Services/WordRepository.cs | 29 +++++++++++++++++++++++ 6 files changed, 81 insertions(+), 11 deletions(-) diff --git a/Backend.Tests/Mocks/WordRepositoryMock.cs b/Backend.Tests/Mocks/WordRepositoryMock.cs index 1a2858a301..3ef5ebd719 100644 --- a/Backend.Tests/Mocks/WordRepositoryMock.cs +++ b/Backend.Tests/Mocks/WordRepositoryMock.cs @@ -4,7 +4,6 @@ using System.Threading.Tasks; using BackendFramework.Interfaces; using BackendFramework.Models; -using BackendFramework.Services; namespace Backend.Tests.Mocks { @@ -45,6 +44,15 @@ public Task Create(Word word) return Task.FromResult(word.Clone()); } + public Task> Create(List words) + { + foreach (var w in words) + { + Create(w); + } + return Task.FromResult(words); + } + public Task DeleteAllWords(string projectId) { _words.Clear(); @@ -63,6 +71,15 @@ public Task AddFrontier(Word word) return Task.FromResult(word.Clone()); } + public Task> AddFrontier(List words) + { + foreach (var w in words) + { + AddFrontier(w); + } + return Task.FromResult(words); + } + public Task DeleteFrontier(string projectId, string wordId) { var origLength = _frontier.Count; diff --git a/Backend/Controllers/LiftController.cs b/Backend/Controllers/LiftController.cs index d3e0940f24..83082d63c2 100644 --- a/Backend/Controllers/LiftController.cs +++ b/Backend/Controllers/LiftController.cs @@ -157,6 +157,7 @@ public async Task UploadLiftFile(string projectId, [FromForm] Fil // Import words from lift file var resp = parser.ReadLiftFile(extractedLiftPath.FirstOrDefault()); + await liftMerger.SaveImportEntries(); // Add character set to project from ldml file var proj = await _projectService.GetProject(projectId); diff --git a/Backend/Interfaces/ILiftService.cs b/Backend/Interfaces/ILiftService.cs index 2e00fd7819..4959608fe2 100644 --- a/Backend/Interfaces/ILiftService.cs +++ b/Backend/Interfaces/ILiftService.cs @@ -1,4 +1,5 @@ -using System.Threading.Tasks; +using System.Collections.Generic; +using System.Threading.Tasks; using BackendFramework.Models; using SIL.Lift.Parsing; @@ -6,16 +7,21 @@ namespace BackendFramework.Interfaces { public interface ILiftService { - ILexiconMerger GetLiftImporterExporter( + ILiftMerger GetLiftImporterExporter( string projectId, IProjectService projectService, IWordRepository wordRepo); void LdmlImport(string filePath, string langTag, IProjectService projectService, Project project); Task LiftExport(string projectId, IWordRepository wordRepo, IProjectService projectService); - // Methods to store, retrieve, and delete an export string in a common dictionary + // Methods to store, retrieve, and delete an export string in a common dictionary. void StoreExport(string key, string filePath); string? RetrieveExport(string key); bool DeleteExport(string key); void SetExportInProgress(string key, bool isInProgress); bool IsExportInProgress(string key); } + + public interface ILiftMerger : ILexiconMerger + { + Task> SaveImportEntries(); + } } diff --git a/Backend/Interfaces/IWordRepository.cs b/Backend/Interfaces/IWordRepository.cs index 604e238cf8..3c8fd44505 100644 --- a/Backend/Interfaces/IWordRepository.cs +++ b/Backend/Interfaces/IWordRepository.cs @@ -9,10 +9,12 @@ public interface IWordRepository Task> GetAllWords(string projectId); Task GetWord(string projectId, string wordId); Task Create(Word word); + Task> Create(List words); Task Add(Word word); Task DeleteAllWords(string projectId); Task> GetFrontier(string projectId); Task AddFrontier(Word word); + Task> AddFrontier(List word); Task DeleteFrontier(string projectId, string wordId); } } diff --git a/Backend/Services/LiftApiServices.cs b/Backend/Services/LiftApiServices.cs index d2cf91698c..67835e98d2 100644 --- a/Backend/Services/LiftApiServices.cs +++ b/Backend/Services/LiftApiServices.cs @@ -474,7 +474,7 @@ private static void LdmlExport(string filePath, string vernacularBcp47, List GetLiftImporterExporter( + public ILiftMerger GetLiftImporterExporter( string projectId, IProjectService projectService, IWordRepository wordRepo) { return new LiftMerger(projectId, projectService, wordRepo); @@ -511,12 +511,12 @@ private static void WriteRangeElement( liftRangesWriter.WriteEndElement(); //end range element } - - private sealed class LiftMerger : ILexiconMerger + private sealed class LiftMerger : ILiftMerger { private readonly string _projectId; private readonly IProjectService _projectService; private readonly IWordRepository _wordRepo; + private readonly List _importEntries = new List(); public LiftMerger(string projectId, IProjectService projectService, IWordRepository wordRepo) { @@ -526,9 +526,23 @@ public LiftMerger(string projectId, IProjectService projectService, IWordReposit } /// - /// The meat of lift import is done here. - /// This reads in all necessary attributes of a word and adds it to the database. + /// s are added to the private field + /// during lift import. This saves the contents of _importEntries to the database. + /// + /// The words saved. + public async Task> SaveImportEntries() + { + var savedWords = new List(await _wordRepo.Create(_importEntries)); + _importEntries.Clear(); + return savedWords; + } + + /// + /// A significant portion of lift import is done here. This reads in all necessary + /// attributes to create a word from , adding the result + /// to to be saved to the database. /// + /// public async void FinishEntry(LiftEntry entry) { var newWord = new Word @@ -616,7 +630,8 @@ public async void FinishEntry(LiftEntry entry) { // Splits on the space between the number and name of the semantic domain var splitSemDom = semanticDomainString.Split(" ", 2); - newSense.SemanticDomains.Add(new SemanticDomain { Id = splitSemDom[0], Name = splitSemDom[1] }); + newSense.SemanticDomains.Add( + new SemanticDomain { Id = splitSemDom[0], Name = splitSemDom[1] }); } newWord.Senses.Add(newSense); @@ -652,7 +667,7 @@ public async void FinishEntry(LiftEntry entry) } newWord.ProjectId = _projectId; - await _wordRepo.Create(newWord); + _importEntries.Add(newWord); } /// Creates the object to transfer all the data from a word diff --git a/Backend/Services/WordRepository.cs b/Backend/Services/WordRepository.cs index 13c262854c..b024e58a8c 100644 --- a/Backend/Services/WordRepository.cs +++ b/Backend/Services/WordRepository.cs @@ -76,6 +76,7 @@ private static void PopulateBlankWordTimes(Word word) /// If the Created or Modified time fields are blank, they will automatically calculated using the current /// time. This allows services to set or clear the values before creation to control these fields. /// + /// /// The word created public async Task Create(Word word) { @@ -85,6 +86,24 @@ public async Task Create(Word word) return word; } + /// Adds a list of s to the WordsCollection and Frontier + /// + /// If the Created or Modified time fields are blank, they will automatically calculated using the current + /// time. This allows services to set or clear the values before creation to control these fields. + /// + /// + /// The words created + public async Task> Create(List words) + { + foreach (var w in words) + { + PopulateBlankWordTimes(w); + } + await _wordDatabase.Words.InsertManyAsync(words); + await AddFrontier(words); + return words; + } + /// Adds a only to the WordsCollection /// /// If the Created or Modified time fields are blank, they will automatically calculated using the current @@ -105,6 +124,7 @@ public async Task> GetFrontier(string projectId) } /// Adds a only to the Frontier + /// /// The word created public async Task AddFrontier(Word word) { @@ -112,6 +132,15 @@ public async Task AddFrontier(Word word) return word; } + /// Adds a list of s only to the Frontier + /// + /// The words created + public async Task> AddFrontier(List words) + { + await _wordDatabase.Frontier.InsertManyAsync(words); + return words; + } + /// Removes from the Frontier with specified wordId and projectId /// A bool: success of operation public async Task DeleteFrontier(string projectId, string wordId)