Merge branch 'master' into user-projects

sillsdev · Mar 26, 2024 · c35cc5e · c35cc5e
2 parents d47bafd + 3c646a2
commit c35cc5e
Show file tree

Hide file tree

Showing 116 changed files with 1,952 additions and 676 deletions.
diff --git a/.gitignore b/.gitignore
@@ -81,6 +81,11 @@ src/resources/dictionaries/*.txt
 deploy/scripts/semantic_domains/json/*.json
 database/semantic_domains/*
 
+# Combine installer
+installer/combine-installer.run
+installer/makeself-*
+installer/README.pdf
+
 # Kubernetes Configuration files
 **/site_files/
 **/charts/*.tgz

diff --git a/Backend.Tests/Controllers/SemanticDomainControllerTests.cs b/Backend.Tests/Controllers/SemanticDomainControllerTests.cs
@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using Backend.Tests.Mocks;
 using BackendFramework.Controllers;
 using BackendFramework.Interfaces;
@@ -39,6 +40,23 @@ public void Setup()
             _semDomController = new SemanticDomainController(_semDomRepository);
         }
 
+        [Test]
+        public void GetAllSemanticDomainNamesFound()
+        {
+            var treeNodes = new List<SemanticDomainTreeNode> { new(_semDom) };
+            ((SemanticDomainRepositoryMock)_semDomRepository).SetNextResponse(treeNodes);
+            var names = ((OkObjectResult)_semDomController.GetAllSemanticDomainNames(Lang).Result).Value;
+            Assert.That(names, Has.Count.EqualTo(1));
+            Assert.That(((Dictionary<string, string>)names!)[Id], Is.EqualTo(Name));
+        }
+
+        [Test]
+        public void GetAllSemanticDomainNamesNotFound()
+        {
+            var names = ((OkObjectResult)_semDomController.GetAllSemanticDomainNames(Lang).Result).Value;
+            Assert.That(names, Has.Count.EqualTo(0));
+        }
+
         [Test]
         public void GetSemanticDomainFullDomainFound()
         {

diff --git a/Backend.Tests/Mocks/SemanticDomainRepositoryMock.cs b/Backend.Tests/Mocks/SemanticDomainRepositoryMock.cs
@@ -8,21 +8,10 @@ namespace Backend.Tests.Mocks
     public class SemanticDomainRepositoryMock : ISemanticDomainRepository
     {
         private object? _responseObj;
-        private List<string>? _validLangs;
 
         public Task<List<SemanticDomainTreeNode>?> GetAllSemanticDomainTreeNodes(string lang)
         {
-            if (_validLangs is null)
-            {
-                return Task.FromResult((List<SemanticDomainTreeNode>?)_responseObj);
-            }
-
-            List<SemanticDomainTreeNode>? semDoms = null;
-            if (_validLangs.Contains(lang))
-            {
-                semDoms = new() { new(new SemanticDomain { Lang = lang }) };
-            }
-            return Task.FromResult(semDoms);
+            return Task.FromResult((List<SemanticDomainTreeNode>?)_responseObj);
         }
 
         public Task<SemanticDomainFull?> GetSemanticDomainFull(string id, string lang)
@@ -44,10 +33,5 @@ internal void SetNextResponse(object? response)
         {
             _responseObj = response;
         }
-
-        internal void SetValidLangs(List<string>? validLangs)
-        {
-            _validLangs = validLangs;
-        }
     }
 }
diff --git a/Backend.Tests/Services/LiftServiceTests.cs b/Backend.Tests/Services/LiftServiceTests.cs
@@ -1,8 +1,5 @@
-using System.Collections.Generic;
-using System.IO;
 using Backend.Tests.Mocks;
 using BackendFramework.Interfaces;
-using BackendFramework.Models;
 using BackendFramework.Services;
 using NUnit.Framework;
 
@@ -64,34 +61,5 @@ public void StoreRetrieveDeleteImportTest()
             Assert.That(_liftService.DeleteImport(UserId), Is.True);
             Assert.That(_liftService.RetrieveImport(UserId), Is.Null);
         }
-
-        [Test]
-        public void CreateLiftRangesTest()
-        {
-            List<SemanticDomain> frDoms = new() { new() { Lang = "fr" }, new() };
-            List<SemanticDomain> ptDoms = new() { new(), new() { Lang = "pt" } };
-            List<SemanticDomain> zzDoms = new() { new() { Lang = "zz" } };
-            List<Word> projWords = new()
-            {
-                // First semantic domain of the second sense of a word
-                new() { Senses = new() { new(), new() { SemanticDomains = frDoms } } },
-                // Second semantic domain of the first sense of a word
-                new() { Senses = new() { new() { SemanticDomains = ptDoms }, new() } },
-                // Semantic domain with unsupported language
-                new() { Senses = new() { new() { SemanticDomains = zzDoms } } }
-            };
-
-            ((SemanticDomainRepositoryMock)_semDomRepo).SetValidLangs(new() { "en", "fr", "pt" });
-            var langs = _liftService.CreateLiftRanges(projWords, new(), FileName).Result;
-            Assert.That(langs, Has.Count.EqualTo(2));
-            Assert.That(langs, Does.Contain("fr"));
-            Assert.That(langs, Does.Contain("pt"));
-
-            var liftRangesText = File.ReadAllText(FileName);
-            Assert.That(liftRangesText, Does.Not.Contain("\"en\""));
-            Assert.That(liftRangesText, Does.Contain("\"fr\""));
-            Assert.That(liftRangesText, Does.Contain("\"pt\""));
-            Assert.That(liftRangesText, Does.Not.Contain("\"zz\""));
-        }
     }
 }
diff --git a/Backend/Controllers/SemanticDomainController.cs b/Backend/Controllers/SemanticDomainController.cs
@@ -1,3 +1,5 @@
+using System.Collections.Generic;
+using System.Linq;
 using System.Threading.Tasks;
 using BackendFramework.Interfaces;
 using BackendFramework.Models;
@@ -19,6 +21,19 @@ public SemanticDomainController(ISemanticDomainRepository semDomRepo)
             _semDomRepo = semDomRepo;
         }
 
+        /// <summary>
+        /// Returns a dictionary mapping domain ids to names in the specified language (fallback: "en").
+        /// </summary>
+        [HttpGet("allDomainNames", Name = "GetAllSemanticDomainNames")]
+        [ProducesResponseType(StatusCodes.Status200OK, Type = typeof(Dictionary<string, string>))]
+        public async Task<IActionResult> GetAllSemanticDomainNames(string lang)
+        {
+            var semDoms = await _semDomRepo.GetAllSemanticDomainTreeNodes(lang)
+                ?? await _semDomRepo.GetAllSemanticDomainTreeNodes("en")
+                ?? new();
+            return Ok(semDoms.ToDictionary(x => x.Id, x => x.Name));
+        }
+
         /// <summary> Returns <see cref="SemanticDomainFull"/> with specified id and in specified language </summary>
         [HttpGet("domainFull", Name = "GetSemanticDomainFull")]
         [ProducesResponseType(StatusCodes.Status200OK, Type = typeof(SemanticDomainFull))]

diff --git a/Backend/Interfaces/ILiftService.cs b/Backend/Interfaces/ILiftService.cs
@@ -10,7 +10,7 @@ public interface ILiftService
         ILiftMerger GetLiftImporterExporter(string projectId, string vernLang, IWordRepository wordRepo);
         Task<bool> LdmlImport(string dirPath, IProjectRepository projRepo, Project project);
         Task<string> LiftExport(string projectId, IWordRepository wordRepo, IProjectRepository projRepo);
-        Task<List<string>> CreateLiftRanges(List<Word> projWords, List<SemanticDomain> projDoms, string rangesDest);
+        Task CreateLiftRanges(List<SemanticDomain> projDoms, string rangesDest);
 
         // Methods to store, retrieve, and delete an export string in a common dictionary.
         void StoreExport(string userId, string filePath);

diff --git a/Backend/Services/LiftService.cs b/Backend/Services/LiftService.cs
@@ -300,6 +300,8 @@ public async Task<string> LiftExport(
             // So the words found in allWords with no matching guid in activeWords are exported as 'deleted'.
             var deletedWords = allWords.Where(
                 x => activeWords.All(w => w.Guid != x.Guid)).DistinctBy(w => w.Guid).ToList();
+            var semDomNames = (await _semDomRepo.GetAllSemanticDomainTreeNodes("en") ?? new())
+                .ToDictionary(x => x.Id, x => x.Name);
             foreach (var wordEntry in activeWords)
             {
                 var id = MakeSafeXmlAttribute(wordEntry.Vernacular) + "_" + wordEntry.Guid;
@@ -321,7 +323,7 @@ public async Task<string> LiftExport(
 
                 AddNote(entry, wordEntry);
                 AddVern(entry, wordEntry, proj.VernacularWritingSystem.Bcp47);
-                AddSenses(entry, wordEntry);
+                AddSenses(entry, wordEntry, semDomNames);
                 await AddAudio(entry, wordEntry.Audio, audioDir, projectId, projSpeakers);
 
                 liftWriter.Add(entry);
@@ -334,7 +336,7 @@ public async Task<string> LiftExport(
 
                 AddNote(entry, wordEntry);
                 AddVern(entry, wordEntry, proj.VernacularWritingSystem.Bcp47);
-                AddSenses(entry, wordEntry);
+                AddSenses(entry, wordEntry, semDomNames);
                 await AddAudio(entry, wordEntry.Audio, audioDir, projectId, projSpeakers);
 
                 liftWriter.AddDeletedEntry(entry);
@@ -367,7 +369,7 @@ public async Task<string> LiftExport(
             // Export semantic domains to lift-ranges
             if (proj.SemanticDomains.Count != 0 || CopyLiftRanges(proj.Id, rangesDest) is null)
             {
-                await CreateLiftRanges(allWords, proj.SemanticDomains, rangesDest);
+                await CreateLiftRanges(proj.SemanticDomains, rangesDest);
             }
 
             // Export character set to ldml.
@@ -408,11 +410,8 @@ public async Task<string> LiftExport(
             return rangesSrc;
         }
 
-        /// <summary> Export semantic domains to lift-ranges </summary>
-        /// <exception cref="ExportException"> If fails to load needed semantic domain list </exception>
-        /// <returns> List of languages found in project sem-doms and included in the lift-ranges file </returns>
-        public async Task<List<string>> CreateLiftRanges(
-            List<Word> projWords, List<SemanticDomain> projDoms, string rangesDest)
+        /// <summary> Export English semantic domains (along with any custom domains) to lift-ranges. </summary>
+        public async Task CreateLiftRanges(List<SemanticDomain> projDoms, string rangesDest)
         {
             await using var liftRangesWriter = XmlWriter.Create(rangesDest, new XmlWriterSettings
             {
@@ -425,21 +424,8 @@ public async Task<List<string>> CreateLiftRanges(
             liftRangesWriter.WriteStartElement("range");
             liftRangesWriter.WriteAttributeString("id", "semantic-domain-ddp4");
 
-            var wordLangs = projWords
-                .SelectMany(w => w.Senses.SelectMany(s => s.SemanticDomains.Select(d => d.Lang))).Distinct();
-            var exportLangs = new List<string>();
-            foreach (var lang in wordLangs)
-            {
-                var semDoms = await _semDomRepo.GetAllSemanticDomainTreeNodes(lang);
-                if (semDoms is not null && semDoms.Count > 0)
-                {
-                    exportLangs.Add(lang);
-                    foreach (var sd in semDoms)
-                    {
-                        WriteRangeElement(liftRangesWriter, sd.Id, sd.Guid, sd.Name, sd.Lang);
-                    }
-                }
-            }
+            (await _semDomRepo.GetAllSemanticDomainTreeNodes("en"))?
+                .ForEach(sd => { WriteRangeElement(liftRangesWriter, sd.Id, sd.Guid, sd.Name, sd.Lang); });
 
             // Pull from new semantic domains in project
             foreach (var sd in projDoms)
@@ -456,7 +442,6 @@ public async Task<List<string>> CreateLiftRanges(
 
             await liftRangesWriter.FlushAsync();
             liftRangesWriter.Close();
-            return exportLangs;
         }
 
         /// <summary> Adds <see cref="Note"/> of a word to be written out to lift </summary>
@@ -486,7 +471,7 @@ private static void AddVern(LexEntry entry, Word wordEntry, string vernacularBcp
         }
 
         /// <summary> Adds each <see cref="Sense"/> of a word to be written out to lift </summary>
-        private static void AddSenses(LexEntry entry, Word wordEntry)
+        private static void AddSenses(LexEntry entry, Word wordEntry, Dictionary<string, string> semDomNames)
         {
             var activeSenses = wordEntry.Senses.Where(
                 s => s.Accessibility == Status.Active || s.Accessibility == Status.Protected).ToList();
@@ -540,7 +525,8 @@ private static void AddSenses(LexEntry entry, Word wordEntry)
                 foreach (var semDom in currentSense.SemanticDomains)
                 {
                     var orc = new OptionRefCollection();
-                    orc.Add(semDom.Id + " " + semDom.Name);
+                    semDomNames.TryGetValue(semDom.Id, out string? name);
+                    orc.Add(semDom.Id + " " + name ?? semDom.Name);
                     lexSense.Properties.Add(new KeyValuePair<string, IPalasoDataObjectProperty>(
                         LexSense.WellKnownProperties.SemanticDomainDdp4, orc));
                 }
@@ -658,16 +644,6 @@ private static void WriteRangeElement(
             liftRangesWriter.WriteEndElement(); //end range element
         }
 
-        [Serializable]
-        public class ExportException : Exception
-        {
-            public ExportException() { }
-
-            public ExportException(string msg) : base(msg) { }
-
-            protected ExportException(SerializationInfo info, StreamingContext context) : base(info, context) { }
-        }
-
         private sealed class LiftMerger : ILiftMerger
         {
             private readonly string _projectId;

diff --git a/Backend/Startup.cs b/Backend/Startup.cs
@@ -105,8 +105,8 @@ public void ConfigureServices(IServiceCollection services)
             const string secretKeyEnvName = "COMBINE_JWT_SECRET_KEY";
             var secretKey = Environment.GetEnvironmentVariable(secretKeyEnvName);
 
-            // The JWT key size must be at least 128 bits long.
-            const int minKeyLength = 128 / 8;
+            // The JWT key size must be at least 256 bits long.
+            const int minKeyLength = 256 / 8;
             if (secretKey is null || secretKey.Length < minKeyLength)
             {
                 _logger.LogError("Must set {EnvName} environment variable to string of length {MinLength} or longer.",

diff --git a/README.md b/README.md
@@ -63,6 +63,7 @@ A rapid word collection tool. See the [User Guide](https://sillsdev.github.io/Th
    6. [Inspect Database](#inspect-database)
    7. [Add or Update Dictionary Files](#add-or-update-dictionary-files)
    8. [Cleanup Local Repository](#cleanup-local-repository)
+   9. [Generate Installer Script for The Combine](#generate-installer-script-for-the-combine-linux-only)
 3. [Setup Local Kubernetes Cluster](#setup-local-kubernetes-cluster)
    1. [Install Rancher Desktop](#install-rancher-desktop)
    2. [Install Docker Desktop](#install-docker-desktop)
@@ -127,6 +128,17 @@ A rapid word collection tool. See the [User Guide](https://sillsdev.github.io/Th
     `dotnet tool update --global dotnet-reportgenerator-globaltool --version 5.0.4`
 11. [dotnet-project-licenses](https://github.com/tomchavakis/nuget-license)
     `dotnet tool update --global dotnet-project-licenses`
+12. Tools for generating the self installer (Linux only):
+
+    - [makeself](https://makeself.io/) - a tool to make self-extracting archives in Unix
+    - [pandoc](https://pandoc.org/installing.html#linux) - a tool to convert Markdown documents to PDF.
+    - `weasyprint` a PDF engine for `pandoc`.
+
+    These can be installed on Debian-based distributions by running:
+
+    ```console
+    sudo apt install -y makeself pandoc weasyprint
+    ```
 
 ### Prepare the Environment
 
@@ -505,6 +517,23 @@ of development setup errors. Run from within a Python virtual environment.
 python scripts/cleanup_local_repo.py
 ```
 
+### Generate Installer Script for The Combine (Linux only)
+
+To generate the installer script, run the following commands starting in the project top level directory:
+
+```console
+cd installer
+./make-combine-installer.sh combine-release-number
+```
+
+where `combine-release-number` is the Combine release to be installed, e.g. `v1.2.0`.
+
+To update the PDF copy of the installer README.md file, run the following from the `installer` directory:
+
+```console
+pandoc --pdf-engine=weasyprint README.md -o README.pdf
+```
+
 ## Setup Local Kubernetes Cluster
 
 This section describes how to create a local Kubernetes cluster using either _Rancher Desktop_ or _Docker Desktop_.

diff --git a/deploy/ansible/group_vars/nuc/main.yml b/deploy/ansible/group_vars/nuc/main.yml
@@ -19,7 +19,17 @@ image_pull_secret: aws-login-credentials
 app_namespace: thecombine
 
 k8s_user: sillsdev
-k8s_group: sillsdev
+
+################################################
+# Helm Installation
+################################################
+install_helm: no
+
+################################################
+# Support Tool Settings
+################################################
+install_ip_viewer: yes
+install_combinectl: yes
 
 #######################################
 # Ingress configuration
@@ -30,7 +40,7 @@ ingress_namespace: ingress-nginx
 # The server name will direct traffic to the production
 # server since it is used to get the certificates for the
 # NUC.
-public_dns_name: "{{ ansible_hostname }}"
+k8s_dns_name: "{{ ansible_hostname }}"
 
 ################################################
 # Ethernet settings

diff --git a/deploy/ansible/group_vars/server/main.yml b/deploy/ansible/group_vars/server/main.yml
@@ -19,6 +19,17 @@ create_namespaces: []
 # k8s namespaces
 app_namespace: thecombine
 
+################################################
+# Helm Installation
+################################################
+install_helm: no
+
+################################################
+# Support Tool Settings
+################################################
+install_ip_viewer: no
+install_combinectl: no
+
 #######################################
 # Ingress configuration
 ingress_namespace: ingress-nginx