Store only changes from the next version for older copies of the UCD (#…

…699)

.github/workflows/build-jsp.yml

@@ -52,22 +52,15 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-maven-
     # TODO: move this DOWN after JSPs once it works
-    - name: Generate Unicode data
-      # TODO: hard coded version
-      # TODO: symlink of security here?
+    - name: Regenerate the property cache files
       run: >
-        mkdir -pv $(pwd)/output/Generated/ &&
-        mvn -s .github/workflows/mvn-settings.xml -B compile exec:java -DskipTests=true
-        -Dexec.mainClass="org.unicode.text.UCD.Main" -Dexec.args="version ${CURRENT_UVERSION} build MakeUnicodeFiles"
-        -am -pl unicodetools -DCLDR_DIR=${GITHUB_WORKSPACE}/cldr
-        -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=${CURRENT_UVERSION}
-        -DUNICODETOOLS_GEN_DIR=$(pwd)/output/Generated
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-    - name: Run unicodetools tests
-      run: >
-        mvn -s .github/workflows/mvn-settings.xml -B test -am -pl unicodetools
-        -DCLDR_DIR=${GITHUB_WORKSPACE}/cldr -T 1C -Dparallel=classes -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=${CURRENT_UVERSION} -DUNICODETOOLS_GEN_DIR=$(pwd)/output/Generated
+        mkdir -pv $(pwd)/Generated/ &&
+        mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.jsp.RebuildPropertyCache"
+        -am -pl unicodetools
+        -DCLDR_DIR=${GITHUB_WORKSPACE}/cldr
+        -DUNICODETOOLS_REPO_DIR=$(pwd)
+        -DUNICODETOOLS_GEN_DIR=$(pwd)/Generated &&
+        tar -cpz -f UnicodeJsps/target/generated.tgz ./Generated
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     - name: Package JSPs

UnicodeJsps/Dockerfile

@@ -9,6 +9,7 @@ RUN cd source && gcc -I ../include/ -static -Os -o3 -o bidiref1 bidiref1.c bruti
 RUN ls -lh /build/source/bidiref1 && (/build/source/bidiref1 || true)
 # copy and unpack to /tmp/data
 ADD ./target/cldr-unicodetools.tgz /build/data/
+ADD ./target/generated.tgz /build/data/
 # move this into place (including unicodetools/unicodetools)
 RUN rm -rf /build/data/cldr/.git  # unneeded
 FROM jetty:9-jre11-alpine-eclipse-temurin AS run
@@ -24,9 +25,10 @@ ENV BIDIREFHOME /usr/local/share
 # copy the bidiref1 bin
 ENV BIDIREF1 /usr/local/bin/bidiref1
 COPY --from=cbuild /build/source/bidiref1 /usr/local/bin/
-RUN mkdir -p /var/lib/jetty/data/unicodetools
+RUN mkdir -p /var/lib/jetty/data/unicodetools/Generated
 COPY --from=cbuild /build/data/cldr /var/lib/jetty/data/cldr
 COPY --from=cbuild /build/data/unicodetools/ /var/lib/jetty/data/unicodetools/unicodetools/
-ENV JAVA_OPTIONS -DCLDR_DIR=/var/lib/jetty/data/cldr -DUNICODETOOLS_REPO_DIR=/var/lib/jetty/data/unicodetools
+COPY --from=cbuild /build/data/Generated /var/lib/jetty/data/unicodetools/Generated
+ENV JAVA_OPTIONS -DCLDR_DIR=/var/lib/jetty/data/cldr -DUNICODETOOLS_REPO_DIR=/var/lib/jetty/data/unicodetools -DUNICODETOOLS_GEN_DIR=/var/lib/jetty/data/unicodetools/Generated -Xmx4g
 # This is the default PORT. Override by setting PORT.
 EXPOSE 8080

UnicodeJsps/src/main/java/org/unicode/jsp/UcdLoader.java

@@ -1,6 +1,5 @@
 package org.unicode.jsp;
 
-import com.ibm.icu.util.ICUException;
 import com.ibm.icu.util.VersionInfo;
 import java.io.IOException;
 import javax.servlet.ServletConfig;
@@ -9,8 +8,6 @@
 import javax.servlet.ServletResponse;
 import javax.servlet.annotation.WebServlet;
 import org.unicode.props.IndexUnicodeProperties;
-import org.unicode.props.UcdProperty;
-import org.unicode.props.UcdPropertyValues.Age_Values;
 import org.unicode.text.utility.Settings;
 
 @WebServlet
@@ -28,57 +25,6 @@ private static synchronized void setOldestLoadedUcd(VersionInfo v) {
         oldestLoadedUcd = v;
     }
 
-    private static void loadUcdHistory(VersionInfo earliest) {
-        System.out.println("Loading back to " + earliest + "...");
-        Age_Values[] ages = Age_Values.values();
-        final long overallStart = System.currentTimeMillis();
-        for (int i = ages.length - 1; i >= 0; --i) {
-            final var age = ages[i];
-            if (age == Age_Values.Unassigned) {
-                continue;
-            }
-            final long ucdStart = System.currentTimeMillis();
-            System.out.println("Loading UCD " + age.getShortName() + "...");
-            for (boolean unihan : new boolean[] {false, true}) {
-                final long partStart = System.currentTimeMillis();
-                final String name = unihan ? "Unihan" : "non-Unihan properties";
-                final var properties = IndexUnicodeProperties.make(age.getShortName());
-                for (UcdProperty property : UcdProperty.values()) {
-                    if (property.getShortName().startsWith("cjk") == unihan) {
-                        try {
-                            properties.load(property);
-                        } catch (ICUException e) {
-                            e.printStackTrace();
-                        }
-                    }
-                }
-                System.out.println(
-                        "Loaded "
-                                + name
-                                + " for "
-                                + age.getShortName()
-                                + " ("
-                                + (System.currentTimeMillis() - partStart)
-                                + " ms)");
-            }
-            System.out.println(
-                    "Loaded UCD "
-                            + age.getShortName()
-                            + " in "
-                            + (System.currentTimeMillis() - ucdStart)
-                            + " ms");
-            var version = VersionInfo.getInstance(age.getShortName());
-            setOldestLoadedUcd(version);
-            if (version == earliest) {
-                break;
-            }
-        }
-        System.out.println(
-                "Loaded all UCD history in "
-                        + (System.currentTimeMillis() - overallStart) / 1000
-                        + " s");
-    }
-
     @Override
     public void destroy() {}
 
@@ -94,12 +40,14 @@ public String getServletInfo() {
 
     @Override
     public void init(ServletConfig config) throws ServletException {
-        loadUcdHistory(Settings.LAST_VERSION_INFO);
+        IndexUnicodeProperties.loadUcdHistory(
+                Settings.LAST_VERSION_INFO, UcdLoader::setOldestLoadedUcd, true);
         new Thread(
                         new Runnable() {
                             @Override
                             public void run() {
-                                loadUcdHistory(null);
+                                IndexUnicodeProperties.loadUcdHistory(
+                                        null, UcdLoader::setOldestLoadedUcd, true);
                             }
                         })
                 .start();

UnicodeJsps/src/main/java/org/unicode/jsp/XPropertyFactory.java

@@ -64,7 +64,7 @@ public static XPropertyFactory make() {
     }
 
     public UnicodeProperty getProperty(String propertyAlias) {
-        var versioned = VersionedProperty.forJSPs().set(propertyAlias);
+        var versioned = VersionedProperty.forJSPs(UcdLoader::getOldestLoadedUcd).set(propertyAlias);
         if (versioned != null) {
             return versioned.getProperty();
         }

docs/unicodejsps/index.md

@@ -122,6 +122,13 @@ CLDR_REF=$(mvn help:evaluate -Dexpression=cldr.version -q -DforceStdout | cut -d
 mkdir -p UnicodeJsps/target && tar -cpz --exclude=.git -f UnicodeJsps/target/cldr-unicodetools.tgz ./cldr/ ./unicodetools/
 ```
 
+- Regenerate the property cache files:
+
+```
+mvn compile exec:java '-Dexec.mainClass="org.unicode.jsp.RebuildPropertyCache"' -am -pl unicodetools   "-DUNICODETOOLS_GEN_DIR=Generated"  "-DUNICODETOOLS_REPO_DIR=." "-DCLDR_DIR=<somewhere>"
+tar -cpz -f UnicodeJsps/target/generated.tgz ./Generated/
+```
+
 Now, finally build.
 
 - `docker build -t unicode/unicode-jsp:latest UnicodeJsps/`

unicodetools/src/main/java/org/unicode/jsp/RebuildPropertyCache.java

@@ -0,0 +1,54 @@
+package org.unicode.jsp;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Queue;
+import org.unicode.props.IndexUnicodeProperties;
+import org.unicode.text.utility.Settings;
+
+public class RebuildPropertyCache {
+
+    public static void main(String[] args) throws IOException {
+        final var binDir = new File(Settings.Output.BIN_DIR);
+        if (binDir.exists()) {
+            final Queue<File> directories = new ArrayDeque<>();
+            final List<File> directoriesToDelete = new ArrayList<>();
+            final List<File> filesToDelete = new ArrayList<>();
+            directories.add(binDir);
+            while (!directories.isEmpty()) {
+                final File directory = directories.poll();
+                for (final var child : directory.listFiles()) {
+                    if (child.isDirectory()) {
+                        directories.add(child);
+                        directoriesToDelete.add(child);
+                    } else {
+                        filesToDelete.add(child);
+                    }
+                }
+            }
+            System.out.println(
+                    "Cleaning "
+                            + filesToDelete.size()
+                            + " existing files in "
+                            + directoriesToDelete.size()
+                            + " existiing directories under "
+                            + Settings.Output.BIN_DIR);
+            for (final var f : filesToDelete) {
+                if (!f.delete()) {
+                    System.err.println("Failed to delete " + f);
+                }
+            }
+            for (final var f : directoriesToDelete) {
+                if (!f.delete()) {
+                    System.err.println("Failed to delete " + f);
+                }
+            }
+        }
+
+        IndexUnicodeProperties.loadUcdHistory(null, null, false);
+        System.out.println("Rebuilt property cache in " + Settings.Output.BIN_DIR);
+    }
+}