CLDR-17288 vagrant/ansible: fix for openliberty / mysql

.github/workflows/ansible-lint.yml

@@ -25,7 +25,7 @@ jobs:
         override-deps: |
           rich>=9.5.1,<11.0.0
   testprovision:
-    runs-on: macos-10.15  # see https://github.com/actions/virtual-environments/issues/4060
+    runs-on: macos-latest
     steps:
     - uses: actions/checkout@v3
       with:
@@ -34,5 +34,12 @@ jobs:
       run: brew install ansible
     - name: Prepare the provision
       run: (cd tools/scripts/ansible && ansible-galaxy install -r requirements.yml && ln -sv test-local-vars local-vars && vagrant validate)
-    - name: Try the provision (ignored, see CLDR-14757)
-      run: (cd tools/scripts/ansible && vagrant up) || ( echo Failing But Not Blocking CLDR-14757 ; true)
+    - name: Try the provision
+      run: (cd tools/scripts/ansible && vagrant up || ( ( echo "# WARNING vagrant failed—we are ignoring the failure until https://unicode-org.atlassian.net/browse/CLDR-17297 … Try it manually." | tee -a $GITHUB_STEP_SUMMARY)  ; true ) )
+      env:
+        VAGRANT_DISABLE_VBOXSYMLINKCREATE: 1
+
+# one at a time please
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true

tools/cldr-apps/src/main/java/org/unicode/cldr/web/XPathTable.java

@@ -140,7 +140,6 @@ private void setupDB(Connection ourConn) throws SQLException {
             sql = null;
             s.close();
             s = null;
-            ourConn.commit();
         } finally {
             DBUtils.close(s);
             if (sql != null) {

tools/cldr-apps/src/test/java/org/unicode/cldr/unittest/web/TestXPathTable.java

@@ -29,7 +29,7 @@ public TestXPathTable() {
     public void TestPutGet() throws SQLException {
         if (TestAll.skipIfNoDb()) return;
         logln("Testing " + TEST_COUNT + " xpaths");
-        Connection conn = DBUtils.getInstance().getDBConnection();
+        Connection conn = DBUtils.getInstance().getAConnection();
         XPathTable xpt = XPathTable.createTable(conn);
         DBUtils.closeDBConnection(conn);
         HashMap<Integer, String> s = new HashMap<>();
@@ -111,7 +111,7 @@ public void TestRemoveDraftAltProposed() {
 
     public void TestNonDistinguishing() throws SQLException {
         if (TestAll.skipIfNoDb()) return;
-        Connection conn = DBUtils.getInstance().getDBConnection();
+        Connection conn = DBUtils.getInstance().getAConnection();
         XPathTable xpt = XPathTable.createTable(conn);
         DBUtils.closeDBConnection(conn);
 

tools/cldr-apps/src/test/java/org/unicode/cldr/web/TestSTFactory.java

@@ -684,7 +684,7 @@ public static STFactory createFactory() throws SQLException {
                 new File(CLDRPaths.BASE_DIRECTORY, "seed/annotations/").getAbsolutePath();
 
         et0 = new ElapsedTimer("setup DB");
-        Connection conn = DBUtils.getInstance().getDBConnection();
+        Connection conn = DBUtils.getInstance().getAConnection();
         System.err.println(et0.toString());
 
         et0 = new ElapsedTimer("setup Registry");

tools/scripts/ansible/README.md

@@ -143,30 +143,68 @@ the operation to a single host.
 
 ## Local Test
 
-- install vagrant and some provider such as virtualbox or libvirt
+Here’s how to deploy the SurveyTool locally and try it out.
+
+### Build
+
+You need a server zipfile to deploy.  This is a file such as `cldr-apps.zip`. When expanded, it contains a directory tree beginning with `wlp/`.
+
+#### Option A: Local Build
+
+- Prerequisites: See <https://cldr.unicode.org/development/maven> and follow instructions to be able to run `mvn package` as shown on that page.
+
+- You can then create a server zipfile locally with maven using these command (from the top `cldr/` directory).  The first command does a full build of CLDR, but skips running tests.
 
 ```shell
+mvn --file=tools/pom.xml install -DskipTests=true
+mvn --file=tools/pom.xml -pl cldr-apps liberty:package
+```
+
+- The output file will be in `tools/cldr-apps/target/cldr-apps.zip`
+
+
+#### Option B: Download a Build
+
+- Server Builds are actually attached to each action run in <https://github.com/unicode-org/cldr/actions/workflows/maven.yml>, look for an artifact entitled `cldr-apps-server` at the bottom of a run.
+
+- *Warning*: Clicking on this artifact will download a zipfile named `cldr-apps-server.zip` which _contains_ `cldr-apps.zip`.  Double clicking or automatic downloading will often extract one too many levels of zipfiles. If you see a folder named `wlp` then you have extracted too much. From the command line you can unpack with `unzip cldr-apps-server.zip` which will extract `cldr-apps.zip`.
+
+### Deploy
+
+- install [vagrant](https://www.vagrantup.com) and some provider such as virtualbox or libvirt, see vagrant docs.
+
+- vagrant up!
+
+```shell
+# (this directory)
+cd tools/scripts/ansible
 vagrant up
 ```
 
 - To log into the new host, run `vagrant ssh`
 
 - To iterate, trying to reapply ansible, run `vagrant provision --provision-with=ansible`
 
-- to deploy ST to this, use the following:
+- to deploy your built server to this, use the following:
 
 ```shell
-(cd ../.. ; mvn package) # go to the tools folder and build ST (cldr-apps.war, etc.) if not already built
-vagrant ssh -- sudo -u surveytool /usr/local/bin/deploy-to-tomcat.sh $(git rev-parse HEAD) < ../../cldr-apps/target/cldr-apps.war
+# Note 1: $(git rev-parse HEAD) just turns into a full git hash such as 72dda8d7386087bf6087de200b5edc002feca2f2, you can use an explicit hash instead.
+# Note 2: change ../../cldr-apps/target/cldr-apps.zip to point to your cldr-apps.zip file if moved
+vagrant ssh -- sudo -u surveytool /usr/local/bin/deploy-to-openliberty.sh $(git rev-parse HEAD) < ../../cldr-apps/target/cldr-apps.zip
 ```
 
-- Now you should be able to login at <http://127.0.0.1:8880/cldr-apps/>
+- Now you should be able to login at <http://127.0.0.1:9081/cldr-apps/>
 
-- If you need to get directly to the tomcat server, use:
+- Use the user `admin@` and the password set in `surveytooldeploy.vap` above.
 
-```shell
-vagrant ssh -- -L 9080:127.0.0.1:9080
-# leave this shell window open.
-```
+- *Note*: <http://127.0.0.1:8880> will go to the nginx proxy, but it has login problems, see <https://unicode-org.atlassian.net/browse/CLDR-14321>
+
+### Operation
 
-Then, you can go to <http://127.0.0.1:9080> and directly access tomcat.
+- the mvn build and `deploy-to-openliberty.sh` steps above can be repeated to redeploy a new version of the server code
+- `vagrant ssh` to login and poke around at the server
+- `sudo nano /srv/st/config/cldr.properties` to edit the configuration file (will be created automatically at first ST boot, restart server to pickup changes).
+- `sudo journalctl -f` to watch server logs
+- `sudo systemctl restart openliberty@cldr` to restart the server
+- Logs are in `/var/log/openliberty/cldr`
+- `sudo -u cldradmin mysql cldrdb` will give you the raw SQL prompt

tools/scripts/ansible/Vagrantfile

@@ -3,11 +3,11 @@
 
 Vagrant.configure("2") do |config|
   config.vm.provider "virtualbox" do |v|
-    v.memory = 1024
+    v.memory = 8192
   end
   # increase the boot timeout
   config.vm.boot_timeout = 600
-  config.vm.box = "ubuntu/bionic64"
+  config.vm.box = "ubuntu/jammy64";
   config.vm.define "surveytool"
   config.vm.hostname = "surveytool"
   # forward http and https
@@ -19,6 +19,7 @@ Vagrant.configure("2") do |config|
     apt-get install -y python3 sudo
   SHELL
   config.vm.provision "ansible" do |ansible|
+    ansible.compatibility_mode = "2.0"
     ansible.playbook = "vagrant-playbook.yml"
   end
 end

tools/scripts/ansible/cldr-apps-playbook.yml

@@ -47,7 +47,6 @@
       notify: Restart OpenLiberty
     - name: Checkout CLDR trunk
       become: yes
-      become_user: "{{ cldr_surveytool_user }}"
       git:
         repo: https://github.com/unicode-org/cldr.git
         dest: "{{ cldr_trunk_path }}"
@@ -57,6 +56,10 @@
         # this is a deep clone because we will need to keep updating
         # it with history. It does not include LFS as that
         # is not needed for the surveytool.
+    - name: Fix CLDR trunk perms
+      file:
+        path: "{{ cldr_trunk_path }}"
+        owner: "{{ cldr_surveytool_user }}"
   handlers:
     - name: Restart OpenLiberty
       service:

tools/scripts/ansible/roles/openliberty/meta/main.yml

@@ -4,7 +4,7 @@ galaxy_info:
 
   license: Unicode-DFS-2016
 
-  min_ansible_version: 2.1
+  min_ansible_version: "2.1"
 
   platforms:
   - name: Ubuntu

tools/scripts/ansible/roles/openliberty/tasks/main.yml

@@ -4,7 +4,7 @@
 - name: Install OpenLiberty GPG key
   become: true
   ansible.builtin.apt_key:
-    url: http://public.dhe.ibm.com/ibmdl/export/pub/software/openliberty/runtime/os-native-packages/public.key
+    url: https://public.dhe.ibm.com/ibmdl/export/pub/software/openliberty/runtime/os-native-packages/public.key
     state: present
   when:
     - ansible_distribution == "Ubuntu"
@@ -19,3 +19,4 @@
   ansible.builtin.apt:
     pkg:
       - openliberty
+      - openliberty

tools/scripts/ansible/vars/mysql.yml

@@ -4,3 +4,5 @@ mysql_databases:
     collation: latin1_bin
 mysql_enabled_on_startup: true
 mysql_bind_address: localhost
+mysql_replication_master: 'localhost'
+mysql_replication_master_inventory_host: 'localhost'