Merge pull request #12631 from keymanapp/fix/developer/12623-full-tes…

…t-in-kmcmplib

fix(developer): reconnect `--full-test` in kmcmplib build and enable for CI

developer/src/kmcmplib/build.bat

@@ -80,7 +80,11 @@ shift
 if "!COMMAND!" == "configure" (
   echo === Configuring Keyman KMX Compiler for Windows !ARCH! !BUILDTYPE! ===
   if exist build\!ARCH!\!BUILDTYPE! rd /s/q build\!ARCH!\!BUILDTYPE!
-  meson setup build\!ARCH!\!BUILDTYPE! !STATIC_LIBRARY! --buildtype !BUILDTYPE! --werror %1 %2 %3 %4 %5 %6 %7 %8 %9 || exit !errorlevel!
+  if "%1"=="--full-test" (
+    meson setup build\!ARCH!\!BUILDTYPE! !STATIC_LIBRARY! --buildtype !BUILDTYPE! --werror -D full_test=true %2 %3 %4 %5 %6 %7 %8 %9 || exit !errorlevel!
+  ) else (
+    meson setup build\!ARCH!\!BUILDTYPE! !STATIC_LIBRARY! --buildtype !BUILDTYPE! --werror %1 %2 %3 %4 %5 %6 %7 %8 %9 || exit !errorlevel!
+  )
   shift
 )
 

developer/src/kmcmplib/build.sh

@@ -7,6 +7,7 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")"
 ## END STANDARD BUILD SCRIPT INCLUDE
 
 . "$KEYMAN_ROOT/resources/shellHelperFunctions.sh"
+. "$KEYMAN_ROOT/resources/build/build-utils-ci.inc.sh"
 . "$THIS_SCRIPT_PATH/checkout-keyboards.inc.sh"
 . "$THIS_SCRIPT_PATH/commands.inc.sh"
 
@@ -78,7 +79,7 @@ do_action() {
   done
 }
 
-if builder_has_option --full-test; then
+if should_do_full_test; then
   locate_keyboards_repo
 fi
 
@@ -101,7 +102,7 @@ if builder_start_action configure; then
   # We have to checkout the keyboards repo in a 'configure' action because
   # otherwise meson will not get the right list of keyboard source files,
   # even though we only use it in the 'test' action
-  if builder_has_option --full-test; then
+  if should_do_full_test; then
     checkout_keyboards
   fi
 

developer/src/kmcmplib/commands.inc.sh

@@ -33,12 +33,19 @@ do_configure() {
     STANDARD_MESON_ARGS="--cross-file wasm.defs.build --cross-file wasm.build --default-library static"
   fi
 
+  local FULL_TEST=
+  local FULL_TEST_WIN=
+  if should_do_full_test; then
+    FULL_TEST="-D full_test=true"
+    FULL_TEST_WIN=--full-test
+  fi
+
   if [[ $target =~ ^(x86|x64)$ ]]; then
-    cmd //C build.bat $target $BUILDER_CONFIGURATION configure "${builder_extra_params[@]}"
+    cmd //C build.bat $target $BUILDER_CONFIGURATION configure $FULL_TEST_WIN "${builder_extra_params[@]}"
   else
     pushd "$THIS_SCRIPT_PATH" > /dev/null
     # Additional arguments are used by Linux build, e.g. -Dprefix=${INSTALLDIR}
-    meson setup "$MESON_PATH" --werror --buildtype $BUILDER_CONFIGURATION $STANDARD_MESON_ARGS "${builder_extra_params[@]}"
+    meson setup "$MESON_PATH" --werror --buildtype $BUILDER_CONFIGURATION $STANDARD_MESON_ARGS $FULL_TEST "${builder_extra_params[@]}"
     popd > /dev/null
   fi
   builder_finish_action success configure:$target
@@ -78,7 +85,7 @@ do_test() {
   # Works on a local clone of keyboards repository, to avoid clobbering
   # user's existing keyboards repo, if present
 
-  if builder_has_option --full-test; then
+  if should_do_full_test; then
     checkout_keyboards
   fi
 
@@ -125,4 +132,12 @@ build_meson_cross_file_for_wasm() {
     local R=$(echo $EMSCRIPTEN_BASE | sed 's_/_\\/_g')
   fi
   sed -e "s/\$EMSCRIPTEN_BASE/$R/g" wasm.build.$BUILDER_OS.in > wasm.build
+}
+
+should_do_full_test() {
+  if builder_has_option --full-test || builder_is_ci_test_build; then
+    return 0
+  fi
+
+  return 1
 }

developer/src/kmcmplib/src/CompilerInterfaces.cpp

@@ -13,7 +13,6 @@ EXTERN bool kmcmp_CompileKeyboard(
   const void* procContext,
   KMCMP_COMPILER_RESULT& result
 ) {
-
   FILE_KEYBOARD fk;
   fk.extra = new KMCMP_COMPILER_RESULT_EXTRA;
   fk.extra->kmnFilename = pszInfile;

developer/src/kmcmplib/tests/get-test-source.sh

@@ -9,5 +9,8 @@
 set -eu
 find "$1" -name '*.kmn' | \
   grep -E '(release|experimental)/([a-z0-9_]+)/([a-z0-9_]+)/source/\3\.kmn$' | \
-  grep -v masaram_gondi
+  grep -vE 'masaram_gondi|anii|sil_kmhmu|fv_statimcets|fv_nuucaanul'
 # #12623: exclude masaram_gondi due to #11806
+# #12631: exclude anii, sil_kmhmu as ico references have mismatching case
+# #12631: exclude fv_statimcets, fv_nuucaanul as these include U+2002 which is not
+#         treated as whitespace on mac arch

developer/src/kmcmplib/tests/util_callbacks.cpp

@@ -1,5 +1,6 @@
 #include <stdio.h>
 #include <string>
+#include <algorithm>
 #include <kmcmplibapi.h>
 #include "util_filesystem.h"
 #include "../src/compfile.h"
@@ -29,6 +30,14 @@ void msgproc(const KMCMP_COMPILER_RESULT_MESSAGE &message, void* context) {
   printf(")\n");
 }
 
+void normalizeSlashes(std::string& s) {
+#ifdef _WIN32
+  std::replace(s.begin(), s.end(), '/', '\\');
+#else
+  std::replace(s.begin(), s.end(), '\\', '/');
+#endif
+}
+
 const std::vector<uint8_t> loadfileProc(const std::string& filename, const std::string& baseFilename) {
   std::string resolvedFilename = filename;
   if(baseFilename.length() && IsRelativePath(filename.c_str())) {
@@ -40,6 +49,8 @@ const std::vector<uint8_t> loadfileProc(const std::string& filename, const std::
     }
   }
 
+  normalizeSlashes(resolvedFilename);
+
   std::vector<uint8_t> buf;
 
   FILE* fp = Open_File(resolvedFilename.c_str(), "rb");

resources/build/build-utils-ci.inc.sh

@@ -7,6 +7,38 @@
 
 . "$KEYMAN_ROOT/resources/build/jq.inc.sh"
 
+#
+# Returns 0 if current build is running in CI, as a pull request test, or as a
+# mainline branch test, or as a release build
+#
+builder_is_ci_build() {
+  if builder_is_ci_release_build || builder_is_ci_test_build; then
+    return 0
+  fi
+  return 1
+}
+
+#
+# Returns 0 if current build is running as a release build in CI
+#
+builder_is_ci_release_build() {
+  if [[ "$VERSION_ENVIRONMENT" =~ ^alpha|beta|stable$ ]]; then
+    return 0
+  fi
+  return 1
+}
+
+#
+# Returns 0 if current build is running in CI, as a pull request test, or as a
+# mainline branch test
+#
+builder_is_ci_test_build() {
+  if [[ "$VERSION_ENVIRONMENT" == test ]]; then
+    return 0
+  fi
+  return 1
+}
+
 #
 # Returns 0 if current build is in CI and triggered from a pull request. If it
 # returns 0, then a call is made to GitHub to get pull request details, and the