Merge branch 'master' into lighting

bin/test_trace

@@ -28,14 +28,14 @@ trace_out=$trace_in/expected/
 site_model=sites/udmi_site_model
 site_out=$site_model/out
 
-# Clean up from other unit testing
-rm -rf $site_model/devices/ACME-2301
+# Clean up from other testing runs
+rm -rf $site_model/devices/ACME-2301 $site_model/devices/*/out
+rm -rf $site_out $site_model/devices/XXX-*
+(cd $site_model; git checkout .)
 
 # Use a static/constant value to make output stable.
 export UDMI_TOOLS=test_trace
 
-# Clean things out, including some potentially corrupt state.
-rm -rf $site_out $site_model/devices/XXX-1
 cmd="$javarun -p -- -s $site_model -r $trace_in --"
 echo $cmd
 $cmd

common/src/main/java/com/google/udmi/util/MessageUpgrader.java

@@ -4,6 +4,7 @@
 import static com.google.udmi.util.Common.VERSION_KEY;
 import static com.google.udmi.util.GeneralUtils.OBJECT_MAPPER_RAW;
 import static com.google.udmi.util.GeneralUtils.ifNotNullThen;
+import static com.google.udmi.util.GeneralUtils.ifTrueThen;
 import static com.google.udmi.util.MessageDowngrader.convertVersion;
 
 import com.fasterxml.jackson.databind.JsonNode;
@@ -24,6 +25,7 @@ public class MessageUpgrader {
   public static final JsonNodeFactory NODE_FACTORY = JsonNodeFactory.instance;
   public static final String STATE_SCHEMA = "state";
   public static final String STATE_SYSTEM_SCHEMA = "state_system";
+  public static final String EVENTS_SYSTEM_SCHEMA = "events_system";
   public static final String METADATA_SCHEMA = "metadata";
   private static final String TARGET_FORMAT = "%d.%d.%d";
   private static final String RAW_GIT_VERSION = "git";
@@ -39,7 +41,7 @@ public class MessageUpgrader {
    * Create basic container for message upgrading.
    *
    * @param schemaName schema name to work with
-   * @param message message to be upgraded
+   * @param message    message to be upgraded
    */
   public MessageUpgrader(String schemaName, JsonNode message) {
     if (!(message instanceof ObjectNode)) {
@@ -152,12 +154,9 @@ private Object upgradeRaw(boolean forceUpgrade) {
       minor = 5;
     }
 
-    if (minor == 5 && patch == 0) {
-      JsonNode before = message.deepCopy();
-      upgradeTo_1_5_1();
-      upgraded |= !before.equals(message);
-      patch = 1;
-      minor = 5;
+    if (minor == 5) {
+      upgraded |= patch == 0 && didMessageChange(this::upgradeTo_1_5_1, patchUpdater(1));
+      upgraded |= patch == 1 && didMessageChange(this::upgradeTo_1_5_2, patchUpdater(2));
     }
 
     if (upgraded && message.get(VERSION_KEY) != null) {
@@ -174,6 +173,17 @@ private Object upgradeRaw(boolean forceUpgrade) {
     return message;
   }
 
+  private Runnable patchUpdater(int newPatch) {
+    return () -> patch = newPatch;
+  }
+
+  private boolean didMessageChange(Runnable updater, Runnable postfix) {
+    JsonNode before = message.deepCopy();
+    updater.run();
+    postfix.run();
+    return !before.equals(message);
+  }
+
   private void upgradeTo_1_3_14() {
     if (STATE_SCHEMA.equals(schemaName)) {
       upgradeTo_1_3_14_state();
@@ -220,9 +230,15 @@ private void upgradeTo_1_4_1() {
   }
 
   private void upgradeTo_1_5_1() {
-    if (METADATA_SCHEMA.equals(schemaName)) {
-      upgradeTo_1_5_1_metadata();
-    }
+    ifTrueThen(METADATA_SCHEMA.equals(schemaName), this::upgradeTo_1_5_1_metadata);
+  }
+
+  private void upgradeTo_1_5_2() {
+    ifTrueThen(EVENTS_SYSTEM_SCHEMA.equals(schemaName), this::upgradeTo_1_5_2_events_system);
+  }
+
+  private void upgradeTo_1_5_2_events_system() {
+    ifNotNullThen(message.remove("event_count"), node -> message.put("event_no", node));
   }
 
   private void upgradeTo_1_5_1_metadata() {

tests/traces/simple/devices/AHU-1/002_events_system.json

@@ -5,10 +5,11 @@
     "timestamp" : "2022-07-19T05:04:36Z",
     "level" : 200
   } ],
+  "event_count": 100,
   "metrics": {
     "mem_total_mb": 25164.88,
     "store_total_mb": null
   },
   "timestamp" : "2022-07-19T05:04:36+0000",
-  "version" : "1.5.2"
+  "version" : "1.5.1"
 }

tests/traces/simple/expected/devices/AHU-1/events_system.json

@@ -1,4 +1,5 @@
 {
+  "event_no" : 100,
   "logentries" : [ {
     "message" : "1999-10-20T01:02:03Z sending test message #70",
     "category" : "device.testing",
@@ -9,5 +10,6 @@
     "mem_total_mb" : 25164.88
   },
   "timestamp" : "2022-07-19T05:04:36+0000",
+  "upgraded_from" : "1.5.1",
   "version" : "1.5.2"
 }

tests/traces/simple/expected/devices/AHU-1/events_system.orig

@@ -0,0 +1,14 @@
+{
+  "event_count" : 100,
+  "logentries" : [ {
+    "message" : "1999-10-20T01:02:03Z sending test message #70",
+    "category" : "device.testing",
+    "timestamp" : "1999-10-20T01:02:03Z",
+    "level" : 200
+  } ],
+  "metrics" : {
+    "mem_total_mb" : 25164.88
+  },
+  "timestamp" : "2022-07-19T05:04:36+0000",
+  "version" : "1.5.1"
+}

tests/traces/upgrade/expected/devices/AHU-1/state.json

@@ -1,12 +1,18 @@
 {
   "system" : {
     "last_config" : "1999-10-20T01:02:03Z",
-    "firmware" : {
-      "version" : [ "D_IP_BW6_V35_6_S14" ]
+    "hardware" : {
+      "model" : "Delmatic IP_BW6",
+      "make" : "unknown"
     },
-    "operational" : true,
-    "make_model" : "Delmatic IP_BW6"
+    "software" : {
+      "firmware" : "D_IP_BW6_V35_6_S14"
+    },
+    "operation" : {
+      "operational" : true
+    }
   },
   "timestamp" : "2023-03-02T19:27:38.819Z",
-  "version" : 1
+  "upgraded_from" : "1",
+  "version" : "1.5.2"
 }

tests/traces/upgrade/expected/devices/AHU-1/state.orig

@@ -0,0 +1,12 @@
+{
+  "system" : {
+    "last_config" : "1999-10-20T01:02:03Z",
+    "firmware" : {
+      "version" : [ "D_IP_BW6_V35_6_S14" ]
+    },
+    "operational" : true,
+    "make_model" : "Delmatic IP_BW6"
+  },
+  "timestamp" : "2023-03-02T19:27:38.819Z",
+  "version" : 1
+}

tests/traces/upgrade/expected/devices/AHU-22/state.json

@@ -1,12 +1,18 @@
 {
   "system" : {
     "last_config" : "1999-10-20T01:02:03Z",
-    "firmware" : {
-      "version" : "D_IP_BW6_V35_6_S14"
+    "hardware" : {
+      "model" : "Delmatic IP_BW6",
+      "make" : "unknown"
     },
-    "operational" : true,
-    "make_model" : "Delmatic IP_BW6"
+    "software" : {
+      "firmware" : "D_IP_BW6_V35_6_S14"
+    },
+    "operation" : {
+      "operational" : true
+    }
   },
   "timestamp" : "2023-03-02T19:27:38.819Z",
-  "version" : 1
+  "upgraded_from" : "1",
+  "version" : "1.5.2"
 }

tests/traces/upgrade/expected/devices/AHU-22/state.orig

@@ -0,0 +1,12 @@
+{
+  "system" : {
+    "last_config" : "1999-10-20T01:02:03Z",
+    "firmware" : {
+      "version" : "D_IP_BW6_V35_6_S14"
+    },
+    "operational" : true,
+    "make_model" : "Delmatic IP_BW6"
+  },
+  "timestamp" : "2023-03-02T19:27:38.819Z",
+  "version" : 1
+}

tests/traces/upgrade/expected/devices/NON-1/state.json

@@ -1,12 +1,18 @@
 {
   "system" : {
     "last_config" : "1999-10-20T01:02:03Z",
-    "firmware" : {
-      "version" : [ "D_IP_BW6_V35_6_S14" ]
+    "hardware" : {
+      "model" : "Delmatic IP_BW6",
+      "make" : "unknown"
     },
-    "operational" : true,
-    "make_model" : "Delmatic IP_BW6"
+    "software" : {
+      "firmware" : "D_IP_BW6_V35_6_S14"
+    },
+    "operation" : {
+      "operational" : true
+    }
   },
   "timestamp" : "2023-03-02T19:27:35.819Z",
-  "version" : 1
+  "upgraded_from" : "1",
+  "version" : "1.5.2"
 }

tests/traces/upgrade/expected/devices/NON-1/state.orig

@@ -0,0 +1,12 @@
+{
+  "system" : {
+    "last_config" : "1999-10-20T01:02:03Z",
+    "firmware" : {
+      "version" : [ "D_IP_BW6_V35_6_S14" ]
+    },
+    "operational" : true,
+    "make_model" : "Delmatic IP_BW6"
+  },
+  "timestamp" : "2023-03-02T19:27:35.819Z",
+  "version" : 1
+}

tests/traces/upgrade/expected/devices/SNS-4/state.json

@@ -2,12 +2,18 @@
   "system" : {
     "serial_no" : "192",
     "last_config" : "1999-10-20T01:02:03Z",
-    "firmware" : {
-      "version" : "D_IP_BW6_V35_6_S14"
+    "hardware" : {
+      "model" : "Delmatic IP_BW6",
+      "make" : "unknown"
     },
-    "operational" : true,
-    "make_model" : "Delmatic IP_BW6"
+    "software" : {
+      "firmware" : "D_IP_BW6_V35_6_S14"
+    },
+    "operation" : {
+      "operational" : true
+    }
   },
   "timestamp" : "2023-03-02T19:27:38.819Z",
-  "version" : 1
+  "upgraded_from" : "1",
+  "version" : "1.5.2"
 }

tests/traces/upgrade/expected/devices/SNS-4/state.orig

@@ -0,0 +1,13 @@
+{
+  "system" : {
+    "serial_no" : "192",
+    "last_config" : "1999-10-20T01:02:03Z",
+    "firmware" : {
+      "version" : "D_IP_BW6_V35_6_S14"
+    },
+    "operational" : true,
+    "make_model" : "Delmatic IP_BW6"
+  },
+  "timestamp" : "2023-03-02T19:27:38.819Z",
+  "version" : 1
+}

udmis/bin/run

@@ -22,7 +22,7 @@ java -version
 
 POD_JAR=build/libs/udmis-1.0-SNAPSHOT-all.jar
 STACK_OPTS=-XX:-OmitStackTraceInFastThrow
-DEBUG_OPTS=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005
+#DEBUG_OPTS=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005
 
 gcloud_project=$(gcloud config get project || true)
 if [[ -z $GCP_PROJECT && -z $gcloud_project ]]; then

validator/src/main/java/com/google/daq/mqtt/validator/Validator.java

@@ -22,6 +22,7 @@
 import static com.google.udmi.util.Common.SUBTYPE_PROPERTY_KEY;
 import static com.google.udmi.util.Common.TIMESTAMP_KEY;
 import static com.google.udmi.util.Common.UPDATE_QUERY_TOPIC;
+import static com.google.udmi.util.Common.UPGRADED_FROM;
 import static com.google.udmi.util.Common.getExceptionMessage;
 import static com.google.udmi.util.Common.getNamespacePrefix;
 import static com.google.udmi.util.Common.removeNextArg;
@@ -108,7 +109,6 @@
 import java.util.stream.Collectors;
 import java.util.stream.StreamSupport;
 import org.apache.commons.io.FileUtils;
-import org.jetbrains.annotations.NotNull;
 import org.slf4j.impl.SimpleLogger;
 import udmi.schema.Category;
 import udmi.schema.DeviceValidationEvents;
@@ -141,6 +141,7 @@ public class Validator {
   public static final String TIMESTAMP_UTC_SUFFIX_2 = "+0000";
   public static final String ATTRIBUTE_FILE_FORMAT = "%s.attr";
   public static final String MESSAGE_FILE_FORMAT = "%s.json";
+  public static final String ORIG_FILE_FORMAT = "%s.orig";
   private static final String SCHEMA_VALIDATION_FORMAT = "Validating %d schemas";
   private static final String TARGET_VALIDATION_FORMAT = "Validating %d files against %s";
   private static final String DEVICE_FILE_FORMAT = "devices/%s";
@@ -503,7 +504,6 @@ private void initializeExpectedDevices() {
     }
   }
 
-  @NotNull
   private ReportingDevice newReportingDevice(String device) {
     ReportingDevice reportingDevice = new ReportingDevice(device);
     ifTrueThen(validateCurrent, () -> reportingDevice.setThreshold(reportingDelay));
@@ -753,8 +753,16 @@ private ReportingDevice validateMessageCore(Object messageObj, Map<String, Strin
       if (processExceptions(attributes, deviceId, device, message)) {
         return device;
       }
+
       validateDeviceMessage(device, message, attributes);
 
+      if (message.containsKey(UPGRADED_FROM)) {
+        File jsonFile = getDeviceOutCaptureFile(deviceId, schemaName, false);
+        File origFile = getDeviceOutCaptureFile(deviceId, schemaName, true);
+        jsonFile.renameTo(origFile);
+        writeDeviceOutCapture(message, attributes, deviceId, schemaName);
+      }
+
       validationStats.update();
 
       if (!device.hasErrors()) {
@@ -960,9 +968,7 @@ private boolean shouldProcessMessage(Map<String, String> attributes) {
   private void writeDeviceOutCapture(Object message, Map<String, String> attributes,
       String deviceId, String schemaName) throws IOException {
 
-    File deviceDir = makeDeviceDir(deviceId);
-
-    File messageFile = new File(deviceDir, format(MESSAGE_FILE_FORMAT, schemaName));
+    File messageFile = getDeviceOutCaptureFile(deviceId, schemaName, false);
 
     if (message instanceof Map) {
       Map<String, Object> messageMap = mapCast(message);
@@ -975,10 +981,17 @@ private void writeDeviceOutCapture(Object message, Map<String, String> attribute
       OBJECT_MAPPER.writeValue(messageFile, message);
     }
 
+    File deviceDir = makeDeviceDir(deviceId);
     File attributesFile = new File(deviceDir, format(ATTRIBUTE_FILE_FORMAT, schemaName));
     OBJECT_MAPPER.writeValue(attributesFile, attributes);
   }
 
+  private File getDeviceOutCaptureFile(String deviceId, String schemaName, boolean orig) {
+    File deviceDir = makeDeviceDir(deviceId);
+    String fileFormat = orig ? ORIG_FILE_FORMAT : MESSAGE_FILE_FORMAT;
+    return new File(deviceDir, format(fileFormat, schemaName));
+  }
+
   private File makeDeviceDir(String deviceId) {
     File deviceDir = new File(outBaseDir, format(DEVICE_FILE_FORMAT, deviceId));
     deviceDir.mkdirs();