Release/1.17.7

DAQAggregator_example.properties

@@ -59,6 +59,13 @@ socksproxy.enableproxy = true
 socksproy.host = localhost
 socksproxy.port = 1080
 
+
+f3.enabled = true
+f3.htl.url = http://es-cdaq.cms/sc/php/stream_summary_last.php
+f3.disk.url = http://es-cdaq.cms/sc/php/summarydisks.php
+f3.crashes.url = http://es-cdaq.cms/sc/php/resource_status.php
+
+
 #
 # Flag indicating if flashlist discovery should be based on static catalog, by default based on dynamic
 #

pom.xml

@@ -3,7 +3,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>ch.cern</groupId>
 	<artifactId>DAQAggregator</artifactId>
-	<version>1.17.6</version>
+	<version>1.17.7</version>
 	<repositories>
 		<repository>
 			<id>libs-release-local</id>

src/main/java/rcms/utilities/daqaggregator/DAQAggregator.java

@@ -69,7 +69,7 @@ public static void main(String[] args) {
             }
 
 			/*
-             * Persist mode from properties file
+			 * Persist mode from properties file
 			 */
             PersistMode persistMode = PersistMode.decode(Application.get().getProp(Settings.PERSISTENCE_MODE));
             logger.info("Persist mode:" + persistMode);
@@ -282,12 +282,13 @@ public static Pair<MonitorManager, PersistorManager> initialize(RunMode runMode)
         boolean f3Enabled = Boolean.parseBoolean(Application.get().getProp(Settings.F3_ENABLED));
         String hltUrl = Application.get().getProp(Settings.F3_HLT_URL);
         String diskUrl = Application.get().getProp(Settings.F3_DISK_URL);
+        String crashesUrl = Application.get().getProp(Settings.F3_CRASHES_URL);
         F3DataRetriever f3DataRetriever = null;
 
 
-        if (f3Enabled && !"".equals(hltUrl) && !"".equals(diskUrl)) {
+        if (f3Enabled && !"".equals(hltUrl) && !"".equals(diskUrl) && !"".equals(crashesUrl)) {
             logger.info("F3 monitoring is enabled and set to following urls: " + hltUrl + ", " + diskUrl);
-            f3DataRetriever = new F3DataRetriever(new Connector(false), hltUrl, diskUrl);
+            f3DataRetriever = new F3DataRetriever(new Connector(false), hltUrl, diskUrl,crashesUrl);
         } else if (f3Enabled) {
             throw new DAQException(DAQExceptionCode.MissingProperty, "Specify url for F3 data retrieval. Required: " + Settings.F3_DISK_URL.getKey() + ", " + Settings.F3_HLT_URL.getKey());
         } else {

src/main/java/rcms/utilities/daqaggregator/Settings.java

@@ -21,6 +21,7 @@ public enum Settings {
 	F3_ENABLED("f3.enabled", true),
 	F3_HLT_URL("f3.htl.url"),
 	F3_DISK_URL("f3.disk.url"),
+	F3_CRASHES_URL("f3.crashes.url"),
 
 	// settings for monitoring
 	MONITOR_SETUPNAME("monitor.setupName"),

src/main/java/rcms/utilities/daqaggregator/data/DAQ.java

@@ -64,6 +64,8 @@ public class DAQ implements FlashlistUpdatable {
 
 	private TCDSGlobalInfo tcdsGlobalInfo;
 
+	private HltInfo hltInfo;
+
 	/**
 	 * HLT stream physics output rate in Hz (events per second)
 	 */
@@ -548,6 +550,15 @@ public void setDaqAggregatorProducer(String daqAggregatorProducer) {
 		this.daqAggregatorProducer = daqAggregatorProducer;
 	}
 
+	public HltInfo getHltInfo() {
+		return hltInfo;
+	}
+
+	public void setHltInfo(HltInfo hltInfo) {
+		this.hltInfo = hltInfo;
+	}
+
+
 	@Override
 	public String toString() {
 		return "DAQ [sessionId=" + sessionId + ", dpsetPath=" + dpsetPath + ", runNumber=" + runNumber + ", runStart="

src/main/java/rcms/utilities/daqaggregator/data/HltInfo.java

@@ -0,0 +1,14 @@
+package rcms.utilities.daqaggregator.data;
+
+public class HltInfo  {
+
+    private Integer crashes;
+
+    public Integer getCrashes() {
+        return crashes;
+    }
+
+    public void setCrashes(Integer crashes) {
+        this.crashes = crashes;
+    }
+}

src/main/java/rcms/utilities/daqaggregator/datasource/F3DataRetriever.java

@@ -31,12 +31,14 @@ public class F3DataRetriever {
     private final ObjectMapper mapper;
     private final String hltUrl;
     private final String diskUrl;
+    private final String crashUrl;
 
-    public F3DataRetriever(Connector connector, String hltUrl, String diskUrl) {
+    public F3DataRetriever(Connector connector, String hltUrl, String diskUrl, String crashUrl) {
         this.mapper = new ObjectMapper();
         this.connector = connector;
         this.hltUrl = hltUrl;
         this.diskUrl = diskUrl;
+        this.crashUrl = crashUrl;
     }
 
     /**
@@ -45,22 +47,128 @@ public F3DataRetriever(Connector connector, String hltUrl, String diskUrl) {
      * @param args
      */
     public static void main(String[] args) {
-        F3DataRetriever f3dr = new F3DataRetriever(new Connector(false), "http://es-cdaq.cms/sc/php/stream_summary_last.php", "http://es-cdaq.cms/sc/php/summarydisks.php");
+        F3DataRetriever f3dr = new F3DataRetriever(new Connector(false), "http://es-cdaq.cms/sc/php/stream_summary_last.php", "http://es-cdaq.cms/sc/php/summarydisks.php", "http://es-cdaq.cms/sc/php/resource_status.php");
         try {
             Application.initialize("DAQAggregator.properties");
             ProxyManager.get().startProxy();
             DiskInfo d = f3dr.getDiskInfo();
             Double h = f3dr.getHLToutputInfo(288498).getEventRate(PHYSICS_STREAM_NAME);
+            Integer c = f3dr.getCrashes();
 
-            logger.info(d);
-            logger.info(h);
+            logger.info("Disk info: " + d);
+            logger.info("Hlt output: " + h);
+            logger.info("Crashes: " + c);
         } catch (JsonMappingException e) {
             e.printStackTrace();
         } catch (IOException e) {
             e.printStackTrace();
         }
     }
 
+    public void dispatch(DAQ daq) {
+
+        long start = System.currentTimeMillis();
+
+        boolean diskSuccessful = dispatchDisk(daq);
+        boolean hltSuccessful = dispatchHLT(daq);
+        boolean crashSuccessful = dispatchCrashes(daq);
+
+        long end = System.currentTimeMillis();
+
+        if (diskSuccessful && hltSuccessful && crashSuccessful)
+            logger.info("F3 data successfully retrieved and mapped in: " + (end - start) + "ms");
+        else {
+            logger.warn("Problem retrieving F3 data [disk successful,hlt successful]=[" + diskSuccessful + ","
+                    + hltSuccessful + "," +crashSuccessful+ "]");
+        }
+    }
+
+    /**
+     * @return true if retrieving HLT output information from F3mon
+     * was successful, false otherwise
+     */
+    protected boolean dispatchHLT(DAQ daq) {
+
+        try {
+            HLToutputInfo hltInfo = getHLToutputInfo(daq.getRunNumber());
+
+            Double hltOutputRate = hltInfo.getEventRate(PHYSICS_STREAM_NAME);
+            Double hltOutputBW = hltInfo.getBandwidth(PHYSICS_STREAM_NAME);
+
+            daq.setHltRate(hltOutputRate);
+            daq.setHltBandwidth(hltOutputBW);
+
+            return hltOutputRate != null;
+
+        } catch (JsonMappingException e) {
+            logger.warn("Could not retrieve F3 HLT rate,  json mapping exception: ", e);
+        } catch (IOException e) {
+            logger.warn("Could not retrieve F3 HLT rate, IO exception: ", e);
+        }
+        return false;
+
+    }
+
+    protected boolean dispatchCrashes(DAQ daq) {
+        Integer crashes = getCrashes();
+        daq.getHltInfo().setCrashes(crashes);
+        if (crashes != null)
+            return true;
+        else
+            return false;
+    }
+
+    protected Integer getCrashes() {
+        Pair<Integer, List<String>> a = null;
+        try {
+            a = connector.retrieveLines(crashUrl + "?setup=cdaq");
+
+            List<String> result = a.getRight();
+
+            long count = result.size();
+            if (count == 1) {
+                JsonNode resultJson = mapper.readValue(result.get(0), JsonNode.class);
+
+                try {
+                    Integer crashes = resultJson.get("crashes").asInt();
+                    return crashes;
+
+                } catch (NoSuchElementException e) {
+                    logger.warn("Cannot retrieve HLT crashes (no such element) from response: " + result.get(0));
+                } catch (NullPointerException e) {
+                    logger.warn("Cannot retrieve HLT crashes from response: " + result.get(0));
+                }
+            }
+
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+        return null;
+    }
+
+
+    protected boolean dispatchDisk(DAQ daq) {
+
+        try {
+            DiskInfo d = getDiskInfo();
+            if (d != null) {
+                daq.getBuSummary().setOutputDiskTotal(d.getOutputTotal());
+                daq.getBuSummary().setOutputDiskUsage(d.getOutputOccupancyFraction());
+                return true;
+            } else {
+                daq.getBuSummary().setOutputDiskTotal(null);
+                daq.getBuSummary().setOutputDiskUsage(null);
+            }
+
+        } catch (JsonMappingException e) {
+            logger.warn("Could not retrieve F3 disk info,  json mapping exception: ", e);
+        } catch (IOException e) {
+            logger.warn("Could not retrieve F3 disk info, IO exception: ", e);
+        }
+        return false;
+
+    }
+
     /**
      * @param events if true, fills event rates otherwise fills bandwidths
      */
@@ -112,7 +220,7 @@ private void fillHLTInfo(int runNumber, HLToutputInfo hltOutputInfo, boolean eve
         }
     }
 
-    public HLToutputInfo getHLToutputInfo(int runNumber) throws IOException {
+    protected HLToutputInfo getHLToutputInfo(int runNumber) throws IOException {
         HLToutputInfo info = new HLToutputInfo();
 
         // fill event rates
@@ -132,7 +240,7 @@ public HLToutputInfo getHLToutputInfo(int runNumber) throws IOException {
      * @throws IOException
      * @throws JsonMappingException
      */
-    public DiskInfo getDiskInfo() throws IOException, JsonMappingException {
+    protected DiskInfo getDiskInfo() throws IOException, JsonMappingException {
 
         Pair<Integer, List<String>> a = connector.retrieveLines(diskUrl);
         List<String> result = a.getRight();
@@ -162,70 +270,6 @@ public DiskInfo getDiskInfo() throws IOException, JsonMappingException {
 
     }
 
-    public void dispatch(DAQ daq) {
-
-        long start = System.currentTimeMillis();
-
-        boolean diskSuccessful = dispatchDisk(daq);
-        boolean hltSuccessful = dispatchHLT(daq);
-
-        long end = System.currentTimeMillis();
-
-        if (diskSuccessful && hltSuccessful)
-            logger.info("F3 data successfully retrieved and mapped in: " + (end - start) + "ms");
-        else {
-            logger.warn("Problem retrieving F3 data [disk successful,hlt successful]=[" + diskSuccessful + ","
-                    + hltSuccessful + "]");
-        }
-    }
-
-    /**
-     * @return true if retrieving HLT output information from F3mon
-     * was successful, false otherwise
-     */
-    public boolean dispatchHLT(DAQ daq) {
-
-        try {
-            HLToutputInfo hltInfo = getHLToutputInfo(daq.getRunNumber());
-
-            Double hltOutputRate = hltInfo.getEventRate(PHYSICS_STREAM_NAME);
-            Double hltOutputBW = hltInfo.getBandwidth(PHYSICS_STREAM_NAME);
-
-            daq.setHltRate(hltOutputRate);
-            daq.setHltBandwidth(hltOutputBW);
-
-            return hltOutputRate != null;
-
-        } catch (JsonMappingException e) {
-            logger.warn("Could not retrieve F3 HLT rate,  json mapping exception: ", e);
-        } catch (IOException e) {
-            logger.warn("Could not retrieve F3 HLT rate, IO exception: ", e);
-        }
-        return false;
-
-    }
-
-    public boolean dispatchDisk(DAQ daq) {
-
-        try {
-            DiskInfo d = getDiskInfo();
-            if (d != null) {
-                daq.getBuSummary().setOutputDiskTotal(d.getOutputTotal());
-                daq.getBuSummary().setOutputDiskUsage(d.getOutputOccupancyFraction());
-                return true;
-            } else {
-                daq.getBuSummary().setOutputDiskTotal(null);
-                daq.getBuSummary().setOutputDiskUsage(null);
-            }
-
-        } catch (JsonMappingException e) {
-            logger.warn("Could not retrieve F3 disk info,  json mapping exception: ", e);
-        } catch (IOException e) {
-            logger.warn("Could not retrieve F3 disk info, IO exception: ", e);
-        }
-        return false;
-
-    }
 
     public class DiskInfo {
         private Double ramdiskOccupancyFraction;

src/main/java/rcms/utilities/daqaggregator/mappers/ObjectMapper.java

@@ -14,25 +14,8 @@
 import org.apache.log4j.Logger;
 
 
+import rcms.utilities.daqaggregator.data.*;
 import rcms.utilities.hwcfg.HardwareConfigurationException;
-import rcms.utilities.daqaggregator.data.BU;
-import rcms.utilities.daqaggregator.data.BUSummary;
-import rcms.utilities.daqaggregator.data.DAQ;
-import rcms.utilities.daqaggregator.data.FED;
-import rcms.utilities.daqaggregator.data.FEDBuilder;
-import rcms.utilities.daqaggregator.data.FEDBuilderSummary;
-import rcms.utilities.daqaggregator.data.FMM;
-import rcms.utilities.daqaggregator.data.FMMApplication;
-import rcms.utilities.daqaggregator.data.FRL;
-import rcms.utilities.daqaggregator.data.FRLPc;
-import rcms.utilities.daqaggregator.data.FRLType;
-import rcms.utilities.daqaggregator.data.GlobalTTSState;
-import rcms.utilities.daqaggregator.data.FMMType;
-import rcms.utilities.daqaggregator.data.RU;
-import rcms.utilities.daqaggregator.data.SubFEDBuilder;
-import rcms.utilities.daqaggregator.data.SubSystem;
-import rcms.utilities.daqaggregator.data.TCDSGlobalInfo;
-import rcms.utilities.daqaggregator.data.TTCPartition;
 import rcms.utilities.daqaggregator.datasource.TCDSFMInfoRetriever;
 import rcms.utilities.hwcfg.HardwareConfigurationException;
 import rcms.utilities.hwcfg.dp.DAQPartition;
@@ -217,6 +200,7 @@ public void mapAllObjects(DAQPartition daqPartition) {
 		daq.setFrls(new ArrayList<>(frls.values()));
 		daq.setFeds(new ArrayList<>(feds.values()));
 		daq.setSubFEDBuilders(new ArrayList<>(subFedBuilders.values()));
+		daq.setHltInfo(new HltInfo());
 
 		logger.info("Retrieval summary " + this.toString());
 		logger.info("Subsystem summary " + subSystems.values());