Seeing timing metrics and config.yaml; timing metrics are failing to …

…delete in some cases, needs investigation

src/main/java/com/emc/mongoose/load/step/weighted/WeightedLoadStepClient.java

@@ -22,9 +22,7 @@
 import static com.github.akurilov.commons.lang.Exceptions.throwUnchecked;
 import static com.github.akurilov.confuse.Config.deepToMap;
 
-public class WeightedLoadStepClient
-				extends LoadStepClientBase {
-
+public class WeightedLoadStepClient extends LoadStepClientBase {
 	public WeightedLoadStepClient(
 					final Config config, final List<Extension> extensions, final List<Config> ctxConfigs,
 					final MetricsManager metricsManager) {
@@ -43,14 +41,14 @@ protected <T extends LoadStepClient> T copyInstance(final Config config, final L
 	}
 
 	@Override
-	protected void init()
-					throws IllegalStateException {
+	protected void init() throws IllegalStateException {
 		final String autoStepId = "weighted_" + LogUtil.getDateTimeStamp();
 		final Config stepConfig = config.configVal("load-step");
 		if (stepConfig.boolVal("idAutoGenerated")) {
 			stepConfig.val("id", autoStepId);
 		}
 		final int subStepCount = ctxConfigs.size();
+
 		// 2nd pass: initialize the sub steps
 		for (int originIndex = 0; originIndex < subStepCount; originIndex++) {
 			final Map<String, Object> mergedConfigTree = reduceForest(
@@ -74,6 +72,7 @@ protected void init()
 				itemDataSize = new SizeInBytes(TypeUtil.typeConvert(itemDataSizeRaw, long.class));
 			}
 			final boolean colorFlag = outputConfig.boolVal("color");
+			stepConfig.val("id", subConfig.stringVal("load-step-id")); // Overwrite step id with sub config step id
 			initMetrics(originIndex, opType, concurrencyLimit, metricsConfig, itemDataSize, colorFlag);
 		}
 	}

src/main/java/com/emc/mongoose/load/step/weighted/WeightedLoadStepLocal.java

@@ -1,12 +1,16 @@
 package com.emc.mongoose.load.step.weighted;
 
 import com.emc.mongoose.base.env.Extension;
+import com.emc.mongoose.base.config.ConfigFormat;
+import com.emc.mongoose.base.config.ConfigUtil;
 import com.emc.mongoose.base.config.IllegalConfigurationException;
+import com.emc.mongoose.base.Constants;
 import com.emc.mongoose.base.data.DataInput;
 import com.emc.mongoose.base.item.op.OpType;
 import com.emc.mongoose.base.item.Item;
 import com.emc.mongoose.base.item.ItemFactory;
 import com.emc.mongoose.base.item.io.ItemInfoFileOutput;
+import com.emc.mongoose.base.item.io.ItemTimingMetricsFileOutput;
 import com.emc.mongoose.base.item.ItemType;
 import com.emc.mongoose.base.load.generator.LoadGenerator;
 import com.emc.mongoose.base.load.generator.LoadGeneratorBuilder;
@@ -34,6 +38,7 @@
 import com.github.akurilov.confuse.impl.BasicConfig;
 
 import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.ThreadContext;
 
 import java.io.IOException;
 import java.nio.file.Files;
@@ -44,9 +49,7 @@
 import java.util.List;
 import java.util.Map;
 
-public class WeightedLoadStepLocal
-				extends LoadStepLocalBase {
-
+public class WeightedLoadStepLocal extends LoadStepLocalBase {
 	public WeightedLoadStepLocal(
 					final Config baseConfig, final List<Extension> extensions, final List<Config> contextConfigs,
 					final MetricsManager metricsManager) {
@@ -59,9 +62,7 @@ public String getTypeName() {
 	}
 
 	@Override
-	protected void init()
-					 {
-
+	protected void init() {
 		final String autoStepId = "weighted_" + LogUtil.getDateTimeStamp();
 		final Config stepConfig = config.configVal("load-step");
 		if (stepConfig.boolVal("idAutoGenerated")) {
@@ -91,9 +92,9 @@ protected void init()
 
 		// 2nd pass: initialize the sub steps
 		for (int originIndex = 0; originIndex < subStepCount; originIndex++) {
-
 			final Config subConfig = subConfigs.get(originIndex);
 			final Config loadConfig = subConfig.configVal("load");
+			final Config subStepConfig = loadConfig.configVal("step");
 			final Config opConfig = loadConfig.configVal("op");
 			final OpType opType = OpType.valueOf(opConfig.stringVal("type").toUpperCase());
 			final Config storageConfig = subConfig.configVal("storage");
@@ -117,10 +118,9 @@ protected void init()
 			final Config limitConfig = stepConfig.configVal("limit");
 			final Config dataLayerConfig = dataInputConfig.configVal("layer");
 
-			final String testStepId = stepConfig.stringVal("id");
+			final String testStepId = subStepConfig.stringVal("id");
 
 			try {
-
 				final Object dataLayerSizeRaw = dataLayerConfig.val("size");
 				final SizeInBytes dataLayerSize;
 				if (dataLayerSizeRaw instanceof String) {
@@ -137,7 +137,6 @@ protected void init()
 				final int batchSize = loadConfig.intVal("batch-size");
 
 				try {
-
 					final StorageDriver driver = StorageDriver.instance(
 									extensions, storageConfig, dataInput, dataConfig.boolVal("verify"), batchSize, testStepId);
 
@@ -159,7 +158,6 @@ protected void init()
 							generatorBuilder.addThrottle(new RateThrottle(rateLimit));
 						}
 						final LoadGenerator generator = generatorBuilder.build();
-
 						final LoadStepContext stepCtx = new LoadStepContextImpl<>(
 										testStepId, generator, driver, metricsContexts.get(originIndex), loadConfig,
 										outputConfig.boolVal("metrics-trace-persist"));
@@ -181,6 +179,22 @@ protected void init()
 								);
 							}
 						}
+
+						final Path itemTimingMetricsOutputPath = Paths.get(System.getProperty("java.io.tmpdir"),
+								"mongoose", "timingMetrics_" + testStepId);
+						try {
+							final Output<? extends Item> itemOutput = new ItemTimingMetricsFileOutput<>(
+									itemTimingMetricsOutputPath);
+							stepCtx.operationsMetricsOutput(itemOutput);
+						} catch (final IOException e) {
+							LogUtil.exception(
+									Level.ERROR, e,
+									"Failed to initialize the item metrics output, the processed items info won't be persisted");
+						}
+
+						// Overwrite step config with sub step config
+						ThreadContext.put(Constants.KEY_STEP_ID, testStepId);
+						Loggers.CONFIG.info(ConfigUtil.toString(subConfig, ConfigFormat.YAML));
 					} catch (final IllegalConfigurationException e) {
 						throw new IllegalStateException("Failed to initialize the load generator", e);
 					}