Skip to content

Commit

Permalink
enhance ag noise analysis + dashboard + unit test such that every ele…
Browse files Browse the repository at this point in the history 
…ment of the enhanced noise single run dashboard is displayed
  • Loading branch information
@simei94
simei94 committed Sep 5, 2024
1 parent 3a219d3 commit aa72a93
Show file tree
Hide file tree
Showing 5 changed files with 197 additions and 75 deletions.
74 changes: 58 additions & 16 deletions src/main/java/org/matsim/analysis/postAnalysis/NoiseAverageAnalysis.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,8 @@
@CommandLine.Command(name = "average-noise", description = "Calculates average noise stats based on several sim runs with different random seeds.")
@CommandSpec(
requires = {"runs"},
produces = {"mean_emission_per_day.csv", "mean_immission_per_day.avro", "mean_immission_per_hour.avro"}
produces = {"mean_emission_per_day.csv", "mean_immission_per_day.avro", "mean_immission_per_hour.avro", "mean_noise_stats.csv",
"mean_damages_receiverPoint_per_day.avro", "mean_damages_receiverPoint_per_hour.avro"}
)
public class NoiseAverageAnalysis implements MATSimAppCommand {
private final Logger log = LogManager.getLogger(NoiseAverageAnalysis.class);
Expand All @@ -58,6 +59,10 @@ public class NoiseAverageAnalysis implements MATSimAppCommand {
private List<GenericRecord> imissionsPerHour = new ArrayList<>();
private Map<String, List<Double>> emissionsPerDay = new HashMap<>();
private Map<String, Double> meanEmissionsPerDay = new HashMap<>();
private Map<String, List<Double>> totalStats = new HashMap<>();
private Map<String, Double> meanTotalStatsPerDay = new HashMap<>();
private List<GenericRecord> damagesPerDay = new ArrayList<>();
private List<GenericRecord> damagesPerHour = new ArrayList<>();


public static void main(String[] args) {
Expand All @@ -81,16 +86,27 @@ public Integer call() throws Exception {
String emissionsCsv = globFile(analysisDir, "*emission_per_day.csv*").toString();
String imissionsPerDayAvro = globFile(analysisDir, "*immission_per_day.avro*").toString();
String imissionsPerHourAvro = globFile(analysisDir, "*immission_per_hour.avro*").toString();
String totalStatsCsv = globFile(analysisDir, "*noise_stats.csv*").toString();
String damagesPerDayAvro = globFile(analysisDir, "*damages_receiverPoint_per_day.avro*").toString();
String damagesPerHourAvro = globFile(analysisDir, "*damages_receiverPoint_per_hour.avro*").toString();

// read
Table emissions = Table.read().csv(CsvReadOptions.builder(IOUtils.getBufferedReader(emissionsCsv))
.columnTypesPartial(Map.of(LINK_ID, ColumnType.STRING, VALUE, ColumnType.DOUBLE))
.sample(false)
.separator(CsvOptions.detectDelimiter(emissionsCsv)).build());

// read avro file
Table totalStatsTable = Table.read().csv(CsvReadOptions.builder(IOUtils.getBufferedReader(totalStatsCsv))
.columnTypes(new ColumnType[]{ColumnType.STRING, ColumnType.DOUBLE})
.header(false)
.sample(false)
.separator(CsvOptions.detectDelimiter(totalStatsCsv)).build());

// read avro files
readAvroFile(imissionsPerDayAvro, imissionsPerDay);
readAvroFile(imissionsPerHourAvro, imissionsPerHour);
readAvroFile(damagesPerDayAvro, damagesPerDay);
readAvroFile(damagesPerHourAvro, damagesPerHour);

// get all emission stats
for (int i = 0; i < emissions.rowCount(); i++) {
Expand All @@ -101,20 +117,28 @@ public Integer call() throws Exception {
}
emissionsPerDay.get(row.getString(LINK_ID)).add(row.getDouble(VALUE));
}
}

// calc emission means and write to mean map
for (Map.Entry<String, List<Double>> e : emissionsPerDay.entrySet()) {
AtomicReference<Double> sum = new AtomicReference<>(0.);
e.getValue().forEach(d -> sum.set(sum.get() + d));
// get all total stats
for (int i = 0; i < totalStatsTable.rowCount(); i++) {
Row row = totalStatsTable.row(i);

meanEmissionsPerDay.put(e.getKey(), sum.get() / e.getValue().size());
if (!totalStats.containsKey(row.getString(0))) {
totalStats.put(row.getString(0), new ArrayList<>());
}
totalStats.get(row.getString(0)).add(row.getDouble(1));
}
}

// calc avro means
XYTData imissionsPerDayMean = calcAvroMeans(imissionsPerDay);
XYTData imissionsPerHourMean = calcAvroMeans(imissionsPerHour);
// calc emission means and write to mean map
calcCsvMeans(emissionsPerDay, meanEmissionsPerDay);
// calc mean total stats and write to mean map
calcCsvMeans(totalStats, meanTotalStatsPerDay);

// calc avro means
XYTData imissionsPerDayMean = calcAvroMeans(imissionsPerDay, "imissions");
XYTData imissionsPerHourMean = calcAvroMeans(imissionsPerHour, "imissions");
XYTData damagesPerDayMean = calcAvroMeans(damagesPerDay, "damages_receiverPoint");
XYTData damagesPerHourMean = calcAvroMeans(damagesPerHour, "damages_receiverPoint");

// write emission mean stats
try (CSVPrinter printer = new CSVPrinter(Files.newBufferedWriter(output.getPath("mean_emission_per_day.csv")), CSVFormat.DEFAULT)) {
Expand All @@ -125,15 +149,33 @@ public Integer call() throws Exception {
}
}

// write total mean stats
try (CSVPrinter printer = new CSVPrinter(Files.newBufferedWriter(output.getPath("mean_noise_stats.csv")), CSVFormat.DEFAULT)) {
for (Map.Entry<String, Double> e : meanTotalStatsPerDay.entrySet()) {
printer.printRecord(e.getKey(), e.getValue());
}
}

// write avro mean files
writeAvro(imissionsPerDayMean, new File(output.getPath("mean_immission_per_day.avro").toString()));
writeAvro(imissionsPerHourMean, new File(output.getPath("mean_immission_per_hour.avro").toString()));
writeAvro(damagesPerDayMean, new File(output.getPath("mean_damages_receiverPoint_per_day.avro").toString()));
writeAvro(damagesPerHourMean, new File(output.getPath("mean_damages_receiverPoint_per_hour.avro").toString()));

return 0;
}

private void calcCsvMeans(Map<String, List<Double>> dataMap, Map<String, Double> meanMap) {
for (Map.Entry<String, List<Double>> e : dataMap.entrySet()) {
AtomicReference<Double> sum = new AtomicReference<>(0.);
e.getValue().forEach(d -> sum.set(sum.get() + d));

meanMap.put(e.getKey(), sum.get() / e.getValue().size());
}
}

/**
* write an .avro file containing immission data.
* write an .avro file containing immission / damage data.
*/
public void writeAvro(XYTData xytData, File outputFile) {
DatumWriter<XYTData> datumWriter = new SpecificDatumWriter<>(XYTData.class);
Expand All @@ -146,7 +188,7 @@ public void writeAvro(XYTData xytData, File outputFile) {
}
}

private XYTData calcAvroMeans(List<GenericRecord> recordList) {
private XYTData calcAvroMeans(List<GenericRecord> recordList, String dataFieldName) {
String crs = null;
List<Float> xCoords = new ArrayList<>();
List<Float> yCoords = new ArrayList<>();
Expand Down Expand Up @@ -189,7 +231,7 @@ private XYTData calcAvroMeans(List<GenericRecord> recordList) {
List<Float> values = new ArrayList<>();

for (Map.Entry<?, ?> entry : ((HashMap<?, ?>) object4).entrySet()) {
if (entry.getKey() instanceof Utf8 && entry.getKey().toString().equals("imissions") && entry.getValue() instanceof GenericData.Array<?>) {
if (entry.getKey() instanceof Utf8 && entry.getKey().toString().equals(dataFieldName) && entry.getValue() instanceof GenericData.Array<?>) {
values.addAll((GenericData.Array<Float>) entry.getValue());

String entryString = ((Utf8) entry.getKey()).toString();
Expand All @@ -211,7 +253,7 @@ private XYTData calcAvroMeans(List<GenericRecord> recordList) {
// calc mean values for each datapoint out of sums and number of records (1 record = 1 run seed)
data.entrySet()
.stream()
.filter(entry -> entry.getKey().equals("imissions"))
.filter(entry -> entry.getKey().equals(dataFieldName))
.forEach(entry -> entry.getValue()
.forEach(value -> entry.getValue().set(entry.getValue().indexOf(value), value / recordList.size())));

Expand All @@ -234,7 +276,7 @@ private void getCoordData(Object object, List<Float> target) {
}

/**
* read an .avro file containing immissions.
* read an .avro file containing immissions / damages.
*/
public void readAvroFile(String input, List<GenericRecord> target) {
try {
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/org/matsim/dashboard/AverageKelheimEmissionsDashboard.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ public void configure(Header header, Layout layout) {
viz.height = 12.0;
viz.datasets.csvFile = postProcess(data, "mean_emissions_per_link_per_m.csv");
viz.datasets.csvBase = Path.of(this.dirs.get(0)).getParent().relativize(Path.of(pathToCsvBase)).toString();
viz.network = new CreateAverageDashboards().copyGeoJsonNetwork(dirs);
viz.network = new CreateAverageDashboards().copyVizNetwork(dirs, ".avro");
viz.display.color.columnName = "CO2_TOTAL [g/m]";
viz.display.color.dataset = "csvFile";
viz.display.width.scaleFactor = 100;
Expand Down
74 changes: 54 additions & 20 deletions src/main/java/org/matsim/dashboard/AverageKelheimNoiseDashboard.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import org.matsim.simwrapper.viz.ColorScheme;
import org.matsim.simwrapper.viz.GridMap;
import org.matsim.simwrapper.viz.MapPlot;
import org.matsim.simwrapper.viz.Tile;

import java.util.ArrayList;
import java.util.List;
Expand All @@ -26,6 +27,8 @@ public class AverageKelheimNoiseDashboard implements Dashboard {
private static final String LIGHT_BLUE = "#95c7df";
private static final String ORANGE = "#f4a986";
private static final String RED = "#cc0c27";
private static final String SAND = "#dfb095";
private static final String YELLOW = "#dfdb95";

public AverageKelheimNoiseDashboard(List<String> dirs, Integer noRuns) {
this.dirs = dirs;
Expand All @@ -45,51 +48,82 @@ public void configure(Header header, Layout layout) {
header.title = "Average Noise";
header.description = "Shows the average noise footprint and spatial distribution for several simulation runs.";

layout.row("aggregate noise")
.el(GridMap.class, (viz, data) -> {
viz.title = "Noise Immissions (Grid)";
viz.description = "Aggregate Noise Immissions per day";
viz.height = 12.0;
viz.cellSize = 250;
viz.opacity = 0.2;
viz.maxHeight = 20;
viz.center = data.context().getCenter();
viz.zoom = data.context().mapZoomLevel;
viz.setColorRamp(new double[]{40, 50, 60}, new String[]{DARK_BLUE, LIGHT_BLUE, ORANGE, RED});
viz.file = postProcess(data, "mean_immission_per_day.avro");
})
layout.row("stats")
.el(Tile.class, (viz, data) -> {
viz.dataset = postProcess(data, "mean_noise_stats.csv");
viz.height = 0.1;
});

layout.row("emissions")
.el(MapPlot.class, (viz, data) -> {
viz.title = "Noise Emissions (Link)";
viz.description = "Aggregate Noise Emissions per day";
viz.description = "Maximum Noise Level per day [dB]";
viz.height = 12.0;
viz.center = data.context().getCenter();
viz.zoom = data.context().mapZoomLevel;
viz.minValue = minDb;
viz.maxValue = maxDb;
viz.setShape(new CreateAverageDashboards().copyGeoJsonNetwork(dirs));
viz.setShape(new CreateAverageDashboards().copyVizNetwork(dirs, ".avro"));
viz.addDataset(NOISE, postProcess(data, "mean_emission_per_day.csv"));
viz.display.lineColor.dataset = NOISE;
viz.display.lineColor.columnName = "value";
viz.display.lineColor.join = "Link Id";
viz.display.lineColor.fixedColors = new String[]{DARK_BLUE, LIGHT_BLUE, ORANGE, RED};
viz.display.lineColor.setColorRamp(ColorScheme.RdYlBu, 4, true, "45, 55, 65");
viz.display.lineColor.setColorRamp(ColorScheme.Oranges, 8, false, "35, 45, 55, 65, 75, 85, 95");
viz.display.lineWidth.dataset = NOISE;
viz.display.lineWidth.columnName = "value";
viz.display.lineWidth.scaleFactor = 8d;
viz.display.lineWidth.join = "Link Id";
});
layout.row("hourly noise")
layout.row("immissions")
.el(GridMap.class, (viz, data) -> {
viz.title = "Hourly Noise Immissions (Grid)";
viz.description = "Noise Immissions per hour";
viz.title = "Noise Immissions (Grid)";
viz.description = "Total Noise Immissions per day";
viz.height = 12.0;
viz.cellSize = 250;
viz.opacity = 0.2;
viz.maxHeight = 20;
viz.center = data.context().getCenter();
viz.zoom = data.context().mapZoomLevel;
viz.setColorRamp(new double[]{40, 50, 60}, new String[]{DARK_BLUE, LIGHT_BLUE, ORANGE, RED});
viz.setColorRamp(new double[]{30, 40, 50, 60, 70}, new String[]{DARK_BLUE, LIGHT_BLUE, YELLOW, SAND, ORANGE, RED});
viz.file = postProcess(data, "mean_immission_per_day.avro");
})
.el(GridMap.class, (viz, data) -> {
viz.title = "Hourly Noise Immissions (Grid)";
viz.description = "Noise Immissions per hour";
viz.height = 12.0;
viz.cellSize = 250;
viz.opacity = 0.1;
viz.maxHeight = 40;
viz.center = data.context().getCenter();
viz.zoom = data.context().mapZoomLevel;
viz.setColorRamp(new double[]{30, 40, 50, 60, 70}, new String[]{DARK_BLUE, LIGHT_BLUE, YELLOW, SAND, ORANGE, RED});
viz.file = postProcess(data, "mean_immission_per_hour.avro");
});
layout.row("damages")
.el(GridMap.class, (viz, data) -> {
viz.title = "Daily Noise Damages (Grid)";
viz.description = "Total Noise Damages per day [€]";
viz.height = 12.0;
viz.cellSize = 250;
viz.opacity = 0.1;
viz.maxHeight = 40;
viz.center = data.context().getCenter();
viz.zoom = data.context().mapZoomLevel;
viz.setColorRamp(new double[]{30, 40, 50, 60, 70}, new String[]{DARK_BLUE, LIGHT_BLUE, YELLOW, SAND, ORANGE, RED});
viz.file = postProcess(data, "mean_damages_receiverPoint_per_day.avro");
})
.el(GridMap.class, (viz, data) -> {
viz.title = "Hourly Noise Damages (Grid)";
viz.description = "Noise Damages per hour [€]";
viz.height = 12.0;
viz.cellSize = 250;
viz.opacity = 0.2;
viz.maxHeight = 40;
viz.center = data.context().getCenter();
viz.zoom = data.context().mapZoomLevel;
viz.setColorRamp(new double[]{30, 40, 50, 60, 70}, new String[]{DARK_BLUE, LIGHT_BLUE, YELLOW, SAND, ORANGE, RED});
viz.file = postProcess(data, "mean_damages_receiverPoint_per_hour.avro");
});
}
}
10 changes: 5 additions & 5 deletions src/main/java/org/matsim/dashboard/CreateAverageDashboards.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,23 +82,23 @@ public Integer call() throws Exception {
}

/**
* A helper method to copy an already existing Geojson network rather than creating it all over again.
* A helper method to copy an already existing Geojson / avro network rather than creating it all over again.
*/
String copyGeoJsonNetwork(List<String> dirs) {
String copyVizNetwork(List<String> dirs, String fileType) {

for (String dir : dirs) {
File networkFile = new File(dir + "/analysis/network/network.geojson");
File networkFile = new File(dir + "/analysis/network/network" + fileType);
Path target = Path.of(Path.of(dir).getParent() + "/analysis/network");

if (Files.notExists(target) && networkFile.exists() && networkFile.isFile()) {
try {
Files.createDirectories(target);
Files.copy(networkFile.toPath(), Path.of(target + "/network.geojson"));
Files.copy(networkFile.toPath(), Path.of(target + "/network" + fileType));
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
}
return "analysis/network/network.geojson";
return "analysis/network/network" + fileType;
}
}
Loading

0 comments on commit aa72a93

Please sign in to comment.