Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

enhance ag noise analysis + dashboard + unit test such that every ele… #86

Merged
merged 1 commit into from
Sep 9, 2024
Merged

enhance ag noise analysis + dashboard + unit test such that every ele… #86

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 58 additions & 16 deletions src/main/java/org/matsim/analysis/postAnalysis/NoiseAverageAnalysis.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,8 @@
@CommandLine.Command(name = "average-noise", description = "Calculates average noise stats based on several sim runs with different random seeds.")
@CommandSpec(
requires = {"runs"},
produces = {"mean_emission_per_day.csv", "mean_immission_per_day.avro", "mean_immission_per_hour.avro"}
produces = {"mean_emission_per_day.csv", "mean_immission_per_day.avro", "mean_immission_per_hour.avro", "mean_noise_stats.csv",
"mean_damages_receiverPoint_per_day.avro", "mean_damages_receiverPoint_per_hour.avro"}
)
public class NoiseAverageAnalysis implements MATSimAppCommand {
private final Logger log = LogManager.getLogger(NoiseAverageAnalysis.class);
Expand All @@ -58,6 +59,10 @@ public class NoiseAverageAnalysis implements MATSimAppCommand {
private List<GenericRecord> imissionsPerHour = new ArrayList<>();
private Map<String, List<Double>> emissionsPerDay = new HashMap<>();
private Map<String, Double> meanEmissionsPerDay = new HashMap<>();
private Map<String, List<Double>> totalStats = new HashMap<>();
private Map<String, Double> meanTotalStatsPerDay = new HashMap<>();
private List<GenericRecord> damagesPerDay = new ArrayList<>();
private List<GenericRecord> damagesPerHour = new ArrayList<>();


public static void main(String[] args) {
Expand All @@ -81,16 +86,27 @@ public Integer call() throws Exception {
String emissionsCsv = globFile(analysisDir, "*emission_per_day.csv*").toString();
String imissionsPerDayAvro = globFile(analysisDir, "*immission_per_day.avro*").toString();
String imissionsPerHourAvro = globFile(analysisDir, "*immission_per_hour.avro*").toString();
String totalStatsCsv = globFile(analysisDir, "*noise_stats.csv*").toString();
String damagesPerDayAvro = globFile(analysisDir, "*damages_receiverPoint_per_day.avro*").toString();
String damagesPerHourAvro = globFile(analysisDir, "*damages_receiverPoint_per_hour.avro*").toString();

// read
Table emissions = Table.read().csv(CsvReadOptions.builder(IOUtils.getBufferedReader(emissionsCsv))
.columnTypesPartial(Map.of(LINK_ID, ColumnType.STRING, VALUE, ColumnType.DOUBLE))
.sample(false)
.separator(CsvOptions.detectDelimiter(emissionsCsv)).build());

// read avro file
Table totalStatsTable = Table.read().csv(CsvReadOptions.builder(IOUtils.getBufferedReader(totalStatsCsv))
.columnTypes(new ColumnType[]{ColumnType.STRING, ColumnType.DOUBLE})
.header(false)
.sample(false)
.separator(CsvOptions.detectDelimiter(totalStatsCsv)).build());

// read avro files
readAvroFile(imissionsPerDayAvro, imissionsPerDay);
readAvroFile(imissionsPerHourAvro, imissionsPerHour);
readAvroFile(damagesPerDayAvro, damagesPerDay);
readAvroFile(damagesPerHourAvro, damagesPerHour);

// get all emission stats
for (int i = 0; i < emissions.rowCount(); i++) {
Expand All @@ -101,20 +117,28 @@ public Integer call() throws Exception {
}
emissionsPerDay.get(row.getString(LINK_ID)).add(row.getDouble(VALUE));
}
}

// calc emission means and write to mean map
for (Map.Entry<String, List<Double>> e : emissionsPerDay.entrySet()) {
AtomicReference<Double> sum = new AtomicReference<>(0.);
e.getValue().forEach(d -> sum.set(sum.get() + d));
// get all total stats
for (int i = 0; i < totalStatsTable.rowCount(); i++) {
Row row = totalStatsTable.row(i);

meanEmissionsPerDay.put(e.getKey(), sum.get() / e.getValue().size());
if (!totalStats.containsKey(row.getString(0))) {
totalStats.put(row.getString(0), new ArrayList<>());
}
totalStats.get(row.getString(0)).add(row.getDouble(1));
}
}

// calc avro means
XYTData imissionsPerDayMean = calcAvroMeans(imissionsPerDay);
XYTData imissionsPerHourMean = calcAvroMeans(imissionsPerHour);
// calc emission means and write to mean map
calcCsvMeans(emissionsPerDay, meanEmissionsPerDay);
// calc mean total stats and write to mean map
calcCsvMeans(totalStats, meanTotalStatsPerDay);

// calc avro means
XYTData imissionsPerDayMean = calcAvroMeans(imissionsPerDay, "imissions");
XYTData imissionsPerHourMean = calcAvroMeans(imissionsPerHour, "imissions");
XYTData damagesPerDayMean = calcAvroMeans(damagesPerDay, "damages_receiverPoint");
XYTData damagesPerHourMean = calcAvroMeans(damagesPerHour, "damages_receiverPoint");

// write emission mean stats
try (CSVPrinter printer = new CSVPrinter(Files.newBufferedWriter(output.getPath("mean_emission_per_day.csv")), CSVFormat.DEFAULT)) {
Expand All @@ -125,15 +149,33 @@ public Integer call() throws Exception {
}
}

// write total mean stats
try (CSVPrinter printer = new CSVPrinter(Files.newBufferedWriter(output.getPath("mean_noise_stats.csv")), CSVFormat.DEFAULT)) {
for (Map.Entry<String, Double> e : meanTotalStatsPerDay.entrySet()) {
printer.printRecord(e.getKey(), e.getValue());
}
}

// write avro mean files
writeAvro(imissionsPerDayMean, new File(output.getPath("mean_immission_per_day.avro").toString()));
writeAvro(imissionsPerHourMean, new File(output.getPath("mean_immission_per_hour.avro").toString()));
writeAvro(damagesPerDayMean, new File(output.getPath("mean_damages_receiverPoint_per_day.avro").toString()));
writeAvro(damagesPerHourMean, new File(output.getPath("mean_damages_receiverPoint_per_hour.avro").toString()));

return 0;
}

private void calcCsvMeans(Map<String, List<Double>> dataMap, Map<String, Double> meanMap) {
for (Map.Entry<String, List<Double>> e : dataMap.entrySet()) {
AtomicReference<Double> sum = new AtomicReference<>(0.);
e.getValue().forEach(d -> sum.set(sum.get() + d));

meanMap.put(e.getKey(), sum.get() / e.getValue().size());
}
}

/**
* write an .avro file containing immission data.
* write an .avro file containing immission / damage data.
*/
public void writeAvro(XYTData xytData, File outputFile) {
DatumWriter<XYTData> datumWriter = new SpecificDatumWriter<>(XYTData.class);
Expand All @@ -146,7 +188,7 @@ public void writeAvro(XYTData xytData, File outputFile) {
}
}

private XYTData calcAvroMeans(List<GenericRecord> recordList) {
private XYTData calcAvroMeans(List<GenericRecord> recordList, String dataFieldName) {
String crs = null;
List<Float> xCoords = new ArrayList<>();
List<Float> yCoords = new ArrayList<>();
Expand Down Expand Up @@ -189,7 +231,7 @@ private XYTData calcAvroMeans(List<GenericRecord> recordList) {
List<Float> values = new ArrayList<>();

for (Map.Entry<?, ?> entry : ((HashMap<?, ?>) object4).entrySet()) {
if (entry.getKey() instanceof Utf8 && entry.getKey().toString().equals("imissions") && entry.getValue() instanceof GenericData.Array<?>) {
if (entry.getKey() instanceof Utf8 && entry.getKey().toString().equals(dataFieldName) && entry.getValue() instanceof GenericData.Array<?>) {
values.addAll((GenericData.Array<Float>) entry.getValue());

String entryString = ((Utf8) entry.getKey()).toString();
Expand All @@ -211,7 +253,7 @@ private XYTData calcAvroMeans(List<GenericRecord> recordList) {
// calc mean values for each datapoint out of sums and number of records (1 record = 1 run seed)
data.entrySet()
.stream()
.filter(entry -> entry.getKey().equals("imissions"))
.filter(entry -> entry.getKey().equals(dataFieldName))
.forEach(entry -> entry.getValue()
.forEach(value -> entry.getValue().set(entry.getValue().indexOf(value), value / recordList.size())));

Expand All @@ -234,7 +276,7 @@ private void getCoordData(Object object, List<Float> target) {
}

/**
* read an .avro file containing immissions.
* read an .avro file containing immissions / damages.
*/
public void readAvroFile(String input, List<GenericRecord> target) {
try {
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/org/matsim/dashboard/AverageKelheimEmissionsDashboard.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ public void configure(Header header, Layout layout) {
viz.height = 12.0;
viz.datasets.csvFile = postProcess(data, "mean_emissions_per_link_per_m.csv");
viz.datasets.csvBase = Path.of(this.dirs.get(0)).getParent().relativize(Path.of(pathToCsvBase)).toString();
viz.network = new CreateAverageDashboards().copyGeoJsonNetwork(dirs);
viz.network = new CreateAverageDashboards().copyVizNetwork(dirs, ".avro");
viz.display.color.columnName = "CO2_TOTAL [g/m]";
viz.display.color.dataset = "csvFile";
viz.display.width.scaleFactor = 100;
Expand Down
74 changes: 54 additions & 20 deletions src/main/java/org/matsim/dashboard/AverageKelheimNoiseDashboard.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import org.matsim.simwrapper.viz.ColorScheme;
import org.matsim.simwrapper.viz.GridMap;
import org.matsim.simwrapper.viz.MapPlot;
import org.matsim.simwrapper.viz.Tile;

import java.util.ArrayList;
import java.util.List;
Expand All @@ -26,6 +27,8 @@ public class AverageKelheimNoiseDashboard implements Dashboard {
private static final String LIGHT_BLUE = "#95c7df";
private static final String ORANGE = "#f4a986";
private static final String RED = "#cc0c27";
private static final String SAND = "#dfb095";
private static final String YELLOW = "#dfdb95";

public AverageKelheimNoiseDashboard(List<String> dirs, Integer noRuns) {
this.dirs = dirs;
Expand All @@ -45,51 +48,82 @@ public void configure(Header header, Layout layout) {
header.title = "Average Noise";
header.description = "Shows the average noise footprint and spatial distribution for several simulation runs.";

layout.row("aggregate noise")
.el(GridMap.class, (viz, data) -> {
viz.title = "Noise Immissions (Grid)";
viz.description = "Aggregate Noise Immissions per day";
viz.height = 12.0;
viz.cellSize = 250;
viz.opacity = 0.2;
viz.maxHeight = 20;
viz.center = data.context().getCenter();
viz.zoom = data.context().mapZoomLevel;
viz.setColorRamp(new double[]{40, 50, 60}, new String[]{DARK_BLUE, LIGHT_BLUE, ORANGE, RED});
viz.file = postProcess(data, "mean_immission_per_day.avro");
})
layout.row("stats")
.el(Tile.class, (viz, data) -> {
viz.dataset = postProcess(data, "mean_noise_stats.csv");
viz.height = 0.1;
});

layout.row("emissions")
.el(MapPlot.class, (viz, data) -> {
viz.title = "Noise Emissions (Link)";
viz.description = "Aggregate Noise Emissions per day";
viz.description = "Maximum Noise Level per day [dB]";
viz.height = 12.0;
viz.center = data.context().getCenter();
viz.zoom = data.context().mapZoomLevel;
viz.minValue = minDb;
viz.maxValue = maxDb;
viz.setShape(new CreateAverageDashboards().copyGeoJsonNetwork(dirs));
viz.setShape(new CreateAverageDashboards().copyVizNetwork(dirs, ".avro"));
viz.addDataset(NOISE, postProcess(data, "mean_emission_per_day.csv"));
viz.display.lineColor.dataset = NOISE;
viz.display.lineColor.columnName = "value";
viz.display.lineColor.join = "Link Id";
viz.display.lineColor.fixedColors = new String[]{DARK_BLUE, LIGHT_BLUE, ORANGE, RED};
viz.display.lineColor.setColorRamp(ColorScheme.RdYlBu, 4, true, "45, 55, 65");
viz.display.lineColor.setColorRamp(ColorScheme.Oranges, 8, false, "35, 45, 55, 65, 75, 85, 95");
viz.display.lineWidth.dataset = NOISE;
viz.display.lineWidth.columnName = "value";
viz.display.lineWidth.scaleFactor = 8d;
viz.display.lineWidth.join = "Link Id";
});
layout.row("hourly noise")
layout.row("immissions")
.el(GridMap.class, (viz, data) -> {
viz.title = "Hourly Noise Immissions (Grid)";
viz.description = "Noise Immissions per hour";
viz.title = "Noise Immissions (Grid)";
viz.description = "Total Noise Immissions per day";
viz.height = 12.0;
viz.cellSize = 250;
viz.opacity = 0.2;
viz.maxHeight = 20;
viz.center = data.context().getCenter();
viz.zoom = data.context().mapZoomLevel;
viz.setColorRamp(new double[]{40, 50, 60}, new String[]{DARK_BLUE, LIGHT_BLUE, ORANGE, RED});
viz.setColorRamp(new double[]{30, 40, 50, 60, 70}, new String[]{DARK_BLUE, LIGHT_BLUE, YELLOW, SAND, ORANGE, RED});
viz.file = postProcess(data, "mean_immission_per_day.avro");
})
.el(GridMap.class, (viz, data) -> {
viz.title = "Hourly Noise Immissions (Grid)";
viz.description = "Noise Immissions per hour";
viz.height = 12.0;
viz.cellSize = 250;
viz.opacity = 0.1;
viz.maxHeight = 40;
viz.center = data.context().getCenter();
viz.zoom = data.context().mapZoomLevel;
viz.setColorRamp(new double[]{30, 40, 50, 60, 70}, new String[]{DARK_BLUE, LIGHT_BLUE, YELLOW, SAND, ORANGE, RED});
viz.file = postProcess(data, "mean_immission_per_hour.avro");
});
layout.row("damages")
.el(GridMap.class, (viz, data) -> {
viz.title = "Daily Noise Damages (Grid)";
viz.description = "Total Noise Damages per day [€]";
viz.height = 12.0;
viz.cellSize = 250;
viz.opacity = 0.1;
viz.maxHeight = 40;
viz.center = data.context().getCenter();
viz.zoom = data.context().mapZoomLevel;
viz.setColorRamp(new double[]{30, 40, 50, 60, 70}, new String[]{DARK_BLUE, LIGHT_BLUE, YELLOW, SAND, ORANGE, RED});
viz.file = postProcess(data, "mean_damages_receiverPoint_per_day.avro");
})
.el(GridMap.class, (viz, data) -> {
viz.title = "Hourly Noise Damages (Grid)";
viz.description = "Noise Damages per hour [€]";
viz.height = 12.0;
viz.cellSize = 250;
viz.opacity = 0.2;
viz.maxHeight = 40;
viz.center = data.context().getCenter();
viz.zoom = data.context().mapZoomLevel;
viz.setColorRamp(new double[]{30, 40, 50, 60, 70}, new String[]{DARK_BLUE, LIGHT_BLUE, YELLOW, SAND, ORANGE, RED});
viz.file = postProcess(data, "mean_damages_receiverPoint_per_hour.avro");
});
}
}
10 changes: 5 additions & 5 deletions src/main/java/org/matsim/dashboard/CreateAverageDashboards.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,23 +82,23 @@ public Integer call() throws Exception {
}

/**
* A helper method to copy an already existing Geojson network rather than creating it all over again.
* A helper method to copy an already existing Geojson / avro network rather than creating it all over again.
*/
String copyGeoJsonNetwork(List<String> dirs) {
String copyVizNetwork(List<String> dirs, String fileType) {

for (String dir : dirs) {
File networkFile = new File(dir + "/analysis/network/network.geojson");
File networkFile = new File(dir + "/analysis/network/network" + fileType);
Path target = Path.of(Path.of(dir).getParent() + "/analysis/network");

if (Files.notExists(target) && networkFile.exists() && networkFile.isFile()) {
try {
Files.createDirectories(target);
Files.copy(networkFile.toPath(), Path.of(target + "/network.geojson"));
Files.copy(networkFile.toPath(), Path.of(target + "/network" + fileType));
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
}
return "analysis/network/network.geojson";
return "analysis/network/network" + fileType;
}
}
Loading
Loading