From 45eececa83d0c529af81db65afe8b4d6e8d7a6ee Mon Sep 17 00:00:00 2001 From: simei94 Date: Wed, 15 May 2024 19:07:37 +0200 Subject: [PATCH] add average emissions dashboard infrastructure WIP --- ...missionsPostProcessingAverageAnalysis.java | 223 ++++++++++++++++++ .../AverageKelheimEmissionsDashboard.java | 119 ++++++++++ .../dashboard/CreateAverageDashboards.java | 11 +- 3 files changed, 350 insertions(+), 3 deletions(-) create mode 100644 src/main/java/org/matsim/analysis/postAnalysis/EmissionsPostProcessingAverageAnalysis.java create mode 100644 src/main/java/org/matsim/dashboard/AverageKelheimEmissionsDashboard.java diff --git a/src/main/java/org/matsim/analysis/postAnalysis/EmissionsPostProcessingAverageAnalysis.java b/src/main/java/org/matsim/analysis/postAnalysis/EmissionsPostProcessingAverageAnalysis.java new file mode 100644 index 00000000..fbc9895c --- /dev/null +++ b/src/main/java/org/matsim/analysis/postAnalysis/EmissionsPostProcessingAverageAnalysis.java @@ -0,0 +1,223 @@ +package org.matsim.analysis.postAnalysis; + +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVPrinter; +import org.matsim.api.core.v01.Coord; +import org.matsim.application.CommandSpec; +import org.matsim.application.MATSimAppCommand; +import org.matsim.application.options.CsvOptions; +import org.matsim.application.options.InputOptions; +import org.matsim.application.options.OutputOptions; +import org.matsim.core.utils.io.IOUtils; +import picocli.CommandLine; +import tech.tablesaw.api.ColumnType; +import tech.tablesaw.api.Row; +import tech.tablesaw.api.Table; +import tech.tablesaw.io.csv.CsvReadOptions; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.*; +import java.util.concurrent.atomic.AtomicReference; + +import static org.matsim.application.ApplicationUtils.globFile; + +@CommandLine.Command(name = "average-emissions", description = "Calculates average emission stats based on several sim runs with different random seeds.") +@CommandSpec( + requires = {"runs"}, + produces = {"mean_emissions_total.csv", "mean_emissions_per_link_per_m.csv", "mean_emissions_grid_per_day.xyt.csv", "mean_emissions_grid_per_hour.csv"} +) +public class EmissionsPostProcessingAverageAnalysis implements MATSimAppCommand { + + @CommandLine.Mixin + private InputOptions input = InputOptions.ofCommand(EmissionsPostProcessingAverageAnalysis.class); + @CommandLine.Mixin + private OutputOptions output = OutputOptions.ofCommand(EmissionsPostProcessingAverageAnalysis.class); + @CommandLine.Option(names = "--no-runs", defaultValue = "5", description = "Number of simulation runs to be averaged.") + private Integer noRuns; + + private final Map> totalStats = new HashMap<>(); + private final Map> perLinkMStats = new HashMap<>(); + private final Map, List> gridPerDayStats = new HashMap<>(); + private final Map, List> gridPerHourStats = new HashMap<>(); + private final Map meanTotal = new HashMap<>(); + private final Map meanPerLinkM = new HashMap<>(); + private final Map, Double> meanGridPerDay = new HashMap<>(); + private final Map, Double> meanGridPerHour = new HashMap<>(); + + private final CsvOptions csv = new CsvOptions(); + String value = "value"; + + public static void main(String[] args) { + new EmissionsPostProcessingAverageAnalysis().execute(args); + } + + @Override + public Integer call() throws Exception { + + String runs = input.getPath("runs"); + + List foldersSeeded = Arrays.stream(runs.split(",")).toList(); + +// add stats from every run to map + for (String folder : foldersSeeded) { + String totalCsv = globFile(Path.of(folder + "/analysis/emissions" ), "*emissions_total.csv*").toString(); + String emissionsPerLinkMCsv = globFile(Path.of(folder + "/analysis/emissions"), "*emissions_per_link_per_m.csv*").toString(); + String emissionsGridPerDayCsv = globFile(Path.of(folder + "/analysis/emissions"), "*emissions_grid_per_day.xyt.csv*").toString(); + String emissionsGridPerHourCsv = globFile(Path.of(folder + "/analysis/emissions"), "*emissions_grid_per_hour.csv*").toString(); + + Table total = Table.read().csv(CsvReadOptions.builder(IOUtils.getBufferedReader(totalCsv)) + .sample(false) + .separator(csv.detectDelimiter(totalCsv)).build()); + + Table emissionsPerLinkM = Table.read().csv(CsvReadOptions.builder(IOUtils.getBufferedReader(emissionsPerLinkMCsv)) + .sample(false) + .separator(csv.detectDelimiter(emissionsPerLinkMCsv)).build()); + +// TODO: update matsim version to newest for necessary changes in detectDelimiter method + Table emissionsGridPerDay = Table.read().csv(CsvReadOptions.builder(IOUtils.getBufferedReader(emissionsGridPerDayCsv)) + .sample(false) + .separator(csv.detectDelimiter(emissionsGridPerDayCsv)).header(true).build()); + + Table emissionsGridPerHour = Table.read().csv(CsvReadOptions.builder(IOUtils.getBufferedReader(emissionsGridPerHourCsv)) + .sample(false) + .separator(csv.detectDelimiter(emissionsGridPerHourCsv)).build()); + +// get all total stats + for (int i = 0; i < total.rowCount(); i++) { + Row row = total.row(i); + + if (!totalStats.containsKey(row.getString(i))) { + totalStats.put(row.getString(i), new ArrayList<>()); + } + +// some values are in format hh:mm:ss or empty + if (row.getString("kg").isEmpty()) { + totalStats.get(row.getString("Pollutant")).add(0.); + } else { + totalStats.get(row.getString("Pollutant")).add(row.getDouble("kg")); + } + } + +// get all per link per m stats + for (int i = 0; i < emissionsPerLinkM.rowCount(); i++) { + Row row = emissionsPerLinkM.row(i); + Double[] values = new Double[emissionsPerLinkM.columnCount() - 1]; + +// iterate through columns. this file contains 23 params per link, as of may24 + for (int j = 1; i < emissionsPerLinkM.columnCount() - 1; j++) { + if (!perLinkMStats.containsKey(row.getString(i))) { + perLinkMStats.put(row.getString(i), new ArrayList<>()); + } + + if (row.getColumnType(j) == ColumnType.INTEGER) { + values[j - 1] = (double) row.getInt(j); + } else { + values[j - 1] = row.getDouble(j); + } + } + perLinkMStats.get(row.getString(i)).add(values); + } + +// get all grid per day stats + getGridData(emissionsGridPerDay, gridPerDayStats); +// get all grid per day stats + getGridData(emissionsGridPerHour, gridPerHourStats); + } + +// calc means for every map +// total means + for (Map.Entry> e : totalStats.entrySet()) { + AtomicReference sum = new AtomicReference<>(0.); + e.getValue().forEach(d -> sum.set(sum.get() + d)); + + meanTotal.put(e.getKey(), sum.get() / e.getValue().size()); + } + +// per linkM means + for (Map.Entry> e : perLinkMStats.entrySet()) { + + Double[] sums = new Double[e.getValue().get(0).length]; + + for (Double[] d : e.getValue()) { + for (int i = 0; i <= d.length - 1; i++) { + sums[i] += d[i]; + } + } + + Double[] means = new Double[sums.length]; + for (int i = 0; i <= sums.length - 1; i++) { + means[i] = sums[i] / e.getValue().size(); + } + meanPerLinkM.put(e.getKey(), means); + } + +// grid per day means + calcGridMeans(gridPerDayStats, meanGridPerDay); +// grid per hour means + calcGridMeans(gridPerHourStats, meanGridPerHour); + +// write total mean stats + try (CSVPrinter printer = new CSVPrinter(Files.newBufferedWriter(output.getPath("mean_emissions_total.csv")), CSVFormat.DEFAULT)) { + printer.printRecord("Pollutant", "kg"); + + for (Map.Entry e : meanTotal.entrySet()) { + printer.printRecord("mean-" + e.getKey(), e.getValue()); + } + } + +// write per linkM mean stats + try (CSVPrinter printer = new CSVPrinter(Files.newBufferedWriter(output.getPath("mean_emissions_per_link_per_m.csv")), CSVFormat.DEFAULT)) { + printer.printRecord("linkId", "CO [g/m]", "CO2_TOTAL [g/m]", "FC [g/m]", "HC [g/m]", "NMHC [g/m]", "NOx [g/m]", "NO2 [g/m]", "PM [g/m]", "SO2 [g/m]", + "FC_MJ [g/m]", "CO2_rep [g/m]", "CO2e [g/m]", "PM2_5 [g/m]", "PM2_5_non_exhaust [g/m]", "PM_non_exhaust [g/m]", "BC_exhaust [g/m]", "BC_non_exhaust [g/m]", + "Benzene [g/m]", "PN [g/m]", "Pb [g/m]", "CH4 [g/m]", "N2O [g/m]", "NH3 [g/m]" + ); + + for (Map.Entry e : meanPerLinkM.entrySet()) { + printer.printRecord(e.getKey(), e.getValue()[0], e.getValue()[1], e.getValue()[2], e.getValue()[3], e.getValue()[4], e.getValue()[5], + e.getValue()[6], e.getValue()[7], e.getValue()[8], e.getValue()[9], e.getValue()[10], e.getValue()[11], e.getValue()[12], e.getValue()[13], + e.getValue()[14], e.getValue()[15], e.getValue()[16], e.getValue()[17], e.getValue()[18], e.getValue()[19], e.getValue()[20], e.getValue()[21], + e.getValue()[22]); + } + } + +// write grid mean stats + writeGridFile("mean_emissions_grid_per_day.xyt.csv", meanGridPerDay); + writeGridFile("mean_emissions_grid_per_hour.csv", meanGridPerHour); + + return 0; + } + + private void calcGridMeans(Map, List> originMap, Map, Double> targetMap) { + for (Map.Entry, List> e : originMap.entrySet()) { + AtomicReference sum = new AtomicReference<>(0.); + e.getValue().forEach(d -> sum.set(sum.get() + d)); + + targetMap.put(e.getKey(), sum.get() / e.getValue().size()); + } + } + + private void getGridData(Table gridTable, Map, List> dataMap) { + for (int i = 0; i < gridTable.rowCount(); i++) { + Row row = gridTable.row(i); + Map.Entry entry = new AbstractMap.SimpleEntry<>(row.getDouble("time"), new Coord(row.getDouble("x"), row.getDouble("y"))); + + dataMap.computeIfAbsent(entry, key -> new ArrayList<>()); + dataMap.get(entry).add(row.getDouble(value)); + } + } + + private void writeGridFile(String fileName, Map, Double> values) throws IOException { + try (CSVPrinter printer = new CSVPrinter(Files.newBufferedWriter(output.getPath(fileName)), CSVFormat.DEFAULT)) { + + printer.printRecord("# EPSG:25832"); + printer.printRecord("time", "x", "y", value); + + for (Map.Entry, Double> e : values.entrySet()) { + printer.printRecord(e.getKey().getKey(), e.getKey().getValue().getX(), e.getKey().getValue().getY(), e.getValue()); + } + } + } +} + diff --git a/src/main/java/org/matsim/dashboard/AverageKelheimEmissionsDashboard.java b/src/main/java/org/matsim/dashboard/AverageKelheimEmissionsDashboard.java new file mode 100644 index 00000000..e5dc6e81 --- /dev/null +++ b/src/main/java/org/matsim/dashboard/AverageKelheimEmissionsDashboard.java @@ -0,0 +1,119 @@ +/* *********************************************************************** * + * project: org.matsim.* + * Controler.java + * * + * *********************************************************************** * + * * + * copyright : (C) 2007 by the members listed in the COPYING, * + * LICENSE and WARRANTY file. * + * email : info at matsim dot org * + * * + * *********************************************************************** * + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * See also COPYING, LICENSE and WARRANTY file * + * * + * *********************************************************************** */ + +package org.matsim.dashboard; + +import org.matsim.analysis.postAnalysis.EmissionsPostProcessingAverageAnalysis; +import org.matsim.analysis.postAnalysis.drt.DrtPostProcessingAverageAnalysis; +import org.matsim.application.prepare.network.CreateGeoJsonNetwork; +import org.matsim.simwrapper.Dashboard; +import org.matsim.simwrapper.Data; +import org.matsim.simwrapper.Header; +import org.matsim.simwrapper.Layout; +import org.matsim.simwrapper.viz.GridMap; +import org.matsim.simwrapper.viz.Links; +import org.matsim.simwrapper.viz.Table; + +import java.util.ArrayList; +import java.util.List; + +/** + * Average emissions dashboard for several runs with the same config but a different random seed. + */ +public class AverageKelheimEmissionsDashboard implements Dashboard{ + private final List dirs; + private final Integer noRuns; + private final String pathToCsvBase; + + public AverageKelheimEmissionsDashboard(List dirs, Integer noRuns) { + this.dirs = dirs; + this.noRuns = noRuns; + this.pathToCsvBase = null; + } + + public AverageKelheimEmissionsDashboard(List dirs, Integer noRuns, String pathToBaseRun) { + this.dirs = dirs; + this.noRuns = noRuns; + + if (!pathToBaseRun.endsWith("/")) { + pathToBaseRun += "/"; + } + this.pathToCsvBase = pathToBaseRun + "analysis/emissions/emissions_per_link_per_m.csv"; + } + + private String postProcess(Data data, String outputFile) { +// args for analysis have to be: list of paths to run dirs + drt modes / folder names + List args = new ArrayList<>(List.of("--input-runs", String.join(",", dirs), "--no-runs", noRuns.toString())); + + return data.compute(EmissionsPostProcessingAverageAnalysis.class, outputFile, args.toArray(new String[0])); + } + + /** + * Produces the dashboard. + */ + public void configure(Header header, Layout layout) { + header.title = "Average Emissions"; + header.description = "Shows the average emissions footprint and spatial distribution for several simulation runs."; + + String linkDescription = "Displays the emissions for each link per meter. Be aware that emission values are provided in the simulation sample size!"; + if (pathToCsvBase != null){ + linkDescription += String.format("\n Base is %s", pathToCsvBase); + } + String finalLinkDescription = linkDescription; + + layout.row("links") + .el(Table.class, (viz, data) -> { + viz.title = "Emissions"; + viz.description = "by pollutant. Total values are scaled from the simulation sample size to 100%."; + viz.dataset = postProcess(data, "mean_emissions_total.csv"); + viz.enableFilter = false; + viz.showAllRows = true; + viz.width = 1.0; + }) + .el(Links.class, (viz, data) -> { + viz.title = "Emissions per Link per Meter"; + viz.description = finalLinkDescription; + viz.height = 12.0; + viz.datasets.csvFile = postProcess(data, "mean_emissions_per_link_per_m.csv"); + viz.datasets.csvBase = this.pathToCsvBase; + viz.network = data.compute(CreateGeoJsonNetwork.class, "network.geojson"); + viz.display.color.columnName = "CO2_TOTAL [g/m]"; + viz.display.color.dataset = "csvFile"; + viz.display.width.scaleFactor = 100; + viz.display.width.columnName = "CO2_TOTAL [g/m]"; + viz.display.width.dataset = "csvFile"; + viz.center = data.context().getCenter(); + viz.width = 3.0; + }); + layout.row("second").el(GridMap.class, (viz, data) -> { + viz.title = "CO₂ Emissions"; + viz.description = "per day. Be aware that CO2 values are provided in the simulation sample size!"; + viz.height = 12.0; + viz.file = postProcess(data, "mean_emissions_grid_per_day.xyt.csv"); + }); + layout.row("third") + .el(GridMap.class, (viz, data) -> { + viz.title = "CO₂ Emissions"; + viz.description = "per hour. Be aware that CO2 values are provided in the simulation sample size!"; + viz.height = 12.; + viz.file = postProcess(data, "mean_emissions_grid_per_hour.csv"); + }); + } +} diff --git a/src/main/java/org/matsim/dashboard/CreateAverageDashboards.java b/src/main/java/org/matsim/dashboard/CreateAverageDashboards.java index 13203a01..8c0dfe7a 100644 --- a/src/main/java/org/matsim/dashboard/CreateAverageDashboards.java +++ b/src/main/java/org/matsim/dashboard/CreateAverageDashboards.java @@ -21,6 +21,8 @@ public class CreateAverageDashboards implements MATSimAppCommand { private String inputPath; @CommandLine.Option(names = "--no-runs", defaultValue = "5", description = "Number of simulation runs to be averaged.") private Integer noRuns; + @CommandLine.Option(names = "--base-run", description = "Path to directory base run.", defaultValue = "/net/ils/matsim-kelheim/v3.0-release/output-base/25pct") + private String pathToBaseRun; public static void main(String[] args) { new CreateAverageDashboards().execute(args); @@ -60,11 +62,14 @@ public Integer call() throws Exception { .context(m); sw.addDashboard(d); -// TODO: rather call generate method with append true than the standard one bc we are in post processing - sw.generate(Path.of(inputPath)); - sw.run(Path.of(inputPath)); } + sw.addDashboard(Dashboard.customize(new AverageKelheimEmissionsDashboard(foldersSeeded, noRuns, pathToBaseRun)).context("emissions")); + +// TODO: rather call generate method with append true than the standard one bc we are in post processing + sw.generate(Path.of(inputPath), true); + sw.run(Path.of(inputPath)); + return 0;