From c7b48d7ebae8c8f34620f25cfcccb4aaa69bcde4 Mon Sep 17 00:00:00 2001 From: sime94 Date: Tue, 20 Aug 2024 16:59:57 +0200 Subject: [PATCH 1/4] add emission analysis as standard after sim analysis --- .../matsim/dashboards/LausitzDashboardProvider.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/matsim/dashboards/LausitzDashboardProvider.java b/src/main/java/org/matsim/dashboards/LausitzDashboardProvider.java index ea816cc..7c2589c 100644 --- a/src/main/java/org/matsim/dashboards/LausitzDashboardProvider.java +++ b/src/main/java/org/matsim/dashboards/LausitzDashboardProvider.java @@ -4,6 +4,7 @@ import org.matsim.simwrapper.Dashboard; import org.matsim.simwrapper.DashboardProvider; import org.matsim.simwrapper.SimWrapper; +import org.matsim.simwrapper.dashboard.EmissionsDashboard; import org.matsim.simwrapper.dashboard.TripDashboard; import java.util.List; @@ -14,12 +15,18 @@ public class LausitzDashboardProvider implements DashboardProvider { @Override public List getDashboards(Config config, SimWrapper simWrapper) { - return List.of(new TripDashboard( + + TripDashboard trips = new TripDashboard( "lausitz_mode_share.csv", "lausitz_mode_share_per_dist.csv", "lausitz_mode_users.csv") .withGroupedRefData("lausitz_mode_share_per_group_dist_ref.csv", "age", "economic_status") - .withDistanceDistribution("lausitz_mode_share_distance_distribution.csv") + .withDistanceDistribution("lausitz_mode_share_distance_distribution.csv"); + + return List.of(trips, + new EmissionsDashboard() +// the NoiseAnalysis is not run here because it needs more RAM than the entire simulation, +// which leads to VM crashes and prevents other analysis to run. We have to run it separately (e.g. with LausitzSimWrapperRunner) ); } } From 3986b356948b088bb8841a8fb0573da850241e4e Mon Sep 17 00:00:00 2001 From: sime94 Date: Tue, 20 Aug 2024 17:00:34 +0200 Subject: [PATCH 2/4] class LausitzSimWrapperRunner for running RAM intense noise analysis separately after sim --- .../dashboards/LausitzSimWrapperRunner.java | 104 ++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 src/main/java/org/matsim/dashboards/LausitzSimWrapperRunner.java diff --git a/src/main/java/org/matsim/dashboards/LausitzSimWrapperRunner.java b/src/main/java/org/matsim/dashboards/LausitzSimWrapperRunner.java new file mode 100644 index 0000000..bb61bda --- /dev/null +++ b/src/main/java/org/matsim/dashboards/LausitzSimWrapperRunner.java @@ -0,0 +1,104 @@ +/* *********************************************************************** * + * project: org.matsim.* + * Controler.java + * * + * *********************************************************************** * + * * + * copyright : (C) 2007 by the members listed in the COPYING, * + * LICENSE and WARRANTY file. * + * email : info at matsim dot org * + * * + * *********************************************************************** * + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * See also COPYING, LICENSE and WARRANTY file * + * * + * *********************************************************************** */ + +package org.matsim.dashboards; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.matsim.application.ApplicationUtils; +import org.matsim.application.MATSimAppCommand; +import org.matsim.application.options.ShpOptions; +import org.matsim.core.config.Config; +import org.matsim.core.config.ConfigUtils; +import org.matsim.simwrapper.Dashboard; +import org.matsim.simwrapper.SimWrapper; +import org.matsim.simwrapper.SimWrapperConfigGroup; +import org.matsim.simwrapper.dashboard.NoiseDashboard; +import picocli.CommandLine; + +import java.io.IOException; +import java.io.InterruptedIOException; +import java.nio.file.Path; +import java.util.List; + +@CommandLine.Command( + name = "simwrapper", + description = "Run additional analysis and create SimWrapper dashboard for existing run output." +) +final class LausitzSimWrapperRunner implements MATSimAppCommand { + + private static final Logger log = LogManager.getLogger(LausitzSimWrapperRunner.class); + + @CommandLine.Parameters(arity = "1..*", description = "Path to run output directories for which dashboards are to be generated.") + private List inputPaths; + + @CommandLine.Mixin + private final ShpOptions shp = new ShpOptions(); + + @CommandLine.Option(names = "--noise", defaultValue = "false", description = "create noise dashboard") + private boolean noise; + + + private LausitzSimWrapperRunner(){ + } + + @Override + public Integer call() throws Exception { + + if (!noise){ + throw new IllegalArgumentException("you have not configured any dashboard to be created! Please use command line parameters!"); + } + + for (Path runDirectory : inputPaths) { + log.info("Running on {}", runDirectory); + + Path configPath = ApplicationUtils.matchInput("config.xml", runDirectory); + Config config = ConfigUtils.loadConfig(configPath.toString()); + SimWrapper sw = SimWrapper.create(config); + + SimWrapperConfigGroup simwrapperCfg = ConfigUtils.addOrGetModule(config, SimWrapperConfigGroup.class); + if (shp.isDefined()){ + simwrapperCfg.defaultParams().shp = shp.getShapeFile(); + } + //skip default dashboards + simwrapperCfg.defaultDashboards = SimWrapperConfigGroup.Mode.disabled; + + //add dashboards according to command line parameters +// TODO: if more dashboards are to be added here, we need to check if noise==true before adding noise dashboard here + sw.addDashboard(Dashboard.customize(new NoiseDashboard()).context("noise")); + + + try { + sw.generate(runDirectory, true); + sw.run(runDirectory); + } catch (IOException e) { + throw new InterruptedIOException(); + } + } + + return 0; + } + + public static void main(String[] args) { + new LausitzSimWrapperRunner().execute(args); + + } + +} From fff091d8a6d5099cf9539dc7f40ea047a184474d Mon Sep 17 00:00:00 2001 From: sime94 Date: Wed, 21 Aug 2024 18:13:41 +0200 Subject: [PATCH 3/4] setup emission analysis correctly + test class --- .../java/org/matsim/run/LausitzScenario.java | 100 ++++++++++++++- .../run/EmissionAnalysisOutputTest.java | 115 ++++++++++++++++++ 2 files changed, 214 insertions(+), 1 deletion(-) create mode 100644 src/test/java/org/matsim/run/EmissionAnalysisOutputTest.java diff --git a/src/main/java/org/matsim/run/LausitzScenario.java b/src/main/java/org/matsim/run/LausitzScenario.java index f9c5445..3592878 100644 --- a/src/main/java/org/matsim/run/LausitzScenario.java +++ b/src/main/java/org/matsim/run/LausitzScenario.java @@ -17,6 +17,10 @@ import org.matsim.application.prepare.network.CreateNetworkFromSumo; import org.matsim.application.prepare.population.*; import org.matsim.application.prepare.pt.CreateTransitScheduleFromGtfs; +import org.matsim.contrib.emissions.HbefaRoadTypeMapping; +import org.matsim.contrib.emissions.HbefaVehicleCategory; +import org.matsim.contrib.emissions.OsmHbefaMapping; +import org.matsim.contrib.emissions.utils.EmissionsConfigGroup; import org.matsim.contrib.vsp.scenario.SnzActivities; import org.matsim.core.config.Config; import org.matsim.core.config.ConfigUtils; @@ -25,6 +29,7 @@ import org.matsim.core.config.groups.ScoringConfigGroup; import org.matsim.core.controler.AbstractModule; import org.matsim.core.controler.Controler; +import org.matsim.core.network.NetworkUtils; import org.matsim.core.replanning.strategies.DefaultPlanStrategiesModule; import org.matsim.core.router.costcalculators.TravelDisutilityFactory; import org.matsim.core.router.util.TravelTime; @@ -33,6 +38,9 @@ import org.matsim.run.prepare.PreparePopulation; import org.matsim.simwrapper.SimWrapperConfigGroup; import org.matsim.simwrapper.SimWrapperModule; +import org.matsim.vehicles.EngineInformation; +import org.matsim.vehicles.VehicleType; +import org.matsim.vehicles.VehicleUtils; import picocli.CommandLine; import playground.vsp.pt.fare.DistanceBasedPtFareParams; import playground.vsp.pt.fare.PtFareConfigGroup; @@ -60,10 +68,21 @@ public class LausitzScenario extends MATSimApplication { public static final String VERSION = "1.1"; private static final String FREIGHT = "freight"; + private static final String AVERAGE = "average"; + +// To decrypt hbefa input files set MATSIM_DECRYPTION_PASSWORD as environment variable. ask VSP for access. + private static final String HBEFA_2020_PATH = "https://svn.vsp.tu-berlin.de/repos/public-svn/3507bb3997e5657ab9da76dbedbb13c9b5991d3e/0e73947443d68f95202b71a156b337f7f71604ae/"; + private static final String HBEFA_FILE_COLD_DETAILED = HBEFA_2020_PATH + "82t7b02rc0rji2kmsahfwp933u2rfjlkhfpi2u9r20.enc"; + private static final String HBEFA_FILE_WARM_DETAILED = HBEFA_2020_PATH + "944637571c833ddcf1d0dfcccb59838509f397e6.enc"; + private static final String HBEFA_FILE_COLD_AVERAGE = HBEFA_2020_PATH + "r9230ru2n209r30u2fn0c9rn20n2rujkhkjhoewt84202.enc" ; + private static final String HBEFA_FILE_WARM_AVERAGE = HBEFA_2020_PATH + "7eff8f308633df1b8ac4d06d05180dd0c5fdf577.enc"; @CommandLine.Mixin private final SampleOptions sample = new SampleOptions( 100, 25, 10, 1); + @CommandLine.Option(names = "--emissions", defaultValue = "PERFORM_EMISSIONS_ANALYSIS", description = "Define if emission analysis should be performed or not.") + private EmissionAnalysisHandling emissions; + public LausitzScenario(@Nullable Config config) { super(config); @@ -124,7 +143,7 @@ protected Config prepareConfig(Config config) { config.routing().setAccessEgressType(RoutingConfigGroup.AccessEgressType.accessEgressModeToLink); prepareCommercialTrafficConfig(config); - // TODO: Config options + // set pt fare calc model to fareZoneBased = fare of vvo tarifzone 20 is paid for trips within fare zone // every other trip: Deutschlandtarif // for more info see FareZoneBasedPtFareHandler class in vsp contrib @@ -138,6 +157,19 @@ protected Config prepareConfig(Config config) { throw new RuntimeException(e); } + + if (emissions == EmissionAnalysisHandling.PERFORM_EMISSIONS_ANALYSIS) { +// set hbefa input files for emission analysis + EmissionsConfigGroup eConfig = ConfigUtils.addOrGetModule(config, EmissionsConfigGroup.class); + eConfig.setDetailedColdEmissionFactorsFile(HBEFA_FILE_COLD_DETAILED); + eConfig.setDetailedWarmEmissionFactorsFile(HBEFA_FILE_WARM_DETAILED); + eConfig.setAverageColdEmissionFactorsFile(HBEFA_FILE_COLD_AVERAGE); + eConfig.setAverageWarmEmissionFactorsFile(HBEFA_FILE_WARM_AVERAGE); + eConfig.setHbefaTableConsistencyCheckingLevel(EmissionsConfigGroup.HbefaTableConsistencyCheckingLevel.consistent); + eConfig.setDetailedVsAverageLookupBehavior(EmissionsConfigGroup.DetailedVsAverageLookupBehavior.tryDetailedThenTechnologyAverageThenAverageTable); + } + + // TODO: recreate counts format with car and trucks return config; @@ -158,6 +190,22 @@ protected void prepareScenario(Scenario scenario) { link.setAllowedModes(newModes); } } + + if (emissions == EmissionAnalysisHandling.PERFORM_EMISSIONS_ANALYSIS) { +// do not use VspHbefaRoadTypeMapping() as it results in almost every road to mapped to "highway"! + HbefaRoadTypeMapping roadTypeMapping = OsmHbefaMapping.build(); +// the type attribute in our network has the prefix "highway" for all links but pt links. +// we need to delete that because OsmHbefaMapping does not handle that. + for (Link link : scenario.getNetwork().getLinks().values()) { + //pt links can be disregarded + if (!link.getAllowedModes().contains("pt")) { + NetworkUtils.setType(link, NetworkUtils.getType(link).replaceFirst("highway.", "")); + } + } + roadTypeMapping.addHbefaMappings(scenario.getNetwork()); + + prepareVehicleTypesForEmissionAnalysis(scenario); + } } @Override @@ -225,4 +273,54 @@ public static void prepareCommercialTrafficConfig(Config config) { ); } } + + private static void prepareVehicleTypesForEmissionAnalysis(Scenario scenario) { + for (VehicleType type : scenario.getVehicles().getVehicleTypes().values()) { + EngineInformation engineInformation = type.getEngineInformation(); + +// only set engine information if none are present + if (engineInformation.getAttributes().isEmpty()) { + switch (type.getId().toString()) { + case TransportMode.car -> { + VehicleUtils.setHbefaVehicleCategory(engineInformation, HbefaVehicleCategory.PASSENGER_CAR.toString()); + VehicleUtils.setHbefaTechnology(engineInformation, AVERAGE); + VehicleUtils.setHbefaSizeClass(engineInformation, AVERAGE); + VehicleUtils.setHbefaEmissionsConcept(engineInformation, AVERAGE); + } + case TransportMode.ride -> { +// ignore ride, the mode routed on network, but then teleported + VehicleUtils.setHbefaVehicleCategory(engineInformation, HbefaVehicleCategory.NON_HBEFA_VEHICLE.toString()); + VehicleUtils.setHbefaTechnology(engineInformation, AVERAGE); + VehicleUtils.setHbefaSizeClass(engineInformation, AVERAGE); + VehicleUtils.setHbefaEmissionsConcept(engineInformation, AVERAGE); + } + case FREIGHT -> { + VehicleUtils.setHbefaVehicleCategory(engineInformation, HbefaVehicleCategory.HEAVY_GOODS_VEHICLE.toString()); + VehicleUtils.setHbefaTechnology(engineInformation, AVERAGE); + VehicleUtils.setHbefaSizeClass(engineInformation, AVERAGE); + VehicleUtils.setHbefaEmissionsConcept(engineInformation, AVERAGE); + } + case TransportMode.bike -> { +// ignore bikes + VehicleUtils.setHbefaVehicleCategory(engineInformation, HbefaVehicleCategory.NON_HBEFA_VEHICLE.toString()); + VehicleUtils.setHbefaTechnology(engineInformation, AVERAGE); + VehicleUtils.setHbefaSizeClass(engineInformation, AVERAGE); + VehicleUtils.setHbefaEmissionsConcept(engineInformation, AVERAGE); + } + default -> throw new IllegalArgumentException("does not know how to handle vehicleType " + type.getId().toString()); + } + } + } + +// ignore all pt veh types + scenario.getTransitVehicles() + .getVehicleTypes() + .values().forEach(type -> VehicleUtils.setHbefaVehicleCategory(type.getEngineInformation(), HbefaVehicleCategory.NON_HBEFA_VEHICLE.toString())); + } + + /** + * Defines if all necessary configs for emissions analysis should be made + * and hence if emissions analysis is performed or not (will fail without configs). + */ + enum EmissionAnalysisHandling {PERFORM_EMISSIONS_ANALYSIS, DO_NOT_PERFORM_EMISSIONS_ANALYSIS} } diff --git a/src/test/java/org/matsim/run/EmissionAnalysisOutputTest.java b/src/test/java/org/matsim/run/EmissionAnalysisOutputTest.java new file mode 100644 index 0000000..cfeeab0 --- /dev/null +++ b/src/test/java/org/matsim/run/EmissionAnalysisOutputTest.java @@ -0,0 +1,115 @@ +package org.matsim.run; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.api.io.TempDir; +import org.matsim.api.core.v01.Coord; +import org.matsim.api.core.v01.Id; +import org.matsim.api.core.v01.TransportMode; +import org.matsim.api.core.v01.population.*; +import org.matsim.application.MATSimApplication; +import org.matsim.core.config.Config; +import org.matsim.core.config.ConfigUtils; +import org.matsim.core.population.PersonUtils; +import org.matsim.core.population.PopulationUtils; +import org.matsim.core.utils.io.IOUtils; +import org.matsim.testcases.MatsimTestUtils; + +import java.io.BufferedReader; +import java.io.IOException; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; + +import static org.matsim.application.ApplicationUtils.globFile; + +class EmissionAnalysisOutputTest { + + @RegisterExtension + public MatsimTestUtils utils = new MatsimTestUtils(); + + @TempDir + public Path p; + + private final static Id ptPersonId = Id.createPersonId("Hoyerswerda-Cottbus_CAR"); + + @Test + void runEmissionAnalysisOutputTest() throws IOException { + Config config = ConfigUtils.loadConfig(String.format("input/v%s/lausitz-v%s-10pct.config.xml", LausitzScenario.VERSION, LausitzScenario.VERSION)); + + Path inputPath = p.resolve("emissions-test-population.xml.gz"); + + Population population = PopulationUtils.createPopulation(config); + PopulationFactory fac = population.getFactory(); + Person person = fac.createPerson(ptPersonId); + Plan plan = PopulationUtils.createPlan(person); + +// home in hoyerswerda, nearest link 28922425#0 + Activity home = fac.createActivityFromCoord("home_2400", new Coord(863538.13,5711028.24)); + home.setEndTime(8 * 3600); + Activity home2 = fac.createActivityFromCoord("home_2400", new Coord(863538.13,5711028.24)); + home2.setEndTime(19 * 3600); +// work in hoyerswerda, nearest link(s): 686055693#1, -686055693#1 + Activity work = fac.createActivityFromCoord("work_2400", new Coord(863866.47,5710961.86)); + work.setEndTime(17 * 3600 + 25 * 60); + + Leg leg = fac.createLeg(TransportMode.car); + + plan.addActivity(home); + plan.addLeg(leg); + plan.addActivity(work); + plan.addLeg(leg); + plan.addActivity(home2); + + person.addPlan(plan); + PersonUtils.setIncome(person, 1000.); + person.getAttributes().putAttribute("subpopulation", "person"); + population.addPerson(person); + + new PopulationWriter(population).write(inputPath.toString()); + + assert MATSimApplication.execute(LausitzScenario.class, config, + "--1pct", + "--iterations", "0", + "--output", utils.getOutputDirectory(), + "--config:plans.inputPlansFile", inputPath.toString(), + "--config:controller.overwriteFiles=deleteDirectoryIfExists") == 0 : "Must return non error code"; + + + Path csvPath = globFile(Path.of(utils.getOutputDirectory() + "/analysis/emissions"), "*emissions_per_link.csv*"); + + Map nonZeroLinks = new HashMap<>(); + + try { + BufferedReader reader = IOUtils.getBufferedReader(csvPath.toUri().toURL()); + String line; + +// skip header + reader.readLine(); + + while ((line = reader.readLine()) != null) { + String[] parts = line.split(","); + +// if first value (CO) is zero, all others are too + if (Double.parseDouble(parts[1]) == 0.) { + continue; + } + + Double[] values = new Double[23]; + + for (int i = 1; i < parts.length; i++) { + values[i - 1] = Double.parseDouble(parts[i]); + } + + nonZeroLinks.put(parts[0], values); + } + } finally { + + } + + Assertions.assertFalse(nonZeroLinks.isEmpty()); + Assertions.assertTrue(nonZeroLinks.containsKey("28922425#0")); + Assertions.assertTrue(nonZeroLinks.containsKey("-686055693#1")); + } +} From d5834ff8461ce65e5d22d532c8014d196c55a499 Mon Sep 17 00:00:00 2001 From: sime94 Date: Thu, 22 Aug 2024 12:25:39 +0200 Subject: [PATCH 4/4] delete todo after manual check of stations --- src/main/java/org/matsim/run/LausitzScenario.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/java/org/matsim/run/LausitzScenario.java b/src/main/java/org/matsim/run/LausitzScenario.java index 3592878..9cbafcc 100644 --- a/src/main/java/org/matsim/run/LausitzScenario.java +++ b/src/main/java/org/matsim/run/LausitzScenario.java @@ -168,10 +168,6 @@ protected Config prepareConfig(Config config) { eConfig.setHbefaTableConsistencyCheckingLevel(EmissionsConfigGroup.HbefaTableConsistencyCheckingLevel.consistent); eConfig.setDetailedVsAverageLookupBehavior(EmissionsConfigGroup.DetailedVsAverageLookupBehavior.tryDetailedThenTechnologyAverageThenAverageTable); } - - - // TODO: recreate counts format with car and trucks - return config; }