Commuter analysis

.github/workflows/build.yaml

@@ -14,7 +14,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-java@v3
         with:
-          java-version: 17
+          java-version: 21
           architecture: x64
           distribution: adopt
           cache: maven
@@ -32,7 +32,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-java@v3
         with:
-          java-version: 17
+          java-version: 21
           architecture: x64
           distribution: adopt
           cache: maven
@@ -51,7 +51,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        java: [17]
+        java: [21]
 
     steps:
       - uses: actions/checkout@v3
@@ -80,7 +80,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-java@v3
         with:
-          java-version: 17
+          java-version: 21
           architecture: x64
           distribution: adopt
           cache: maven

.mvn/wrapper/maven-wrapper.properties

@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.5/apache-maven-3.9.5-bin.zip
+wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar

Makefile

@@ -1,13 +1,13 @@
 
 N := lausitz
-V := v1.0
+V := v1.1
 CRS := EPSG:25832
 JAR := matsim-$(N)-*.jar
 
 osmosis := osmosis/bin/osmosis
 germany := ../shared-svn/projects/matsim-germany
 shared := ../shared-svn/projects/DiTriMo
-lausitz := ../public-svn/matsim/scenarios/countries/de/lausitz/lausitz-$V
+lausitz := ../public-svn/matsim/scenarios/countries/de/lausitz/input/$V
 
 MEMORY ?= 20G
 SUMO_HOME ?= $(abspath ../../sumo-1.18.0/)
@@ -18,33 +18,27 @@ sc := java -Xmx$(MEMORY) -XX:+UseParallelGC -jar $(JAR)
 
 .PHONY: prepare
 
-# Bast count data
-input/2019_A_S.zip:
-	curl https://www.bast.de/videos/2019_A_S.zip -o $@
-
-input/2019_B_S.zip:
-	curl https://www.bast.de/videos/2019_B_S.zip -o $@
-
-input/Jawe2019.csv:
-	curl "https://www.bast.de/DE/Verkehrstechnik/Fachthemen/v2-verkehrszaehlung/Daten/2019_1/Jawe2019.csv?view=renderTcDataExportCSV&cms_strTyp=A" -o $@
-
 input/network.osm: $(NETWORK)
 
+#	retrieve detailed network (see param highway) from OSM
 	$(osmosis) --rb file=$<\
 	 --tf accept-ways bicycle=yes highway=motorway,motorway_link,trunk,trunk_link,primary,primary_link,secondary_link,secondary,tertiary,motorway_junction,residential,unclassified,living_street\
 	 --bounding-polygon file="$(shared)/data/cottbus.poly"\
 	 --used-node --wb input/network-detailed.osm.pbf
 
 	# This includes residential as well, since multiple cities are covered by the study area
+	#	retrieve coarse network (see param highway) from OSM
 	$(osmosis) --rb file=$<\
 	 --tf accept-ways highway=motorway,motorway_link,trunk,trunk_link,primary,primary_link,secondary_link,secondary,tertiary,motorway_junction,residential\
 	 --bounding-polygon file="$(shared)/data/lausitz.poly"\
 	 --used-node --wb input/network-coarse.osm.pbf
 
+	#	retrieve germany wide network (see param highway) from OSM
 	$(osmosis) --rb file=$<\
 	 --tf accept-ways highway=motorway,motorway_link,motorway_junction,trunk,trunk_link,primary,primary_link\
 	 --used-node --wb input/network-germany.osm.pbf
 
+#	put the 3 above networks together and remove railway
 	$(osmosis) --rb file=input/network-germany.osm.pbf --rb file=input/network-coarse.osm.pbf --rb file=input/network-detailed.osm.pbf\
   	 --merge --merge\
   	 --tag-transform file=input/remove-railway.xml\
@@ -57,7 +51,9 @@ input/network.osm: $(NETWORK)
 
 input/sumo.net.xml: input/network.osm
 
+#	create sumo network from osm network
 	$(SUMO_HOME)/bin/netconvert --geometry.remove --ramps.guess --ramps.no-split\
+#	roadTypes are taken either from the general file "osmNetconvert.typ.xml" or from the german one "osmNetconvertUrbanDe.ty.xml"
 	 --type-files $(SUMO_HOME)/data/typemap/osmNetconvert.typ.xml,$(SUMO_HOME)/data/typemap/osmNetconvertUrbanDe.typ.xml\
 	 --tls.guess-signals true --tls.discard-simple --tls.join --tls.default-type actuated\
 	 --junctions.join --junctions.corner-detail 5\
@@ -69,12 +65,21 @@ input/sumo.net.xml: input/network.osm
 	 --osm-files $< -o=$@
 
 
+# transform sumo network to matsim network and clean it afterwards
+# free-speed-factor 0.75 (standard is 0.9): see VSP WP 24-08. lausitz is mix between rural and city (~0.7 - 0.8)
 input/$V/$N-$V-network.xml.gz: input/sumo.net.xml
 	$(sc) prepare network-from-sumo $< --output $@ --free-speed-factor 0.75
 	$(sc) prepare clean-network $@ --output $@ --modes car --modes bike
 
+# add freight modes as allowed modes
+# add hbefa attributes as link attributes
+input/$V/$N-$V-network-freight-hbefa.xml.gz: input/$V/$N-$V-network.xml.gz
+	$(sc) prepare network\
+	 --network $<\
+	 --output $@
 
-input/$V/$N-$V-network-with-pt.xml.gz: input/$V/$N-$V-network.xml.gz
+#add pt to network from german wide gtfs, but only for area of shp file
+input/$V/$N-$V-network-with-pt.xml.gz: input/$V/$N-$V-network-freight-hbefa.xml.gz
 	$(sc) prepare transit-from-gtfs --network $<\
 	 --output=input/$V\
 	 --name $N-$V --date "2023-01-11" --target-crs $(CRS) \
@@ -86,73 +91,126 @@ input/$V/$N-$V-network-with-pt.xml.gz: input/$V/$N-$V-network.xml.gz
 	 --shp $(shared)/data/network-area/network-area.shp\
 	 --shp $(shared)/data/germany-area/germany-area.shp\
 
+# extract lausitz long haul freight traffic trips from german wide file
 input/plans-longHaulFreight.xml.gz: input/$V/$N-$V-network.xml.gz
-	$(sc) prepare extract-freight-trips ../public-svn/matsim/scenarios/countries/de/german-wide-freight/v2/german_freight.25pct.plans.xml.gz\
+	$(sc) prepare extract-freight-trips ../public-svn/matsim/scenarios/countries/de/german-wide-freight/v2/german_freight.100pct.plans.xml.gz\
 	 --network ../public-svn/matsim/scenarios/countries/de/german-wide-freight/v2/germany-europe-network.xml.gz\
 	 --input-crs $(CRS)\
 	 --target-crs $(CRS)\
 	 --shp input/shp/lausitz.shp --shp-crs $(CRS)\
 	 --cut-on-boundary\
+	 --LegMode "longDistanceFreight"\
 	 --output $@
-
+# create facilities for commercial traffic
+input/commercialFacilities.xml.gz:
+	$(sc) prepare create-data-distribution-of-structure-data\
+	 --outputFacilityFile $@\
+	 --outputDataDistributionFile $(shared)/data/commercial_traffic/input/commercialTraffic/dataDistributionPerZone.csv\
+	 --landuseConfiguration useOSMBuildingsAndLanduse\
+ 	 --regionsShapeFileName $(shared)/data/commercial_traffic/input/commercialTraffic/lausitz_regions_25832.shp\
+	 --regionsShapeRegionColumn "GEN"\
+	 --zoneShapeFileName $(shared)/data/commercial_traffic/input/commercialTraffic/lausitz_zones_25832.shp\
+	 --zoneShapeFileNameColumn "GEN"\
+	 --buildingsShapeFileName $(shared)/data/commercial_traffic/input/commercialTraffic/lausitz_buildings_25832.shp\
+	 --shapeFileBuildingTypeColumn "building"\
+	 --landuseShapeFileName $(shared)/data/commercial_traffic/input/commercialTraffic/lausitz_landuse_25832.shp\
+	 --shapeFileLanduseTypeColumn "landuse"\
+	 --shapeCRS "EPSG:25832"\
+	 --pathToInvestigationAreaData $(shared)/data/commercial_traffic/input/commercialTraffic/commercialTrafficAreaData.csv
+# generate small scale commercial traffic
+input/lausitz-small-scale-commercialTraffic-$V-100pct.plans.xml.gz: input/$V/$N-$V-network.xml.gz input/commercialFacilities.xml.gz
+	$(sc) prepare generate-small-scale-commercial-traffic\
+	  input/$V/lausitz-$V-100pct.config.xml\
+	 --pathToDataDistributionToZones $(shared)/data/commercial_traffic/input/commercialTraffic/dataDistributionPerZone.csv\
+	 --pathToCommercialFacilities $(word 2,$^)\
+	 --sample 1.0\
+	 --jspritIterations 10\
+	 --creationOption createNewCarrierFile\
+	 --network $<\
+	 --smallScaleCommercialTrafficType completeSmallScaleCommercialTraffic\
+	 --zoneShapeFileName $(shared)/data/commercial_traffic/input/commercialTraffic/lausitz_zones_25832.shp\
+	 --zoneShapeFileNameColumn "GEN"\
+	 --shapeCRS "EPSG:25832"\
+	 --numberOfPlanVariantsPerAgent 5\
+	 --nameOutputPopulation $@\
+	 --pathOutput output/commercialPersonTraffic
+
+	mv output/commercialPersonTraffic/$@ $@
+
+
+# trajectory-to-plans formerly was a collection of methods to prepare a given population
+# now, most of the functions of this class do have their own class (downsample, splitduration types...)
+# it basically only transforms the old attribute format to the new one
 input/$V/prepare-100pct.plans.xml.gz:
 	$(sc) prepare trajectory-to-plans\
 	 --name prepare --sample-size 1 --output input/$V\
 	 --max-typical-duration 0\
-	 --population $(shared)/matsim-input-files/senozon/20230111_teilmodell_lausitz/population.xml.gz\
-	 --attributes  $(shared)/matsim-input-files/senozon/20230111_teilmodell_lausitz/additionalPersonAttributes.xml.gz
+	 --population $(shared)/data/matsim-input-files/senozon/20230111_teilmodell_lausitz/population.xml.gz\
+	 --attributes  $(shared)/data/matsim-input-files/senozon/20230111_teilmodell_lausitz/additionalPersonAttributes.xml.gz
 
+	# resolve senozon aggregated grid coords (activities): distribute them based on landuse.shp
 	$(sc) prepare resolve-grid-coords\
 	 input/$V/prepare-100pct.plans.xml.gz\
 	 --input-crs $(CRS)\
 	 --grid-resolution 300\
 	 --landuse $(germany)/landuse/landuse.shp\
 	 --output $@
 
-input/$V/$N-$V-100pct.plans-initial.xml.gz: input/plans-longHaulFreight.xml.gz input/$V/prepare-100pct.plans.xml.gz
+input/$V/$N-$V-100pct.plans-initial.xml.gz: input/plans-longHaulFreight.xml.gz input/$V/prepare-100pct.plans.xml.gz input/lausitz-small-scale-commercialTraffic-$V-100pct.plans.xml.gz
 
+#	generate some short distance trips, which in senozon data generally are missing
+# trip range 700m because:
+# when adding 1km trips (default value), too many trips of bin 1km-2km were also added.
+#the range value is beeline, so the trip distance (routed) often is higher than 1km
 	$(sc) prepare generate-short-distance-trips\
  	 --population input/$V/prepare-100pct.plans.xml.gz\
  	 --input-crs $(CRS)\
 	 --shp input/shp/lausitz.shp --shp-crs $(CRS)\
 	 --range 700\
  	 --num-trips 324430
 
+#	adapt coords of activities in the wider network such that they are closer to a link
+# 	such that agents do not have to walk as far as before
 	$(sc) prepare adjust-activity-to-link-distances input/$V/prepare-100pct.plans-with-trips.xml.gz\
 	 --shp input/shp/lausitz.shp --shp-crs $(CRS)\
      --scale 1.15\
      --input-crs $(CRS)\
      --network input/$V/$N-$V-network.xml.gz\
      --output input/$V/prepare-100pct.plans-adj.xml.gz
 
+#	change modes in subtours with chain based AND non-chain based by choosing mode for subtour randomly
 	$(sc) prepare fix-subtour-modes --coord-dist 100 --input input/$V/prepare-100pct.plans-adj.xml.gz --output $@
 
+#	set car availability for agents below 18 to false, standardize some person attrs, set home coords, set income
 	$(sc) prepare population $@ --output $@
 
+#	split activity types to type_duration for the scoring to take into account the typical duration
 	$(sc) prepare split-activity-types-duration\
 		--input $@\
-		--exclude commercial_start,commercial_end,freight_start,freight_end\
+		--exclude commercial_start,commercial_end,freight_start,freight_end,service\
 		--output $@
 
-	$(sc) prepare merge-populations $@ $< --output $@
+#	merge person and freight pops
+	$(sc) prepare merge-populations $@ $< $(word 3,$^) --output $@
 
 	$(sc) prepare downsample-population $@\
     	 --sample-size 1\
     	 --samples 0.25 0.1 0.01\
 
-input/$V/$N-$V-counts-car-bast.xml.gz: input/2019_A_S.zip input/2019_B_S.zip input/Jawe2019.csv input/$V/$N-$V-network-with-pt.xml.gz
+# create matsim counts file
+input/$V/$N-$V-counts-bast.xml.gz: input/$V/$N-$V-network-with-pt.xml.gz
 
 	$(sc) prepare counts-from-bast\
 		--network input/$V/$N-$V-network-with-pt.xml.gz\
-		--motorway-data input/2019_A_S.zip\
-		--primary-data input/2019_B_S.zip\
-		--station-data input/Jawe2019.csv\
+		--motorway-data $(germany)/bast-counts/2019/2019_A_S.zip\
+		--primary-data $(germany)/bast-counts/2019/2019_B_S.zip\
+		--station-data $(germany)/bast-counts/2019/Jawe2019.csv\
 		--year 2019\
 		--shp input/shp/lausitz.shp --shp-crs $(CRS)\
-		--car-output $@\
-		--freight-output $(subst car,freight,$@)
+		--output $@
 
 check: input/$V/$N-$V-100pct.plans-initial.xml.gz
+	#commuter analysis, still TODO
 	$(sc) analysis commuter\
 	 --population $<\
  	 --input-crs $(CRS)\

README.md

@@ -123,4 +123,4 @@ For more information about MATSim, see here: https://www.matsim.org/
 
 ### Internal Documentation
 Internal documentation can be found here:
-https://docs.google.com/document/d/1igBcrRTFklb7THSU1bNNIyG-pwiYgXBg6WeI-Pf7zj8/edit
+https://docs.google.com/document/d/1igBcrRTFklb7THSU1bNNIyG-pwiYgXBg6WeI-Pf7zj8/edit?usp=sharing

checkstyle.xml

@@ -10,7 +10,7 @@
     </module>
 
     <module name="NewlineAtEndOfFile">
-        <property name="lineSeparator" value="lf"/>
+        <property name="lineSeparator" value="lf_cr_crlf"/>
     </module>
 
     <module name="TreeWalker">

input/v1.0/lausitz-v1.0-25pct.config.xml

@@ -4,6 +4,7 @@
 	<module name="timeAllocationMutator">
 		<param name="mutationRange" value="1800.0"/>
 		<param name="mutateAroundInitialEndTimeOnly" value="true"/>
+<!--		what is this?; please convert boolean to enum!-->
 	</module>
 
 	<module name="controller">
@@ -73,6 +74,9 @@
 		<param name="useLanes" value="false"/>
 
 		<param name="vehiclesSource" value="modeVehicleTypesFromVehiclesData"/>
+		<!-- this setting is consistent with drt / dvrp necessities
+		if using drt + set to false it can happen that drt travel times estimates are not as precise as needed.
+		see comments in DvrpConfigGroup.checkConsistency-->
 		<param name="insertingWaitingVehiclesBeforeDrivingVehicles" value="true"/>
 	</module>
 
@@ -90,9 +94,12 @@
 		<param name="activateAnnealingModule" value="true" />
 		<parameterset type="AnnealingVariable" >
 			<param name="annealParameter" value="globalInnovationRate" />
+<!--			globalInnovationRate: fraction of innovated agents is annealed-->
 			<param name="annealType" value="sigmoid" />
+<!--			sigmoid: see comment above shapeFactor-->
 			<param name="subpopulation" value="person" />
 			<!-- this parameter enters the exponential and sigmoid formulas. May be an iteration or a share, i.e. 0.5 for halfLife at 50% of iterations. Exponential: startValue / exp(it/halfLife) -->
+<!--			the exponential entry can be ignored-->
 			<param name="halfLife" value="0.5" />
 			<!-- sigmoid: 1/(1+e^(shapeFactor*(it - halfLife))); geometric: startValue * shapeFactor^it; msa: startValue / it^shapeFactor -->
 			<param name="shapeFactor" value="0.01" />
@@ -101,6 +108,7 @@
 		</parameterset>
 	</module>
 	<module name="replanning">
+		<!--		this also switches off annealing-->
 		<param name="fractionOfIterationsToDisableInnovation" value="0.9"/>
 		<parameterset type="strategysettings">
 			<param name="strategyName" value="ChangeExpBeta"/>
@@ -138,9 +146,14 @@
 	<module name="subtourModeChoice">
 		<param name="chainBasedModes" value="car,bike"/>
 		<param name="modes" value="car,ride,pt,bike,walk"/>
+		<!--		car availability is set to false only for agents with age < 18 years! -->
 		<param name="considerCarAvailability" value="true"/>
+		<!--		Same as "fromSpecifiedModesToSpecifiedModes", but also allow agents with open subtours to switch modes,
+		       	which they are not allowed to do with fromSpecifiedModesToSpecifiedModes -->
 		<param name="behavior" value="betweenAllAndFewerConstraints"/>
 		<param name="probaForRandomSingleTripMode" value="0.5" />
+		<!--	If greater than 0, activities that are closer than coordDistance, to each other, will be considered part of the same subtour." +
+			"i.e. if two activities are close to each other, the agent is allowed to use the same 'chain-based' vehicle for both subtours.-->
 		<param name="coordDistance" value="100"/>
 	</module>
 
@@ -149,6 +162,9 @@
 	</module>
 
 	<module name="travelTimeCalculator">
+		<!-- This parameter decides which modes are analyzed, when separateModes = false && filterModes = true. The 2 latter parameters are only here for backwards compability.
+		standard is separateModes = true (see below), so analyzedModes is ignored.
+		In case of separateModes = false && filterModes = true: travel time measurement for all vehicle types together (seperatemodes = false), BUT filtered (filterModes = true) for the modes in analyzedModes-->
 		<param name="analyzedModes" value="car,freight"/>
 
 		<param name="separateModes" value="true"/>
@@ -171,6 +187,7 @@
 				<param name="marginalUtilityOfTraveling_util_hr" value="-0.5"/>
 				<param name="mode" value="car"/>
 				<param name="monetaryDistanceRate" value="-0.00025"/>
+<!--				the cost values come from: https://docs.google.com/spreadsheets/d/1HzWpmmws1dBJuo-1Z0TnJ6-w4vD7eXBTUUEu68Hl9Q4/edit#gid=180748432-->
 				<param name="dailyMonetaryConstant" value="-13.22"/>
 			</parameterset>
 			<parameterset type="modeParams">
@@ -201,6 +218,7 @@
 				<param name="marginalUtilityOfTraveling_util_hr" value="-10.0"/>
 				<param name="mode" value="freight"/>
 				<param name="monetaryDistanceRate" value="-0.002"/>
+<!--				-->
 			</parameterset>
 
 		</parameterset>