update calibration

Makefile

@@ -252,9 +252,8 @@ eval-opt: $p/berlin-initial-$V-25pct.experienced_plans.xml.gz
 	 --config $p/berlin-$V.config.xml
 
 
-# TODO: these needs to be renamed to plans-initial, because they are uncalibrated at this stage
 # These depend on the output of optimization runs
-$p/berlin-$V-25pct.plans.xml.gz: $p/berlin-$V-facilities.xml.gz $p/berlin-$V-network.xml.gz $p/berlin-longHaulFreight-$V-25pct.plans.xml.gz
+$p/berlin-$V-25pct.plans-initial.xml.gz: $p/berlin-$V-facilities.xml.gz $p/berlin-$V-network.xml.gz $p/berlin-longHaulFreight-$V-25pct.plans.xml.gz
 	$(sc) prepare filter-relevant-agents\
 	 --input $p/berlin-$V-25pct.plans_log_error.xml.gz --output $@\
 	 --shp input/$V/area/area.shp\
@@ -278,19 +277,22 @@ $p/berlin-$V-25pct.plans.xml.gz: $p/berlin-$V-facilities.xml.gz $p/berlin-$V-net
 		 --sample-size 0.25\
 		 --samples 0.1 0.01 0.001\
 
-clean-plans:
+$p/berlin-$V-25pct.plans.xml.gz:
 	$(sc) prepare clean-population\
-	 --plans mode-choice-final-25pct/runs/004/004.output_plans.xml.gz\
+	 --plans mode-choice-10pct/runs/004/004.output_plans.xml.gz\
 	 --remove-unselected-plans\
-	 --output mode-choice-final-25pct/runs/004/berlin-$V-25pct.plans.xml.gz
+	 --output mode-choice-final/runs/004/berlin-$V-10pct.plans.xml.gz
 
 	$(sc) prepare clean-population\
-	 --plans mode-choice-final/runs/004/004.output_plans.xml.gz\
+	 --plans mode-choice-final-25pct/runs/004/004.output_plans.xml.gz\
 	 --remove-unselected-plans\
-	 --output mode-choice-final/runs/004/berlin-$V-10pct.plans.xml.gz
+	 --output $@
 
 prepare-calibration: $p/berlin-cadyts-input-$V-25pct.plans.xml.gz $p/berlin-$V-network-with-pt.xml.gz $p/berlin-$V-counts-vmz.xml.gz
 	echo "Done"
 
-prepare: $p/berlin-$V-25pct.plans.xml.gz $p/berlin-$V-network-with-pt.xml.gz
+prepare-initial: $p/berlin-$V-25pct.plans-initial.xml.gz $p/berlin-$V-network-with-pt.xml.gz
 	echo "Done"
+
+prepare: $p/berlin-$V-25pct.plans.xml.gz
+	echo "Done"

input/v6.1/berlin-v6.1.config.xml

@@ -3,6 +3,7 @@
 <config>
 	<module name="TimeAllocationMutator">
 		<param name="mutationRange" value="1800.0"/>
+		<param name="mutateAroundInitialEndTimeOnly" value="true"/>
 	</module>
 	<module name="controler">
 		<param name="lastIteration" value="500"/>
@@ -22,7 +23,7 @@
 		<param name="inputNetworkFile" value="./berlin-v6.1-network-with-pt.xml.gz"/>
 	</module>
 	<module name="plans">
-		<param name="inputPlansFile" value="./berlin-v6.1-25pct.plans.xml.gz"/>
+		<param name="inputPlansFile" value="./berlin-v6.1-25pct.plans-initial.xml.gz"/>
 		<param name="removingUnnecessaryPlanAttributes" value="true"/>
 	</module>
 	<module name="vehicles">

src/main/java/org/matsim/dashboard/BerlinDashboardProvider.java

@@ -23,11 +23,11 @@ public List<Dashboard> getDashboards(Config config, SimWrapper simWrapper) {
 		TripDashboard trips = new TripDashboard("mode_share_ref.csv", "mode_share_per_dist_ref.csv", "mode_users_ref.csv");
 		trips.setAnalysisArgs("--match-id", "^berlin.+", "--shp-filter", "none");
 
-		// TODO: the freight mode is not separated correctly from car yet
+		// TODO: Freight counts should go into extra tab on counts dashboard
 		return List.of(
 			trips,
 			new TravelTimeComparisonDashboard(ApplicationUtils.resolve(config.getContext(), "berlin-v" + RunOpenBerlinScenario.VERSION + "-routes-ref.csv.gz")),
-			Dashboard.customize(new TrafficCountsDashboard(ApplicationUtils.resolve(config.getContext(), "berlin-v" + RunOpenBerlinScenario.VERSION + "-counts-hgv-vmz.xml.gz"), Set.of("freight")))
+			Dashboard.customize(new TrafficCountsDashboard(ApplicationUtils.resolve(config.getContext(), "berlin-v" + RunOpenBerlinScenario.VERSION + "-counts-vmz.xml.gz"), Set.of("truck")))
 				.title("Truck counts")
 				.context("freight")
 		);

src/main/python/calibrate.py

@@ -1,49 +1,51 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-import calibration
+from matsim.calibration import create_calibration, ASCCalibrator, utils
 
 # %%
 
-modes = ["walk", "car", "pt", "bike"]
+modes = ["walk", "car", "pt", "bike", "ride"]
 fixed_mode = "walk"
 initial = {
-    "bike": -2,
-    "pt": 0,
-    "car": 0,
-    "ride": -4
+    "bike": -2.14,
+    "pt": -0.39,
+    "car": -0.30,
+    "ride": -1.20
 }
 
 # Original modal split
 target = {
-    "walk": 0.3026,
-    "bike": 0.1883,
-    "pt": 0.2437,
-    "car": 0.2046,
-    "ride": 0.0608
+    "walk": 0.296769,
+    "bike": 0.177878,
+    "pt": 0.265073,
+    "car": 0.200673,
+    "ride": 0.059607
 }
 
-def f(persons):
+
+def filter_persons(persons):
     df = persons[persons.person.str.startswith("berlin")]
     print("Filtered %s persons" % len(df))
     return df
 
+
 def filter_modes(df):
     return df[df.main_mode.isin(modes)]
 
-study, obj = calibration.create_mode_share_study(
+
+study, obj = create_calibration(
     "calib",
+    ASCCalibrator(modes, initial, target, lr=utils.linear_scheduler(start=0.3, interval=15)),
     "matsim-berlin-6.x-SNAPSHOT.jar",
-    "../input/v6.0/berlin-v6.0-base-calib.config.xml",
-    modes, target,
-    initial_asc=initial,
-    args="--10pct --config:TimeAllocationMutator.mutationRange=600",
-    jvm_args="-Xmx55G -Xms55G -XX:+AlwaysPreTouch -XX:+UseParallelGC",
-    lr=calibration.linear_lr_scheduler(start=0.3, interval=10),
-    person_filter=f, map_trips=filter_modes,
-    chain_runs=calibration.default_chain_scheduler
+    "../input/v6.1/berlin-v6.1.config.xml",
+    args="--10pct",
+    jvm_args="-Xmx60G -Xms60G -XX:+AlwaysPreTouch -XX:+UseParallelGC",
+    transform_persons=filter_persons,
+    transform_trips=filter_modes,
+    chain_runs=utils.default_chain_scheduler, debug=False
 )
 
 # %%
 
-study.optimize(obj, 10)
+study.optimize(obj, 5)

src/main/python/extract_ref_data.py

@@ -1,27 +1,34 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+import os
+
 from matsim.scenariogen.data import TripMode, run_create_ref_data
 
 
 def person_filter(df):
     """ Default person filter for reference data. """
-    return df[df.reporting_day <= 5]
+    df = df[df.reporting_day <= 4]
+    df = df[df.location == "Berlin"]
+
+    return df
 
 
 def trip_filter(df):
     # Motorcycles are counted as cars
     df.loc[df.main_mode == TripMode.MOTORCYCLE, "main_mode"] = TripMode.CAR
 
-    return df
+    # Other mode are ignored in the total share
+    df = df[df.main_mode != TripMode.OTHER]
 
+    return df
 
 
 if __name__ == "__main__":
-    person, trips, share = run_create_ref_data.create("../../../../shared-svn/projects/NaMAV/data/SrV_2018",
+    d = os.path.expanduser("~/Development/matsim-scenarios/shared-svn/projects/matsim-berlin/data/SrV/")
+
+    person, trips, share = run_create_ref_data.create(d + "Berlin+Umland",
                                                       person_filter, trip_filter,
                                                       run_create_ref_data.InvalidHandling.REMOVE_TRIPS)
 
-    # TODO: double check
-
     print(share)

src/main/resources/mode_share_per_dist_ref.csv

@@ -1,31 +1,31 @@
 dist_group,main_mode,mean_dist,share
-0 - 1000,bike,642.7362254533504,0.14296559258490224
-0 - 1000,car,664.7137300405267,0.036195467606192945
-0 - 1000,pt,737.7097226889715,0.012072125831713259
-0 - 1000,ride,671.3913613714266,0.01422073315004362
-0 - 1000,walk,462.49633729925984,0.794546080827148
-1000 - 2000,bike,1459.0188605554372,0.3168640332584639
-1000 - 2000,car,1516.0373712895853,0.1533805232501156
-1000 - 2000,pt,1561.9485358897075,0.12539086638753746
-1000 - 2000,ride,1504.786904018702,0.0679566257681203
-1000 - 2000,walk,1357.1883390502437,0.3364079513357628
-2000 - 5000,bike,3236.6372239756583,0.27606188732819215
-2000 - 5000,car,3369.037825840531,0.25175450941633504
-2000 - 5000,pt,3472.0945689576815,0.31180268014754664
-2000 - 5000,ride,3345.124776877841,0.09498177141050178
-2000 - 5000,walk,2827.1062729794758,0.06539915169742452
-5000 - 10000,bike,6841.379492215497,0.16674049030249002
-5000 - 10000,car,7189.242991435152,0.30735790672296154
-5000 - 10000,pt,7207.889944153758,0.43089260502311605
-5000 - 10000,ride,6988.590589470452,0.08252117457567074
-5000 - 10000,walk,6833.960515847983,0.012487823375761794
-10000 - 20000,bike,12887.681896313305,0.06784990932503386
-10000 - 20000,car,14157.926434083272,0.3565961449145108
-10000 - 20000,pt,14018.197174068706,0.4990432172479679
-10000 - 20000,ride,13843.147524650036,0.06845571208553118
-10000 - 20000,walk,13525.723972986301,0.008055016426956165
-20000+,bike,35153.626492130796,0.015277455048963355
-20000+,car,33189.87041354405,0.4590512538650451
-20000+,pt,31436.68734269047,0.442722773287289
-20000+,ride,39682.9627424628,0.07234552964136147
-20000+,walk,75238.71714274038,0.010602988157341257
+0 - 1000,bike,641.923969696311,0.13955969280974076
+0 - 1000,car,667.6048862627572,0.03521419946265174
+0 - 1000,pt,736.884946458635,0.012431042155741298
+0 - 1000,ride,679.8383383516772,0.0132676962262736
+0 - 1000,walk,463.4736508146024,0.7995273693455925
+1000 - 2000,bike,1459.439889557046,0.30782141268240654
+1000 - 2000,car,1515.6654055247768,0.15165417213254306
+1000 - 2000,pt,1560.8610569998507,0.1315562766954925
+1000 - 2000,ride,1503.658665706513,0.06692865094292362
+1000 - 2000,walk,1360.5449036473765,0.3420394875466344
+2000 - 5000,bike,3233.1310858469933,0.26593557247872057
+2000 - 5000,car,3369.780420245613,0.24416722005700903
+2000 - 5000,pt,3471.43175799158,0.33266543742394744
+2000 - 5000,ride,3335.6923181538796,0.09339568053754725
+2000 - 5000,walk,2787.6143446620194,0.06383608950277564
+5000 - 10000,bike,6833.120011695687,0.1521334976557492
+5000 - 10000,car,7188.906623537019,0.2946173106872074
+5000 - 10000,pt,7225.581516386351,0.46575206267904273
+5000 - 10000,ride,6996.618119909154,0.0789244586744906
+5000 - 10000,walk,6686.771851964966,0.008572670303510228
+10000 - 20000,bike,12851.96000741612,0.05908246187835078
+10000 - 20000,car,14116.230268840298,0.34227361194279915
+10000 - 20000,pt,13977.58266859862,0.5286620898290071
+10000 - 20000,ride,13778.927405383118,0.06561641189606941
+10000 - 20000,walk,12158.890739372593,0.0043654244537734786
+20000+,bike,44640.07139716297,0.012375425810734026
+20000+,car,52570.26447893369,0.43007953377722014
+20000+,pt,80453.71634721293,0.4775389794365375
+20000+,ride,96167.87960571941,0.07604216515678314
+20000+,walk,182710.51794030122,0.003963895818725283

src/main/resources/mode_share_ref.csv

@@ -1,31 +1,31 @@
 dist_group,main_mode,mean_dist,share
-0 - 1000,bike,642.7362254533504,0.041773762171279594
-0 - 1000,car,664.7137300405267,0.01057611714903656
-0 - 1000,pt,737.7097226889715,0.0035274089679743436
-0 - 1000,ride,671.3913613714266,0.004155220244048345
-0 - 1000,walk,462.49633729925984,0.2321620077564082
-1000 - 2000,bike,1459.0188605554372,0.04976149343536908
-1000 - 2000,car,1516.0373712895853,0.02408744161442384
-1000 - 2000,pt,1561.9485358897075,0.019691842934755097
-1000 - 2000,ride,1504.786904018702,0.010672158503681558
-1000 - 2000,walk,1357.1883390502437,0.05283074222673194
-2000 - 5000,bike,3236.6372239756583,0.05896581500309014
-2000 - 5000,car,3369.037825840531,0.05377384749525085
-2000 - 5000,pt,3472.0945689576815,0.06659991834798387
-2000 - 5000,ride,3345.124776877841,0.02028776089253914
-2000 - 5000,walk,2827.1062729794758,0.013969020923793221
-5000 - 10000,bike,6841.379492215497,0.029113427426160545
-5000 - 10000,car,7189.242991435152,0.053665681892875745
-5000 - 10000,pt,7207.889944153758,0.07523523867569203
-5000 - 10000,ride,6988.590589470452,0.014408463251918585
-5000 - 10000,walk,6833.960515847983,0.002180414240724605
-10000 - 20000,bike,12887.681896313305,0.008036568653056839
-10000 - 20000,car,14157.926434083272,0.042237483123113095
-10000 - 20000,pt,14018.197174068706,0.059109807458149484
-10000 - 20000,ride,13843.147524650036,0.008108323730158346
-10000 - 20000,walk,13525.723972986301,0.0009540866474356413
-20000+,bike,35153.626492130796,0.0006739794008152106
-20000+,car,33189.87041354405,0.020251480893372212
-20000+,pt,31436.68734269047,0.019531134505786824
-20000+,ride,39682.9627424628,0.003191591568299355
-20000+,walk,75238.71714274038,0.00046776086607569733
+0 - 1000,bike,641.923969696311,0.03993953574665
+0 - 1000,car,667.6048862627572,0.010077686113465548
+0 - 1000,pt,736.884946458635,0.0035575461836548946
+0 - 1000,ride,679.8383383516772,0.0037969819009802397
+0 - 1000,walk,463.4736508146024,0.22881070677001764
+1000 - 2000,bike,1459.439889557046,0.04698342341346555
+1000 - 2000,car,1515.6654055247768,0.023147292190076742
+1000 - 2000,pt,1560.8610569998507,0.02007970854535886
+1000 - 2000,ride,1503.658665706513,0.010215459406613094
+1000 - 2000,walk,1360.5449036473765,0.05220619945665914
+2000 - 5000,bike,3233.1310858469933,0.05624516000252578
+2000 - 5000,car,3369.780420245613,0.05164117094781402
+2000 - 5000,pt,3471.43175799158,0.07035847284671681
+2000 - 5000,ride,3335.6923181538796,0.019753111426262702
+2000 - 5000,walk,2787.6143446620194,0.013501281662145714
+5000 - 10000,bike,6833.120011695687,0.02693069701653698
+5000 - 10000,car,7188.906623537019,0.05215320525856776
+5000 - 10000,pt,7225.581516386351,0.08244750747280563
+5000 - 10000,ride,6996.618119909154,0.01397122077983446
+5000 - 10000,walk,6686.771851964966,0.001517535520605139
+10000 - 20000,bike,12851.96000741612,0.007136782872703979
+10000 - 20000,car,14116.230268840298,0.04134445948649564
+10000 - 20000,pt,13977.58266859862,0.06385899348452925
+10000 - 20000,ride,13778.927405383118,0.007926042173942492
+10000 - 20000,walk,12158.890739372593,0.0005273153061550004
+20000+,bike,44640.07139716297,0.0006419443250597948
+20000+,car,52570.26447893369,0.022309302342807483
+20000+,pt,80453.71634721293,0.024771142628339887
+20000+,ride,96167.87960571941,0.003944497517436229
+20000+,walk,182710.51794030122,0.00020561720177352156

src/main/resources/mode_users_ref.csv

@@ -1,6 +1,6 @@
 main_mode,user
-pt,0.4030693354976342
-walk,0.44792354586802313
-ride,0.11136192739923934
-bike,0.2560857889063021
-car,0.2576979954344417
+pt,0.39817910774050397
+bike,0.2197728757291537
+car,0.2303877524396695
+ride,0.09882073740585559
+walk,0.3989282280381705

src/main/sh/runCalib.sh

@@ -3,9 +3,9 @@
 #$ -j y
 #$ -m a
 #$ -cwd
-#$ -pe mp 8
-#$ -l mem_free=6G
-#$ -N calib-scenario
+#$ -pe mp 16
+#$ -l mem_free=4G
+#$ -N calib-berlin
 
 date
 hostname

src/main/sh/setup.sh

@@ -1,13 +1,8 @@
 #!/usr/bin/env bash
 
-python3 -m venv env
+python3.9 -m venv env
 source env/bin/activate
 
-module load gcc/9.2.0
-
 pip install --upgrade pip
-pip install optuna geopandas rtree pygeos
 
-# Download latest calibration script
-wget "https://github.com/matsim-vsp/matsim-python-tools/raw/master/matsim/calibration.py"
-wget "https://github.com/matsim-vsp/matsim-python-tools/raw/master/matsim/analysis.py"
+pip install "matsim-tools[calibration]==0.0.16"