update workflow and makefile

matsim-scenarios · Jul 15, 2023 · aac18d8 · aac18d8
1 parent e59dfa7
commit aac18d8
Show file tree

Hide file tree

Showing 9 changed files with 503 additions and 96 deletions.
diff --git a/Makefile b/Makefile
@@ -91,7 +91,7 @@ input/sumo.net.xml: input/network.osm
 	 --osm-files $< -o=$@
 
 
-$p/berlin-$V-network.xml.gz:
+$p/berlin-v6.0-network.xml.gz:
 	# Use 5.x network
 	$(sc) prepare reproject-network\
 	 --input $(berlin)/../berlin-v5.5-10pct/input/berlin-v5.5-network.xml.gz\
@@ -102,13 +102,12 @@ $p/berlin-$V-network.xml.gz:
 	 --target-crs $(CRS)
 
 
-$p/berlin-$V-network-with-pt.xml.gz: $p/berlin-$V-network.xml.gz
+$p/berlin-v6.0-network-with-pt.xml.gz: $p/berlin-v6.0-network.xml.gz
 	# Copy 5.x network stuff
 	cp $< $@
 	cp $(berlin)/../berlin-v5.5-10pct/input/berlin-v5.5-transit-vehicles.xml.gz $p/berlin-v6.0-transitVehicles.xml.gz
 
 
-# TODO: Will be the updated network
 $p/berlin-v6.1-network.xml.gz: input/sumo.net.xml
 	$(sc) prepare network-from-sumo $< --target-crs $(CRS) --output $@
 
@@ -123,28 +122,26 @@ $p/berlin-v6.1-network.xml.gz: input/sumo.net.xml
 
 $p/berlin-v6.1-network-with-pt.xml.gz: $p/berlin-v6.1-network.xml.gz
 	$(sc) prepare transit-from-gtfs --network $< --output=$p\
-	 --name berlin-v6.1 --date "2023-06-07" --target-crs $(CRS) \
+	 --name berlin-$V --date "2023-06-07" --target-crs $(CRS) \
 	 $(germany)/gtfs/complete-pt-2023-06-06.zip\
 	 --shp $p/pt-area/pt-area.shp
 
-$p/berlin-v6.1-car-counts-from-vmz.xml.gz: $p/berlin-v6.1-network.xml.gz
+$p/berlin-v6.1-counts-car-vmz.xml.gz: $p/berlin-v6.1-network.xml.gz
 	$(sc) prepare counts-from-vmz\
-	--excel ../shared-svn/projects/matsim-berlin/berlin-v5.5/original_data/vmz_counts_2018/Datenexport_2018_TU_Berlin.xlsx\
-	--network $<\
-	--network-geometries $p/berlin-v6.1-network-linkGeometries.csv\
-	--output $p\
-	--version berlin-$(V)\
-	--input-crs EPSG:31468\
-	--target-crs $(CRS)\
-	--ignored-counts input/v6.0/ignored_6_1.csv
-
-# TODO: naming scheme must be updated
-$p/berlin-$V-car-counts.xml.gz: $p/berlin-$V-network.xml.gz
-	$(sc) prepare create-counts\
+	 --excel ../shared-svn/projects/matsim-berlin/berlin-v5.5/original_data/vmz_counts_2018/Datenexport_2018_TU_Berlin.xlsx\
 	 --network $<\
-	 --shp $(berlin)/Verkehrsmengen_DTVw_2019.zip\
-	 --output $p/berlin-$V-
-	# TODO: output argument not ideal
+	 --network-geometries $p/berlin-v6.0-network-linkGeometries.csv\
+	 --output $p/\
+	 --version berlin-$(V)\
+	 --input-crs EPSG:31468\
+	 --target-crs $(CRS)\
+	 --ignored-counts input/ignored_counts.csv
+
+input/v6.0/berlin-v6.0-counts-car-vmz.xml.gz:
+	$(sc) prepare counts-from-vmz-old\
+	 --csv ../shared-svn/projects/matsim-berlin/berlin-v5.5/original_data/vmz_counts_2018/CountsId_to_linkId.csv\
+	 --excel ../shared-svn/projects/matsim-berlin/berlin-v5.5/original_data/vmz_counts_2018/Datenexport_2018_TU_Berlin.xlsx\
+ 	 --output $@
 
 $p/berlin-$V-facilities.xml.gz: $p/berlin-$V-network.xml.gz input/facilities.shp
 	$(sc) prepare facilities --network $< --shp $(word 2,$^)\
@@ -173,28 +170,22 @@ $p/berlin-static-$V-25pct.plans.xml.gz: $p/berlin-only-$V-25pct.plans.xml.gz $p/
 	$(sc) prepare lookup-regiostar --input $@ --output $@ --xls $(germany)/RegioStaR-Referenzdateien.xlsx
 
 
-$p/berlin-activities-$V-25pct.plans-1.xml.gz: $p/berlin-static-$V-25pct.plans.xml.gz
-	# Create five separate sets of activities
-	$(sc) prepare activity-sampling --seed 2 --input $< --output $(subst plans-1,plans-2,$@) --persons src/main/python/table-persons.csv --activities src/main/python/table-activities.csv
-	$(sc) prepare activity-sampling --seed 3 --input $< --output $(subst plans-1,plans-3,$@) --persons src/main/python/table-persons.csv --activities src/main/python/table-activities.csv
-	$(sc) prepare activity-sampling --seed 4 --input $< --output $(subst plans-1,plans-4,$@) --persons src/main/python/table-persons.csv --activities src/main/python/table-activities.csv
-	$(sc) prepare activity-sampling --seed 5 --input $< --output $(subst plans-1,plans-5,$@) --persons src/main/python/table-persons.csv --activities src/main/python/table-activities.csv
-
+$p/berlin-activities-$V-25pct.plans.xml.gz: $p/berlin-static-$V-25pct.plans.xml.gz
 	$(sc) prepare activity-sampling --seed 1 --input $< --output $@ --persons src/main/python/table-persons.csv --activities src/main/python/table-activities.csv
 
-$p/berlin-initial-$V-25pct.plans.xml.gz: $p/berlin-activities-$V-25pct.plans-1.xml.gz $p/berlin-$V-facilities.xml.gz $p/berlin-$V-network.xml.gz
+$p/berlin-initial-$V-25pct.plans.xml.gz: $p/berlin-activities-$V-25pct.plans.xml.gz $p/berlin-$V-facilities.xml.gz $p/berlin-$V-network.xml.gz
 	$(sc) prepare init-location-choice\
-	 --input "$(subst plans-1,plans-*,$<)"\
+	 --input $<\
 	 --output $@\
 	 --facilities $(word 2,$^)\
 	 --network $(word 3,$^)\
 	 --shp $(germany)/vg5000/vg5000_ebenen_0101/VG5000_GEM.shp\
 	 --commuter $(germany)/regionalstatistik/commuter.csv\
 
 	# For debugging and visualization
-	$(sc) prepare downsample-population $p/berlin-initial-$V-25pct.plans-1.xml.gz\
+	$(sc) prepare downsample-population $@\
 		 --sample-size 0.25\
-		 --samples 0.1\
+		 --samples 0.1 0.01\
 
 
 $p/berlin-longHaulFreight-$V-25pct.plans.xml.gz: $p/berlin-$V-network.xml.gz
@@ -244,12 +235,9 @@ $p/berlin-goodsTraffic-$V-25pct.plans.xml.gz:
 
 	mv output/goodsTraffic/$(notdir $@) $@
 
-# Depends on location choice runs and freight model
-$p/berlin-cadyts-input-$V-25pct.plans.xml.gz: $p/berlin-commercialPersonTraffic-$V-25pct.plans.xml.gz
-	$(sc) prepare merge-plans output/lc-*/*output_selected_plans.xml.gz\
-		--output $@
-
-	$(sc) prepare merge-populations $@ $< --output $@
+$p/berlin-cadyts-input-$V-25pct.plans.xml.gz: $p/berlin-initial-$V-25pct.plans.xml.gz $p/berlin-commercialPersonTraffic-$V-25pct.plans.xml.gz
+	$(sc) prepare merge-populations $^\
+	 --output $@
 
 # This file requires eval runs
 $p/berlin-initial-$V-25pct.experienced_plans.xml.gz:
@@ -275,7 +263,7 @@ eval-opt: $p/berlin-initial-$V-25pct.experienced_plans.xml.gz
  	 --csv $p/berlin-$V-25pct.plans_selection_$(ERROR_METRIC).csv\
  	 --output $p/berlin-$V-25pct.plans_$(ERROR_METRIC).xml.gz
 
-	$(sc) run --mode "eval" --output "output/eval-$(ERROR_METRIC)" --25pct --population "berlin-$V-25pct.plans_$(ERROR_METRIC).xml.gz"\
+	$(sc) run --mode "eval" --all-car --output "output/eval-$(ERROR_METRIC)" --25pct --population "berlin-$V-25pct.plans_$(ERROR_METRIC).xml.gz"\
 	 --config $p/berlin-$V-base-calib.config.xml
 
 
@@ -304,7 +292,7 @@ $p/berlin-$V-25pct.plans.xml.gz: $p/berlin-$V-facilities.xml.gz $p/berlin-$V-net
 		 --sample-size 0.25\
 		 --samples 0.1 0.01 0.001\
 
-prepare-calibration: $p/berlin-initial-$V-25pct.plans.xml.gz
+prepare-calibration: $p/berlin-cadyts-input-$V-25pct.plans.xml.gz $p/berlin-$V-network-with-pt.xml.gz $p/berlin-$V-counts-car-vmz.xml.gz
 	echo "Done"
 
 prepare: $p/berlin-$V-25pct.plans.xml.gz $p/berlin-$V-network-with-pt.xml.gz

diff --git a/input/v6.0/ignored_6_1.csv b/input/v6.0/ignored_6_1.csv
diff --git a/src/main/java/org/matsim/prepare/InitLocationChoice.java b/src/main/java/org/matsim/prepare/InitLocationChoice.java
@@ -33,10 +33,7 @@
 import org.opengis.feature.simple.SimpleFeature;
 import picocli.CommandLine;
 
-import java.nio.file.FileSystems;
-import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.PathMatcher;
 import java.util.*;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Collectors;
@@ -57,12 +54,15 @@ public class InitLocationChoice implements MATSimAppCommand, PersonAlgorithm {
 
 	private static final Logger log = LogManager.getLogger(InitLocationChoice.class);
 
-	@CommandLine.Option(names = "--input", description = "Path to input population, can be a pattern if * is used.", required = true)
+	@CommandLine.Option(names = "--input", description = "Path to input population.")
 	private Path input;
 
 	@CommandLine.Option(names = "--output", description = "Path to output population", required = true)
 	private Path output;
 
+	@CommandLine.Option(names = "--k", description = "Number of choices to generate", defaultValue = "5")
+	private int k;
+
 	@CommandLine.Option(names = "--commuter", description = "Path to commuter.csv", required = true)
 	private Path commuterPath;
 
@@ -75,6 +75,9 @@ public class InitLocationChoice implements MATSimAppCommand, PersonAlgorithm {
 	@CommandLine.Option(names = "--sample", description = "Sample size of the population", defaultValue = "0.25")
 	private double sample;
 
+	@CommandLine.Option(names = "--seed", description = "Seed used to sample locations", defaultValue = "1")
+	private long seed;
+
 	@CommandLine.Mixin
 	private ShpOptions shp;
 
@@ -99,6 +102,15 @@ public static void main(String[] args) {
 		new InitLocationChoice().execute(args);
 	}
 
+	private static Coord rndCoord(SplittableRandom rnd, double dist, Coord origin) {
+		var angle = rnd.nextDouble() * Math.PI * 2;
+
+		var x = Math.cos(angle) * dist;
+		var y = Math.sin(angle) * dist;
+
+		return new Coord(RunOpenBerlinCalibration.roundNumber(origin.getX() + x), RunOpenBerlinCalibration.roundNumber(origin.getY() + y));
+	}
+
 	@Override
 	public Integer call() throws Exception {
 
@@ -107,8 +119,6 @@ public Integer call() throws Exception {
 			return 2;
 		}
 
-		ctxs = ThreadLocal.withInitial(Context::new);
-
 		Network completeNetwork = NetworkUtils.readNetwork(networkPath.toString());
 		TransportModeNetworkFilter filter = new TransportModeNetworkFilter(completeNetwork);
 		network = NetworkUtils.createNetwork();
@@ -142,35 +152,50 @@ public Integer call() throws Exception {
 		// Build all trees
 		trees.values().forEach(STRtree::build);
 
-		PathMatcher matcher = FileSystems.getDefault().getPathMatcher("glob:**/" + input.getFileName().toString());
+		log.info("Using input file: {}", input);
+
+		List<Population> populations = new ArrayList<>();
 
-		List<Path> input = Files.list(this.input.getParent())
-			.filter(matcher::matches)
-			.sorted()
-			.toList();
+		for (int i = 0; i < k; i++) {
 
-		log.info("Using input files: {}", input);
+			long seed = this.seed + i;
 
-		int i = 1;
-		for (Path p : input) {
+			log.info("Generating plan {} with seed {}", i , seed);
 
+			ctxs = ThreadLocal.withInitial(() -> new Context(seed));
 			commuter = new CommuterAssignment(zones, commuterPath, sample);
 
-			Population population = PopulationUtils.readPopulation(p.toString());
+			Population population = PopulationUtils.readPopulation(input.toString());
 			ParallelPersonAlgorithmUtils.run(population, 8, this);
 
-			String filename = output.toString().replace("plans.xml.gz", "plans-" + i + ".xml.gz");
-			PopulationUtils.writePopulation(population, filename);
+			populations.add(population);
 
-			log.info("Written population to {}", filename);
 			log.info("Processed {} activities with {} warnings", total.get(), warning.get());
 
 			total.set(0);
 			warning.set(0);
+		}
+
+
+		Population population = populations.get(0);
+
+		// Merge all plans into the first population
+		for (int i = 1; i < populations.size(); i++) {
 
-			i++;
+			Population pop = populations.get(i);
+
+			for (Person p : pop.getPersons().values()) {
+				Person destPerson = population.getPersons().get(p.getId());
+				if (destPerson == null) {
+					log.warn("Person {} not present in all populations.", p.getId());
+					continue;
+				}
+
+				destPerson.addPlan(p.getPlans().get(0));
+			}
 		}
 
+		PopulationUtils.writePopulation(population, output.toString());
 
 		return 0;
 	}
@@ -289,7 +314,6 @@ private ActivityFacility sampleCommute(Context ctx, double dist, Coord refCoord,
 		return workPlace;
 	}
 
-
 	/**
 	 * Sample a coordinate for which the associated link is not one of the ignored types.
 	 */
@@ -339,17 +363,12 @@ private boolean checkDistanceBound(double target, Coord refCoord, Coord other, d
 		return dist >= lower && dist <= upper;
 	}
 
-	private static Coord rndCoord(SplittableRandom rnd, double dist, Coord origin) {
-		var angle = rnd.nextDouble() * Math.PI * 2;
-
-		var x = Math.cos(angle) * dist;
-		var y = Math.sin(angle) * dist;
-
-		return new Coord(RunOpenBerlinCalibration.roundNumber(origin.getX() + x), RunOpenBerlinCalibration.roundNumber(origin.getY() + y));
-	}
-
 	private static final class Context {
-		private final SplittableRandom rnd = new SplittableRandom(1);
+		private final SplittableRandom rnd;
+
+		Context(long seed) {
+			rnd = new SplittableRandom(seed);
+		}
 	}
 
 }
diff --git a/src/main/java/org/matsim/prepare/RunOpenBerlinCalibration.java b/src/main/java/org/matsim/prepare/RunOpenBerlinCalibration.java
@@ -8,10 +8,7 @@
 import org.matsim.api.core.v01.Scenario;
 import org.matsim.api.core.v01.TransportMode;
 import org.matsim.api.core.v01.network.Link;
-import org.matsim.api.core.v01.population.Leg;
-import org.matsim.api.core.v01.population.Person;
-import org.matsim.api.core.v01.population.Plan;
-import org.matsim.api.core.v01.population.PlanElement;
+import org.matsim.api.core.v01.population.*;
 import org.matsim.application.MATSimAppCommand;
 import org.matsim.application.MATSimApplication;
 import org.matsim.application.options.SampleOptions;
@@ -45,7 +42,9 @@
 import org.matsim.core.scoring.SumScoringFunction;
 import org.matsim.core.scoring.functions.ScoringParameters;
 import org.matsim.core.scoring.functions.ScoringParametersForPerson;
+import org.matsim.core.utils.geometry.CoordUtils;
 import org.matsim.prepare.counts.CreateCountsFromGeoPortalBerlin;
+import org.matsim.prepare.counts.CreateCountsFromVMZOld;
 import org.matsim.prepare.counts.CreateCountsFromVMZ;
 import org.matsim.prepare.download.DownloadCommuterStatistic;
 import org.matsim.prepare.network.FreeSpeedOptimizer;
@@ -78,7 +77,7 @@
 	LookupRegioStaR.class, ExtractFacilityShp.class, DownSamplePopulation.class, DownloadCommuterStatistic.class,
 	RunActitopp.class, CreateNetworkFromSumo.class, CreateTransitScheduleFromGtfs.class,
 	CleanNetwork.class, SampleNetwork.class, CreateMATSimFacilities.class, InitLocationChoice.class, FilterRelevantAgents.class,
-	CreateCountsFromGeoPortalBerlin.class, CreateCountsFromVMZ.class, ReprojectNetwork.class, RunActivitySampling.class,
+	CreateCountsFromGeoPortalBerlin.class, CreateCountsFromVMZOld.class, CreateCountsFromVMZ.class, ReprojectNetwork.class, RunActivitySampling.class,
 	MergePlans.class, SplitActivityTypesDuration.class, CleanPopulation.class, CleanAttributes.class,
 	GenerateSmallScaleCommercialTrafficDemand.class, RunCountOptimization.class, SelectPlansFromIndex.class,
 	ExtractRelevantFreightTrips.class, CheckCarAvailability.class, FixSubtourModes.class,
@@ -333,10 +332,15 @@ protected void prepareScenario(Scenario scenario) {
 								planElements.indexOf(trip.getOriginActivity()) + 1,
 								planElements.indexOf(trip.getDestinationActivity()));
 
-						if (!Objects.equals(mmi.identifyMainMode(fullTrip), TransportMode.car)) {
+						double dist = CoordUtils.calcEuclideanDistance(getCoord(scenario, trip.getOriginActivity()), getCoord(scenario, trip.getDestinationActivity()));
+
+						// very short trips remain walk
+						String desiredMode = dist <= 100 ? TransportMode.walk : TransportMode.car;
+
+						if (!Objects.equals(mmi.identifyMainMode(fullTrip), desiredMode)) {
 							fullTrip.clear();
-							Leg leg = PopulationUtils.createLeg(TransportMode.car);
-							TripStructureUtils.setRoutingMode(leg, TransportMode.car);
+							Leg leg = PopulationUtils.createLeg(desiredMode);
+							TripStructureUtils.setRoutingMode(leg, desiredMode);
 							fullTrip.add(leg);
 						}
 					}
@@ -345,6 +349,17 @@ protected void prepareScenario(Scenario scenario) {
 		}
 	}
 
+	private Coord getCoord(Scenario scenario, Activity act) {
+
+		if (act.getCoord() != null)
+			return act.getCoord();
+
+		if (act.getFacilityId() != null)
+			return scenario.getActivityFacilities().getFacilities().get(act.getFacilityId()).getCoord();
+
+		return scenario.getNetwork().getLinks().get(act.getLinkId()).getCoord();
+	}
+
 	@Override
 	protected void prepareControler(Controler controler) {