From aac18d8f4ccb6dbe500464207f311d17ae95f2b5 Mon Sep 17 00:00:00 2001 From: rakow Date: Sat, 15 Jul 2023 14:36:47 +0200 Subject: [PATCH] update workflow and makefile --- Makefile | 66 ++- input/v6.0/ignored_6_1.csv | 9 - .../matsim/prepare/InitLocationChoice.java | 79 ++-- .../prepare/RunOpenBerlinCalibration.java | 31 +- .../prepare/counts/CreateCountsFromVMZ.java | 12 +- .../counts/CreateCountsFromVMZOld.java | 379 ++++++++++++++++++ .../prepare/opt/PlanAssignmentProblem.java | 19 +- .../prepare/opt/RunCountOptimization.java | 2 +- src/main/resources/solver.xml | 2 +- 9 files changed, 503 insertions(+), 96 deletions(-) delete mode 100644 input/v6.0/ignored_6_1.csv create mode 100644 src/main/java/org/matsim/prepare/counts/CreateCountsFromVMZOld.java diff --git a/Makefile b/Makefile index af6cde20..39395226 100644 --- a/Makefile +++ b/Makefile @@ -91,7 +91,7 @@ input/sumo.net.xml: input/network.osm --osm-files $< -o=$@ -$p/berlin-$V-network.xml.gz: +$p/berlin-v6.0-network.xml.gz: # Use 5.x network $(sc) prepare reproject-network\ --input $(berlin)/../berlin-v5.5-10pct/input/berlin-v5.5-network.xml.gz\ @@ -102,13 +102,12 @@ $p/berlin-$V-network.xml.gz: --target-crs $(CRS) -$p/berlin-$V-network-with-pt.xml.gz: $p/berlin-$V-network.xml.gz +$p/berlin-v6.0-network-with-pt.xml.gz: $p/berlin-v6.0-network.xml.gz # Copy 5.x network stuff cp $< $@ cp $(berlin)/../berlin-v5.5-10pct/input/berlin-v5.5-transit-vehicles.xml.gz $p/berlin-v6.0-transitVehicles.xml.gz -# TODO: Will be the updated network $p/berlin-v6.1-network.xml.gz: input/sumo.net.xml $(sc) prepare network-from-sumo $< --target-crs $(CRS) --output $@ @@ -123,28 +122,26 @@ $p/berlin-v6.1-network.xml.gz: input/sumo.net.xml $p/berlin-v6.1-network-with-pt.xml.gz: $p/berlin-v6.1-network.xml.gz $(sc) prepare transit-from-gtfs --network $< --output=$p\ - --name berlin-v6.1 --date "2023-06-07" --target-crs $(CRS) \ + --name berlin-$V --date "2023-06-07" --target-crs $(CRS) \ $(germany)/gtfs/complete-pt-2023-06-06.zip\ --shp $p/pt-area/pt-area.shp -$p/berlin-v6.1-car-counts-from-vmz.xml.gz: $p/berlin-v6.1-network.xml.gz +$p/berlin-v6.1-counts-car-vmz.xml.gz: $p/berlin-v6.1-network.xml.gz $(sc) prepare counts-from-vmz\ - --excel ../shared-svn/projects/matsim-berlin/berlin-v5.5/original_data/vmz_counts_2018/Datenexport_2018_TU_Berlin.xlsx\ - --network $<\ - --network-geometries $p/berlin-v6.1-network-linkGeometries.csv\ - --output $p\ - --version berlin-$(V)\ - --input-crs EPSG:31468\ - --target-crs $(CRS)\ - --ignored-counts input/v6.0/ignored_6_1.csv - -# TODO: naming scheme must be updated -$p/berlin-$V-car-counts.xml.gz: $p/berlin-$V-network.xml.gz - $(sc) prepare create-counts\ + --excel ../shared-svn/projects/matsim-berlin/berlin-v5.5/original_data/vmz_counts_2018/Datenexport_2018_TU_Berlin.xlsx\ --network $<\ - --shp $(berlin)/Verkehrsmengen_DTVw_2019.zip\ - --output $p/berlin-$V- - # TODO: output argument not ideal + --network-geometries $p/berlin-v6.0-network-linkGeometries.csv\ + --output $p/\ + --version berlin-$(V)\ + --input-crs EPSG:31468\ + --target-crs $(CRS)\ + --ignored-counts input/ignored_counts.csv + +input/v6.0/berlin-v6.0-counts-car-vmz.xml.gz: + $(sc) prepare counts-from-vmz-old\ + --csv ../shared-svn/projects/matsim-berlin/berlin-v5.5/original_data/vmz_counts_2018/CountsId_to_linkId.csv\ + --excel ../shared-svn/projects/matsim-berlin/berlin-v5.5/original_data/vmz_counts_2018/Datenexport_2018_TU_Berlin.xlsx\ + --output $@ $p/berlin-$V-facilities.xml.gz: $p/berlin-$V-network.xml.gz input/facilities.shp $(sc) prepare facilities --network $< --shp $(word 2,$^)\ @@ -173,18 +170,12 @@ $p/berlin-static-$V-25pct.plans.xml.gz: $p/berlin-only-$V-25pct.plans.xml.gz $p/ $(sc) prepare lookup-regiostar --input $@ --output $@ --xls $(germany)/RegioStaR-Referenzdateien.xlsx -$p/berlin-activities-$V-25pct.plans-1.xml.gz: $p/berlin-static-$V-25pct.plans.xml.gz - # Create five separate sets of activities - $(sc) prepare activity-sampling --seed 2 --input $< --output $(subst plans-1,plans-2,$@) --persons src/main/python/table-persons.csv --activities src/main/python/table-activities.csv - $(sc) prepare activity-sampling --seed 3 --input $< --output $(subst plans-1,plans-3,$@) --persons src/main/python/table-persons.csv --activities src/main/python/table-activities.csv - $(sc) prepare activity-sampling --seed 4 --input $< --output $(subst plans-1,plans-4,$@) --persons src/main/python/table-persons.csv --activities src/main/python/table-activities.csv - $(sc) prepare activity-sampling --seed 5 --input $< --output $(subst plans-1,plans-5,$@) --persons src/main/python/table-persons.csv --activities src/main/python/table-activities.csv - +$p/berlin-activities-$V-25pct.plans.xml.gz: $p/berlin-static-$V-25pct.plans.xml.gz $(sc) prepare activity-sampling --seed 1 --input $< --output $@ --persons src/main/python/table-persons.csv --activities src/main/python/table-activities.csv -$p/berlin-initial-$V-25pct.plans.xml.gz: $p/berlin-activities-$V-25pct.plans-1.xml.gz $p/berlin-$V-facilities.xml.gz $p/berlin-$V-network.xml.gz +$p/berlin-initial-$V-25pct.plans.xml.gz: $p/berlin-activities-$V-25pct.plans.xml.gz $p/berlin-$V-facilities.xml.gz $p/berlin-$V-network.xml.gz $(sc) prepare init-location-choice\ - --input "$(subst plans-1,plans-*,$<)"\ + --input $<\ --output $@\ --facilities $(word 2,$^)\ --network $(word 3,$^)\ @@ -192,9 +183,9 @@ $p/berlin-initial-$V-25pct.plans.xml.gz: $p/berlin-activities-$V-25pct.plans-1.x --commuter $(germany)/regionalstatistik/commuter.csv\ # For debugging and visualization - $(sc) prepare downsample-population $p/berlin-initial-$V-25pct.plans-1.xml.gz\ + $(sc) prepare downsample-population $@\ --sample-size 0.25\ - --samples 0.1\ + --samples 0.1 0.01\ $p/berlin-longHaulFreight-$V-25pct.plans.xml.gz: $p/berlin-$V-network.xml.gz @@ -244,12 +235,9 @@ $p/berlin-goodsTraffic-$V-25pct.plans.xml.gz: mv output/goodsTraffic/$(notdir $@) $@ -# Depends on location choice runs and freight model -$p/berlin-cadyts-input-$V-25pct.plans.xml.gz: $p/berlin-commercialPersonTraffic-$V-25pct.plans.xml.gz - $(sc) prepare merge-plans output/lc-*/*output_selected_plans.xml.gz\ - --output $@ - - $(sc) prepare merge-populations $@ $< --output $@ +$p/berlin-cadyts-input-$V-25pct.plans.xml.gz: $p/berlin-initial-$V-25pct.plans.xml.gz $p/berlin-commercialPersonTraffic-$V-25pct.plans.xml.gz + $(sc) prepare merge-populations $^\ + --output $@ # This file requires eval runs $p/berlin-initial-$V-25pct.experienced_plans.xml.gz: @@ -275,7 +263,7 @@ eval-opt: $p/berlin-initial-$V-25pct.experienced_plans.xml.gz --csv $p/berlin-$V-25pct.plans_selection_$(ERROR_METRIC).csv\ --output $p/berlin-$V-25pct.plans_$(ERROR_METRIC).xml.gz - $(sc) run --mode "eval" --output "output/eval-$(ERROR_METRIC)" --25pct --population "berlin-$V-25pct.plans_$(ERROR_METRIC).xml.gz"\ + $(sc) run --mode "eval" --all-car --output "output/eval-$(ERROR_METRIC)" --25pct --population "berlin-$V-25pct.plans_$(ERROR_METRIC).xml.gz"\ --config $p/berlin-$V-base-calib.config.xml @@ -304,7 +292,7 @@ $p/berlin-$V-25pct.plans.xml.gz: $p/berlin-$V-facilities.xml.gz $p/berlin-$V-net --sample-size 0.25\ --samples 0.1 0.01 0.001\ -prepare-calibration: $p/berlin-initial-$V-25pct.plans.xml.gz +prepare-calibration: $p/berlin-cadyts-input-$V-25pct.plans.xml.gz $p/berlin-$V-network-with-pt.xml.gz $p/berlin-$V-counts-car-vmz.xml.gz echo "Done" prepare: $p/berlin-$V-25pct.plans.xml.gz $p/berlin-$V-network-with-pt.xml.gz diff --git a/input/v6.0/ignored_6_1.csv b/input/v6.0/ignored_6_1.csv deleted file mode 100644 index 6d04d030..00000000 --- a/input/v6.0/ignored_6_1.csv +++ /dev/null @@ -1,9 +0,0 @@ -20149 -20151 -20139 -20154 -20140 -20070 -21905 -21904 -21819 diff --git a/src/main/java/org/matsim/prepare/InitLocationChoice.java b/src/main/java/org/matsim/prepare/InitLocationChoice.java index 37ae17a4..de6ce341 100644 --- a/src/main/java/org/matsim/prepare/InitLocationChoice.java +++ b/src/main/java/org/matsim/prepare/InitLocationChoice.java @@ -33,10 +33,7 @@ import org.opengis.feature.simple.SimpleFeature; import picocli.CommandLine; -import java.nio.file.FileSystems; -import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.PathMatcher; import java.util.*; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; @@ -57,12 +54,15 @@ public class InitLocationChoice implements MATSimAppCommand, PersonAlgorithm { private static final Logger log = LogManager.getLogger(InitLocationChoice.class); - @CommandLine.Option(names = "--input", description = "Path to input population, can be a pattern if * is used.", required = true) + @CommandLine.Option(names = "--input", description = "Path to input population.") private Path input; @CommandLine.Option(names = "--output", description = "Path to output population", required = true) private Path output; + @CommandLine.Option(names = "--k", description = "Number of choices to generate", defaultValue = "5") + private int k; + @CommandLine.Option(names = "--commuter", description = "Path to commuter.csv", required = true) private Path commuterPath; @@ -75,6 +75,9 @@ public class InitLocationChoice implements MATSimAppCommand, PersonAlgorithm { @CommandLine.Option(names = "--sample", description = "Sample size of the population", defaultValue = "0.25") private double sample; + @CommandLine.Option(names = "--seed", description = "Seed used to sample locations", defaultValue = "1") + private long seed; + @CommandLine.Mixin private ShpOptions shp; @@ -99,6 +102,15 @@ public static void main(String[] args) { new InitLocationChoice().execute(args); } + private static Coord rndCoord(SplittableRandom rnd, double dist, Coord origin) { + var angle = rnd.nextDouble() * Math.PI * 2; + + var x = Math.cos(angle) * dist; + var y = Math.sin(angle) * dist; + + return new Coord(RunOpenBerlinCalibration.roundNumber(origin.getX() + x), RunOpenBerlinCalibration.roundNumber(origin.getY() + y)); + } + @Override public Integer call() throws Exception { @@ -107,8 +119,6 @@ public Integer call() throws Exception { return 2; } - ctxs = ThreadLocal.withInitial(Context::new); - Network completeNetwork = NetworkUtils.readNetwork(networkPath.toString()); TransportModeNetworkFilter filter = new TransportModeNetworkFilter(completeNetwork); network = NetworkUtils.createNetwork(); @@ -142,35 +152,50 @@ public Integer call() throws Exception { // Build all trees trees.values().forEach(STRtree::build); - PathMatcher matcher = FileSystems.getDefault().getPathMatcher("glob:**/" + input.getFileName().toString()); + log.info("Using input file: {}", input); + + List populations = new ArrayList<>(); - List input = Files.list(this.input.getParent()) - .filter(matcher::matches) - .sorted() - .toList(); + for (int i = 0; i < k; i++) { - log.info("Using input files: {}", input); + long seed = this.seed + i; - int i = 1; - for (Path p : input) { + log.info("Generating plan {} with seed {}", i , seed); + ctxs = ThreadLocal.withInitial(() -> new Context(seed)); commuter = new CommuterAssignment(zones, commuterPath, sample); - Population population = PopulationUtils.readPopulation(p.toString()); + Population population = PopulationUtils.readPopulation(input.toString()); ParallelPersonAlgorithmUtils.run(population, 8, this); - String filename = output.toString().replace("plans.xml.gz", "plans-" + i + ".xml.gz"); - PopulationUtils.writePopulation(population, filename); + populations.add(population); - log.info("Written population to {}", filename); log.info("Processed {} activities with {} warnings", total.get(), warning.get()); total.set(0); warning.set(0); + } + + + Population population = populations.get(0); + + // Merge all plans into the first population + for (int i = 1; i < populations.size(); i++) { - i++; + Population pop = populations.get(i); + + for (Person p : pop.getPersons().values()) { + Person destPerson = population.getPersons().get(p.getId()); + if (destPerson == null) { + log.warn("Person {} not present in all populations.", p.getId()); + continue; + } + + destPerson.addPlan(p.getPlans().get(0)); + } } + PopulationUtils.writePopulation(population, output.toString()); return 0; } @@ -289,7 +314,6 @@ private ActivityFacility sampleCommute(Context ctx, double dist, Coord refCoord, return workPlace; } - /** * Sample a coordinate for which the associated link is not one of the ignored types. */ @@ -339,17 +363,12 @@ private boolean checkDistanceBound(double target, Coord refCoord, Coord other, d return dist >= lower && dist <= upper; } - private static Coord rndCoord(SplittableRandom rnd, double dist, Coord origin) { - var angle = rnd.nextDouble() * Math.PI * 2; - - var x = Math.cos(angle) * dist; - var y = Math.sin(angle) * dist; - - return new Coord(RunOpenBerlinCalibration.roundNumber(origin.getX() + x), RunOpenBerlinCalibration.roundNumber(origin.getY() + y)); - } - private static final class Context { - private final SplittableRandom rnd = new SplittableRandom(1); + private final SplittableRandom rnd; + + Context(long seed) { + rnd = new SplittableRandom(seed); + } } } diff --git a/src/main/java/org/matsim/prepare/RunOpenBerlinCalibration.java b/src/main/java/org/matsim/prepare/RunOpenBerlinCalibration.java index 651c0d60..d777bfce 100644 --- a/src/main/java/org/matsim/prepare/RunOpenBerlinCalibration.java +++ b/src/main/java/org/matsim/prepare/RunOpenBerlinCalibration.java @@ -8,10 +8,7 @@ import org.matsim.api.core.v01.Scenario; import org.matsim.api.core.v01.TransportMode; import org.matsim.api.core.v01.network.Link; -import org.matsim.api.core.v01.population.Leg; -import org.matsim.api.core.v01.population.Person; -import org.matsim.api.core.v01.population.Plan; -import org.matsim.api.core.v01.population.PlanElement; +import org.matsim.api.core.v01.population.*; import org.matsim.application.MATSimAppCommand; import org.matsim.application.MATSimApplication; import org.matsim.application.options.SampleOptions; @@ -45,7 +42,9 @@ import org.matsim.core.scoring.SumScoringFunction; import org.matsim.core.scoring.functions.ScoringParameters; import org.matsim.core.scoring.functions.ScoringParametersForPerson; +import org.matsim.core.utils.geometry.CoordUtils; import org.matsim.prepare.counts.CreateCountsFromGeoPortalBerlin; +import org.matsim.prepare.counts.CreateCountsFromVMZOld; import org.matsim.prepare.counts.CreateCountsFromVMZ; import org.matsim.prepare.download.DownloadCommuterStatistic; import org.matsim.prepare.network.FreeSpeedOptimizer; @@ -78,7 +77,7 @@ LookupRegioStaR.class, ExtractFacilityShp.class, DownSamplePopulation.class, DownloadCommuterStatistic.class, RunActitopp.class, CreateNetworkFromSumo.class, CreateTransitScheduleFromGtfs.class, CleanNetwork.class, SampleNetwork.class, CreateMATSimFacilities.class, InitLocationChoice.class, FilterRelevantAgents.class, - CreateCountsFromGeoPortalBerlin.class, CreateCountsFromVMZ.class, ReprojectNetwork.class, RunActivitySampling.class, + CreateCountsFromGeoPortalBerlin.class, CreateCountsFromVMZOld.class, CreateCountsFromVMZ.class, ReprojectNetwork.class, RunActivitySampling.class, MergePlans.class, SplitActivityTypesDuration.class, CleanPopulation.class, CleanAttributes.class, GenerateSmallScaleCommercialTrafficDemand.class, RunCountOptimization.class, SelectPlansFromIndex.class, ExtractRelevantFreightTrips.class, CheckCarAvailability.class, FixSubtourModes.class, @@ -333,10 +332,15 @@ protected void prepareScenario(Scenario scenario) { planElements.indexOf(trip.getOriginActivity()) + 1, planElements.indexOf(trip.getDestinationActivity())); - if (!Objects.equals(mmi.identifyMainMode(fullTrip), TransportMode.car)) { + double dist = CoordUtils.calcEuclideanDistance(getCoord(scenario, trip.getOriginActivity()), getCoord(scenario, trip.getDestinationActivity())); + + // very short trips remain walk + String desiredMode = dist <= 100 ? TransportMode.walk : TransportMode.car; + + if (!Objects.equals(mmi.identifyMainMode(fullTrip), desiredMode)) { fullTrip.clear(); - Leg leg = PopulationUtils.createLeg(TransportMode.car); - TripStructureUtils.setRoutingMode(leg, TransportMode.car); + Leg leg = PopulationUtils.createLeg(desiredMode); + TripStructureUtils.setRoutingMode(leg, desiredMode); fullTrip.add(leg); } } @@ -345,6 +349,17 @@ protected void prepareScenario(Scenario scenario) { } } + private Coord getCoord(Scenario scenario, Activity act) { + + if (act.getCoord() != null) + return act.getCoord(); + + if (act.getFacilityId() != null) + return scenario.getActivityFacilities().getFacilities().get(act.getFacilityId()).getCoord(); + + return scenario.getNetwork().getLinks().get(act.getLinkId()).getCoord(); + } + @Override protected void prepareControler(Controler controler) { diff --git a/src/main/java/org/matsim/prepare/counts/CreateCountsFromVMZ.java b/src/main/java/org/matsim/prepare/counts/CreateCountsFromVMZ.java index 359e5e8f..4507ddb8 100644 --- a/src/main/java/org/matsim/prepare/counts/CreateCountsFromVMZ.java +++ b/src/main/java/org/matsim/prepare/counts/CreateCountsFromVMZ.java @@ -130,7 +130,7 @@ private void matchWithNetwork(Path network) throws TransformException, IOExcepti Map, ? extends Link> links = net.getLinks(); - for (var it = stations.entrySet().iterator(); it.hasNext(); ) { + for (var it = stations.entrySet().iterator(); it.hasNext();) { Map.Entry next = it.next(); BerlinCount station = next.getValue(); @@ -194,7 +194,7 @@ private void createCountsFile(String outputFile) { //create hour volumes from 'Tagesganglinie' double[] carShareAtHour = station.carShareAtHour; for (int i = 1; i < 25; i++) { - car.createVolume(i, (station.carVolume * carShareAtHour[i - 1])); + car.createVolume(i, ( (station.totalVolume - station.freightVolume) * carShareAtHour[i - 1])); } if (station.hasFreightShare) { Count freight = countsLkw.createAndAddCount(station.linkId, station.id + "_" + station.position + "_" + station.orientation); @@ -211,8 +211,8 @@ private void createCountsFile(String outputFile) { version += "-"; log.info("Write down {} count stations to file", counter); - new CountsWriter(countsPkw).write(outputFile + version + "car-counts-from-vmz.xml"); - new CountsWriter(countsLkw).write(outputFile + version + "freight-counts-from-vmz.xml"); + new CountsWriter(countsPkw).write(outputFile + version + "counts-car-vmz.xml.gz"); + new CountsWriter(countsLkw).write(outputFile + version + "counts-freight-vmz.xml.gz"); log.info("Write down {} unmatched count stations to file", unmatched.size()); try (CSVPrinter printer = CSVFormat.Builder.create().setHeader("id", "position", "x", "y").build() @@ -267,7 +267,7 @@ private void extractCarVolumes(Sheet sheet) { Row row = it.next(); int id = (int) row.getCell(0).getNumericCellValue(); BerlinCount station = this.stations.get(id); - station.carVolume = (int) row.getCell(1).getNumericCellValue(); + station.totalVolume = (int) row.getCell(1).getNumericCellValue(); } } @@ -342,7 +342,7 @@ private String replaceUmlaute(String str) { private static final class BerlinCount { private final int id; - private int carVolume; + private int totalVolume; private int freightVolume; private final double[] carShareAtHour = new double[24]; private final double[] freightShareAtHour = new double[24]; diff --git a/src/main/java/org/matsim/prepare/counts/CreateCountsFromVMZOld.java b/src/main/java/org/matsim/prepare/counts/CreateCountsFromVMZOld.java new file mode 100644 index 00000000..8dad0789 --- /dev/null +++ b/src/main/java/org/matsim/prepare/counts/CreateCountsFromVMZOld.java @@ -0,0 +1,379 @@ +package org.matsim.prepare.counts; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.poi.xssf.usermodel.XSSFSheet; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.matsim.api.core.v01.Id; +import org.matsim.api.core.v01.network.Link; +import org.matsim.application.MATSimAppCommand; +import org.matsim.counts.Counts; +import org.matsim.counts.CountsWriter; +import picocli.CommandLine; + +import java.io.*; +import java.nio.file.Path; +import java.util.HashMap; +import java.lang.String; + +@CommandLine.Command( + name = "counts-from-vmz-old", + description = "Create counts from the now deprecated (VMZ) data" +) +@Deprecated +public class CreateCountsFromVMZOld implements MATSimAppCommand { + + private static final Logger log = LogManager.getLogger(CreateCountsFromVMZOld.class); + + @CommandLine.Option(names = "--excel", description = "Path to excel file containing the counts") + private Path excel; + + @CommandLine.Option(names = "--csv", description = "Path to csv file with the mapping") + private Path csv; + + @CommandLine.Option(names = "--output", description = "Base path for the output") + private String output; + + private final HashMap berlinCountsMap = new HashMap<>(); + + public static void main(String[] args) { + new CreateCountsFromVMZOld().execute(args); + } + + @Override + public Integer call() throws Exception { + + readExcelFile(excel.toString()); + readMappingFile(csv.toString()); + createCountsFile(output); + + return 0; + } + + /** + * reads the given excel file and creates an BerlinCounts object for every count + * + * @param excel + */ + private void readExcelFile(String excel) { + try { + XSSFWorkbook wb = new XSSFWorkbook(excel); + for (int i = 0; i < wb.getNumberOfSheets(); i++) { + XSSFSheet sheet = wb.getSheetAt(i); + if (sheet.getRow(0).getCell(0).getStringCellValue().equals("MQ_ID")) { + ExcelDataFormat.handleSheet(berlinCountsMap, i, sheet); + } else { + log.warn("sheets should start with MQ_ID, skipping sheet number: {}", i); + } + } + } catch (Exception e) { + log.error(e); + } + } + + /** + * uses the BerlinCounts object to create a car and a truck counts file for matsim + * + * @param outputFile + */ + private void createCountsFile(String outputFile) { + Counts countsPkw = new Counts(); + countsPkw.setYear(2018); + countsPkw.setDescription("data from the berliner senate to matsim counts"); + Counts countsLkw = new Counts(); + countsLkw.setYear(2018); + countsLkw.setDescription("data from the berliner senate to matsim counts"); + + for (BerlinCounts berlinCounts : berlinCountsMap.values()) { + if (!berlinCounts.isUsing()) { + continue; + } + countsPkw.createAndAddCount(Id.createLinkId(berlinCounts.getLinkid()), berlinCounts.getMQ_ID() + "_" + berlinCounts.getPosition() + "_" + berlinCounts.getOrientation()); + double[] PERC_Q_PKW_TYPE = berlinCounts.getPERC_Q_KFZ_TYPE(); + for (int i = 1; i < 25; i++) { + countsPkw.getCount(Id.createLinkId(berlinCounts.getLinkid())).createVolume(i, ( (berlinCounts.getDTVW_KFZ() - berlinCounts.getDTVW_LKW()) * PERC_Q_PKW_TYPE[i - 1])); + } + if (berlinCounts.isLKW_Anteil()) { + countsLkw.createAndAddCount(Id.createLinkId(berlinCounts.getLinkid()), berlinCounts.getMQ_ID() + "_" + berlinCounts.getPosition() + "_" + berlinCounts.getOrientation()); + double[] PERC_Q_LKW_TYPE = berlinCounts.getPERC_Q_LKW_TYPE(); + for (int i = 1; i < 25; i++) { + countsLkw.getCount(Id.createLinkId(berlinCounts.getLinkid())).createVolume(i, (berlinCounts.getDTVW_LKW() * PERC_Q_LKW_TYPE[i - 1])); + } + } + } + CountsWriter writerPkw = new CountsWriter(countsPkw); + CountsWriter writerLkw = new CountsWriter(countsLkw); + writerPkw.write(outputFile); + + if (outputFile.contains("-car")) + writerLkw.write(outputFile.replace("-car", "-hgv")); + } + + /** + * reads a given csv file and adds the data to the BerlinCounts object + * + * @param file + */ + private void readMappingFile(String file) { + try (BufferedReader br = new BufferedReader(new FileReader(file))) { + String headerLine = br.readLine(); + String line; + while ((line = br.readLine()) != null) { + String[] information = line.split(";"); + if (information.length < 3) { + continue; + } + int MQ_ID = Integer.parseInt(information[0]); + int linkid = -999; + if (!information[1].isBlank()) { + linkid = Integer.parseInt(information[1]); + } + String using = information[2]; + BerlinCounts count = berlinCountsMap.get(MQ_ID); + count.setLinkid(linkid); + if (using.equals("x")) { + count.setUsing(true); + } + + if (linkid == -999) { + berlinCountsMap.remove(MQ_ID); + log.warn("No link id: {}", count); + } + } + } catch (IOException e) { + log.error(e); + } + } + + + /** + * a description for every excel sheet that is given in the input file to process the data correctly + */ + private static final class ExcelDataFormat { + + private static final String[] sheet0 = {"MQ_ID", "DTVW_KFZ", "QUALITY"}; + private static final String[] sheet1 = {"MQ_ID", "DTVW_LKW"}; + private static final String[] sheet2 = {"MQ_ID", "PERC_LKW"}; + private static final String[] sheet3 = {"MQ_ID", "HOUR", "PERC_Q_KFZ_TYPE", "PERC_Q_PKW_TYPE", "PERC_Q_LKW_TYPE"}; + private static final String[] sheet4 = {"MQ_ID", "POSITION", "DETAIL", "ORIENTATION", "X_GK4", "Y_GK4", "linkid"}; + + public static HashMap handleSheet(HashMap berlinCountsMap, int i, XSSFSheet sheet) { + if (i == 0) { + for (int j = 1; j <= sheet.getLastRowNum(); j++) { + BerlinCounts berlinCounts = new BerlinCounts((int) sheet.getRow(j).getCell(0).getNumericCellValue()); + berlinCounts.setDTVW_KFZ((int) sheet.getRow(j).getCell(1).getNumericCellValue()); + berlinCountsMap.put(berlinCounts.getMQ_ID(), berlinCounts); + } + } else if (i == 1) { + for (int j = 1; j <= sheet.getLastRowNum(); j++) { + BerlinCounts berlinCounts = berlinCountsMap.get((int) sheet.getRow(j).getCell(0).getNumericCellValue()); + berlinCounts.setDTVW_LKW((int) sheet.getRow(j).getCell(1).getNumericCellValue()); + } + } else if (i == 2) { + for (int j = 1; j <= sheet.getLastRowNum(); j++) { + BerlinCounts berlinCounts = berlinCountsMap.get((int) sheet.getRow(j).getCell(0).getNumericCellValue()); + berlinCounts.setPERC_LKW(sheet.getRow(j).getCell(1).getNumericCellValue()); + berlinCounts.setLKW_Anteil(true); + } + } else if (i == 3) { + for (int j = 1; j <= sheet.getLastRowNum(); j++) { + BerlinCounts berlinCounts = berlinCountsMap.get((int) sheet.getRow(j).getCell(0).getNumericCellValue()); + int hour = (int) sheet.getRow(j).getCell(1).getNumericCellValue(); + double PERC_Q_KFZ_TYPE = 0.0; + double PERC_Q_PKW_TYPE = 0.0; + double PERC_Q_LKW_TYPE = 0.0; + if (sheet.getRow(j).getCell(2) != null) { + PERC_Q_KFZ_TYPE = sheet.getRow(j).getCell(2).getNumericCellValue(); + } + if (sheet.getRow(j).getCell(3) != null) { + PERC_Q_PKW_TYPE = sheet.getRow(j).getCell(3).getNumericCellValue(); + } + if (sheet.getRow(j).getCell(4) != null) { + PERC_Q_LKW_TYPE = sheet.getRow(j).getCell(4).getNumericCellValue(); + } + berlinCounts.setArrays(hour, PERC_Q_KFZ_TYPE, PERC_Q_PKW_TYPE, PERC_Q_LKW_TYPE); + } + } else if (i == 4) { + for (int j = 1; j <= sheet.getLastRowNum(); j++) { + BerlinCounts berlinCounts = berlinCountsMap.get((int) sheet.getRow(j).getCell(0).getNumericCellValue()); + berlinCounts.setPosition(replaceUmlaute(sheet.getRow(j).getCell(1).getStringCellValue())); + if (sheet.getRow(j).getCell(2) != null) { + berlinCounts.setDetail(replaceUmlaute(sheet.getRow(j).getCell(2).getStringCellValue())); + } else { + berlinCounts.setDetail(""); + } + berlinCounts.setOrientation(replaceUmlaute(sheet.getRow(j).getCell(3).getStringCellValue())); +// berlinCounts.setLinkid((int) sheet.getRow(j).getCell(6).getNumericCellValue()); + } + } + return berlinCountsMap; + } + + /** + * replaces the german umlauts + * + * @param str + * @return + */ + private static String replaceUmlaute(String str) { + str = str.replace("ü", "ue") + .replace("ö", "oe") + .replace("ä", "ae") + .replace("ß", "ss") + .replaceAll("Ü(?=[a-zäöüß ])", "Ue") + .replaceAll("Ö(?=[a-zäöüß ])", "Oe") + .replaceAll("Ä(?=[a-zäöüß ])", "Ae") + .replaceAll("Ü", "UE") + .replaceAll("Ö", "OE") + .replaceAll("Ä", "AE"); + return str; + } + + } + + /** + * the BerlinCounts object to save the scanned data for further processing + */ + private static final class BerlinCounts { + + private int MQ_ID; + private int DTVW_KFZ; + private int DTVW_LKW; + private double PERC_LKW; + private double[] PERC_Q_KFZ_TYPE = new double[24]; + private double[] PERC_Q_PKW_TYPE = new double[24]; + private double[] PERC_Q_LKW_TYPE = new double[24]; + private int linkid; + private String position; + private String orientation; + private String detail; + private boolean LKW_Anteil = false; + private boolean using = false; + + public boolean isUsing() { + return using; + } + + public void setUsing(boolean using) { + this.using = using; + } + + public String getDetail() { + return detail; + } + + public void setDetail(String detail) { + this.detail = detail; + } + + public String getPosition() { + return position; + } + + public void setPosition(String position) { + this.position = position; + } + + public String getOrientation() { + return orientation; + } + + public void setOrientation(String orientation) { + this.orientation = orientation; + } + + public boolean isLKW_Anteil() { + return LKW_Anteil; + } + + public void setLKW_Anteil(boolean LKW_Anteil) { + this.LKW_Anteil = LKW_Anteil; + } + + public BerlinCounts(int MQ_ID) { + this.MQ_ID = MQ_ID; + } + + public int getMQ_ID() { + return MQ_ID; + } + + public void setMQ_ID(int MQ_ID) { + this.MQ_ID = MQ_ID; + } + + public int getDTVW_KFZ() { + return DTVW_KFZ; + } + + public void setDTVW_KFZ(int DTVW_KFZ) { + this.DTVW_KFZ = DTVW_KFZ; + } + + public int getDTVW_LKW() { + return DTVW_LKW; + } + + public void setDTVW_LKW(int DTVW_LKW) { + this.DTVW_LKW = DTVW_LKW; + } + + public double getPERC_LKW() { + return PERC_LKW; + } + + public void setPERC_LKW(double PERC_LKW) { + this.PERC_LKW = PERC_LKW; + } + + public double[] getPERC_Q_KFZ_TYPE() { + return PERC_Q_KFZ_TYPE; + } + + public void setPERC_Q_KFZ_TYPE(double[] PERC_Q_KFZ_TYPE) { + this.PERC_Q_KFZ_TYPE = PERC_Q_KFZ_TYPE; + } + + public double[] getPERC_Q_PKW_TYPE() { + return PERC_Q_PKW_TYPE; + } + + public void setPERC_Q_PKW_TYPE(double[] PERC_Q_PKW_TYPE) { + this.PERC_Q_PKW_TYPE = PERC_Q_PKW_TYPE; + } + + public double[] getPERC_Q_LKW_TYPE() { + return PERC_Q_LKW_TYPE; + } + + public void setPERC_Q_LKW_TYPE(double[] PERC_Q_LKW_TYPE) { + this.PERC_Q_LKW_TYPE = PERC_Q_LKW_TYPE; + } + + public int getLinkid() { + return linkid; + } + + public void setLinkid(int linkid) { + this.linkid = linkid; + } + + public void setArrays(int i, double PERC_Q_KFZ_TYPE, double PERC_Q_PKW_TYPE, double PERC_Q_LKW_TYPE) { + this.PERC_Q_KFZ_TYPE[i] = PERC_Q_KFZ_TYPE; + this.PERC_Q_PKW_TYPE[i] = PERC_Q_PKW_TYPE; + this.PERC_Q_LKW_TYPE[i] = PERC_Q_LKW_TYPE; + } + + @Override + public String toString() { + return "BerlinCounts{" + + "MQ_ID=" + MQ_ID + + ", linkid=" + linkid + + ", position='" + position + '\'' + + ", orientation='" + orientation + '\'' + + ", detail='" + detail + '\'' + + '}'; + } + } + +} diff --git a/src/main/java/org/matsim/prepare/opt/PlanAssignmentProblem.java b/src/main/java/org/matsim/prepare/opt/PlanAssignmentProblem.java index a4da9084..b42c826c 100644 --- a/src/main/java/org/matsim/prepare/opt/PlanAssignmentProblem.java +++ b/src/main/java/org/matsim/prepare/opt/PlanAssignmentProblem.java @@ -91,13 +91,28 @@ public void iterate(int n, double prob, double beta, double w) { double step = prob / n; + double best = score.score().doubleValue(); + int noBest = 0; + for (int i = 0; i < n; i++) { calc.resetWorkingSolution(this); score = calc.calculateScore(); - if (n % 500 == 0) - RunCountOptimization.log.info("Iteration {} score: {}", n, score); + if (i % 100 == 0) + RunCountOptimization.log.info("Iteration {} score: {}", i, score); + + if (score.score().doubleValue() >= best) { + best = score.score().doubleValue(); + noBest = 0; + } else { + noBest++; + } + + if (noBest >= 30) { + RunCountOptimization.log.info("Stopping after {} with score: {}", i, score); + break; + } // Best p and beta are not known, so it will be annealed double p = prob - step * i; diff --git a/src/main/java/org/matsim/prepare/opt/RunCountOptimization.java b/src/main/java/org/matsim/prepare/opt/RunCountOptimization.java index ad94c1a6..9897bab8 100644 --- a/src/main/java/org/matsim/prepare/opt/RunCountOptimization.java +++ b/src/main/java/org/matsim/prepare/opt/RunCountOptimization.java @@ -234,7 +234,7 @@ private PlanAssignmentProblem solve(PlanAssignmentProblem problem) { solver.addEventListener(event -> { // Only log every x seconds - if (ts.get() + 30_000 < System.currentTimeMillis()) { + if (ts.get() + 60_000 < System.currentTimeMillis()) { log.info("New best solution: {}", event.getNewBestScore()); ts.set(System.currentTimeMillis()); } diff --git a/src/main/resources/solver.xml b/src/main/resources/solver.xml index 73b164c0..19c7e0ae 100644 --- a/src/main/resources/solver.xml +++ b/src/main/resources/solver.xml @@ -22,7 +22,7 @@ - 50000 + 100000