From dd3d638b81fb98c3d152441d11f0ee4c56247fe1 Mon Sep 17 00:00:00 2001 From: Mark Idleman Date: Fri, 25 Sep 2020 16:39:45 -0700 Subject: [PATCH 1/4] Use map matching with GTFS shapes to find routes in GTFS link mapper --- replica-common/pom.xml | 22 ++ .../graphhopper/replica/GtfsLinkMapper.java | 190 ++++++++++++++++++ .../http/cli/GtfsLinkMapperCommand.java | 2 +- 3 files changed, 213 insertions(+), 1 deletion(-) diff --git a/replica-common/pom.xml b/replica-common/pom.xml index 7786d392ccb..fd0c524658c 100644 --- a/replica-common/pom.xml +++ b/replica-common/pom.xml @@ -21,6 +21,28 @@ graphhopper-matrix 1.0.17-replica1 + + com.graphhopper + graphhopper-map-matching-core + 1.0 + + + * + * + + + + + com.graphhopper + graphhopper-map-matching-web-bundle + 1.0 + + + * + * + + + diff --git a/web-bundle/src/main/java/com/graphhopper/replica/GtfsLinkMapper.java b/web-bundle/src/main/java/com/graphhopper/replica/GtfsLinkMapper.java index fd89787455f..5b4cc543a67 100644 --- a/web-bundle/src/main/java/com/graphhopper/replica/GtfsLinkMapper.java +++ b/web-bundle/src/main/java/com/graphhopper/replica/GtfsLinkMapper.java @@ -4,6 +4,7 @@ import com.conveyal.gtfs.model.Route; import com.conveyal.gtfs.model.Stop; import com.conveyal.gtfs.model.StopTime; +import com.conveyal.gtfs.model.Trip; import com.google.common.collect.HashMultimap; import com.google.common.collect.Lists; import com.google.common.collect.SetMultimap; @@ -13,8 +14,19 @@ import com.graphhopper.GraphHopper; import com.graphhopper.gtfs.GraphHopperGtfs; import com.graphhopper.gtfs.GtfsStorage; +import com.graphhopper.matching.MapMatching; +import com.graphhopper.matching.MatchResult; +import com.graphhopper.matching.Observation; +import com.graphhopper.routing.ev.BooleanEncodedValue; +import com.graphhopper.routing.ev.SimpleBooleanEncodedValue; +import com.graphhopper.routing.util.AllEdgesIterator; +import com.graphhopper.routing.util.CarFlagEncoder; +import com.graphhopper.routing.util.EncodingManager; +import com.graphhopper.util.PMap; import com.graphhopper.util.details.PathDetail; +import com.graphhopper.util.shapes.GHPoint; import org.apache.commons.lang3.tuple.Pair; +import org.locationtech.jts.geom.LineString; import org.mapdb.DB; import org.mapdb.DBMaker; import org.mapdb.HTreeMap; @@ -37,6 +49,184 @@ public GtfsLinkMapper(GraphHopper graphHopper) { this.graphHopper = graphHopper; } + public void setGtfsLinkMappingsMapMatching() { + GtfsStorage gtfsStorage = ((GraphHopperGtfs) graphHopper).getGtfsStorage(); + Map gtfsFeedMap = gtfsStorage.getGtfsFeeds(); + final Set STREET_BASED_ROUTE_TYPES = Sets.newHashSet(0, 3, 5); + + Set matchedEdgeSet = Sets.newHashSet(); + PMap hints = new PMap(); + hints.putObject("profile", "car"); + MapMatching matching = new MapMatching(graphHopper, hints); + + for (String feedId : gtfsFeedMap.keySet()) { + GTFSFeed feed = gtfsFeedMap.get(feedId); + + // For mapping purposes, only look at routes for transit that use the street network + Set streetBasedRouteIdsForFeed = feed.routes.values().stream() + .filter(route -> STREET_BASED_ROUTE_TYPES.contains(route.route_type)) + .map(route -> route.route_id) + .collect(Collectors.toSet()); + + // Find all GTFS trips for each route + Set tripsForStreetBasedRoutes = feed.trips.values().stream() + .filter(trip -> streetBasedRouteIdsForFeed.contains(trip.route_id)) + .map(trip -> trip.trip_id) + .collect(Collectors.toSet()); + + for (String tripid : tripsForStreetBasedRoutes) { + LineString tripGeometry = feed.getTripGeometry(tripid); + + // do map matching, store results in set + List pointsToMatch = Arrays.stream(tripGeometry.getCoordinates()) + .map(coordinate -> new GHPoint(coordinate.x, coordinate.y)) + .map(ghPoint -> new Observation(ghPoint)) + .collect(Collectors.toList()); + + MatchResult result = matching.doWork(pointsToMatch); + result.getMergedPath().calcEdges().stream() + .map(edge -> edge.getEdge()) + .forEach(edgeId -> matchedEdgeSet.add(edgeId)); + } + + //make new flag encoder; loop through set and mark each of them as accessible (all others not) + //route using this flag encoder profile + BooleanEncodedValue carAccessEncoder = graphHopper.getEncodingManager().getEncoder("car").getAccessEnc(); + AllEdgesIterator edgesIterator = graphHopper.getGraphHopperStorage().getAllEdges(); + while (edgesIterator.next()) { + if (!matchedEdgeSet.contains(edgesIterator.getEdge())) { + edgesIterator.set(carAccessEncoder, false); + } + } + } + + + // Do standard stop->stop routing on new "subgraph" + + // Initialize mapdb database to store link mappings and route info + logger.info("Initializing new mapdb file to store link mappings"); + DB db = DBMaker.newFileDB(new File("transit_data/gtfs_link_mappings.db")).make(); + HTreeMap gtfsLinkMappings = db + .createHashMap("gtfsLinkMappings") + .keySerializer(Serializer.STRING) + .valueSerializer(Serializer.STRING) + .make(); + + Set allStableIds = Sets.newHashSet(); + + // For each GTFS feed, pull out all stops for trips on GTFS routes that travel on the street network, + // and then for each trip, route via car between each stop pair in sequential order, storing the returned IDs + for (String feedId : gtfsFeedMap.keySet()) { + GTFSFeed feed = gtfsFeedMap.get(feedId); + logger.info("Processing GTFS feed " + feedId + " " + feed.feedId); + + // For mapping purposes, only look at routes for transit that use the street network + Set streetBasedRouteIdsForFeed = feed.routes.values().stream() + .filter(route -> STREET_BASED_ROUTE_TYPES.contains(route.route_type)) + .map(route -> route.route_id) + .collect(Collectors.toSet()); + + // Find all GTFS trips for each route + Set tripsForStreetBasedRoutes = feed.trips.values().stream() + .filter(trip -> streetBasedRouteIdsForFeed.contains(trip.route_id)) + .map(trip -> trip.trip_id) + .collect(Collectors.toSet()); + + // Find all stops for each trip + SetMultimap tripIdToStopsInTrip = HashMultimap.create(); + feed.stop_times.values().stream() + .filter(stopTime -> tripsForStreetBasedRoutes.contains(stopTime.trip_id)) + .forEach(stopTime -> tripIdToStopsInTrip.put(stopTime.trip_id, stopTime)); + + Set stopIdsForStreetBasedTrips = tripIdToStopsInTrip.values().stream() + .map(stopTime -> stopTime.stop_id) + .collect(Collectors.toSet()); + + Map stopsForStreetBasedTrips = feed.stops.values().stream() + .filter(stop -> stopIdsForStreetBasedTrips.contains(stop.stop_id)) + .collect(Collectors.toMap(stop -> stop.stop_id, stop -> stop)); + + logger.info("There are " + streetBasedRouteIdsForFeed.size() + " GTFS routes containing " + + tripsForStreetBasedRoutes.size() + " total trips to process for this feed. Routes to be computed for " + + stopIdsForStreetBasedTrips.size() + "/" + feed.stops.values().size() + " stop->stop pairs"); + + int processedTripCount = 0; + int odStopCount = 0; + int nonUniqueODPairs = 0; + int routeNotFoundCount = 0; + // For each trip, route with auto between all O/D stop pairs, + // and store returned stable edge IDs for each route in mapdb file + for (String tripId : tripIdToStopsInTrip.keySet()) { + if (processedTripCount % (tripIdToStopsInTrip.keySet().size() / 10) == 0) { + logger.info(processedTripCount + "/" + tripIdToStopsInTrip.keySet().size() + " trips for feed " + + feedId + " processed so far; " + nonUniqueODPairs + "/" + odStopCount + + " O/D stop pairs were non-unique, and were not routed between."); + } + + // Fetch all sequentially-ordered stop->stop pairs for this trip + List> odStopsForTrip = getODStopsForTrip(tripIdToStopsInTrip.get(tripId), stopsForStreetBasedTrips); + + // Route a car between each stop->stop pair, and store the returned stable edge IDs in mapdb map + for (Pair odStopPair : odStopsForTrip) { + odStopCount++; + + // Create String from the ID of each stop in pair, to use as key for map + String stopPairString = odStopPair.getLeft().stop_id + "," + odStopPair.getRight().stop_id; + + // Don't route for any stop->stop pairs we've already routed between + if (gtfsLinkMappings.containsKey(stopPairString)) { + nonUniqueODPairs++; + continue; + } + + // Form stop->stop auto routing requests and request a route + GHRequest odRequest = new GHRequest( + odStopPair.getLeft().stop_lat, odStopPair.getLeft().stop_lon, + odStopPair.getRight().stop_lat, odStopPair.getRight().stop_lon + ); + odRequest.setProfile("car"); + odRequest.setPathDetails(Lists.newArrayList("r5_edge_id")); + GHResponse response = graphHopper.route(odRequest); + + // If stop->stop path couldn't be found by GH, don't store anything + if (response.getAll().size() == 0 || response.getAll().get(0).hasErrors()) { + routeNotFoundCount++; + continue; + } + + // Parse stable IDs for each edge from response + List responsePathEdgeIdDetails = response.getAll().get(0).getPathDetails().get("r5_edge_id"); + List pathEdgeIds = responsePathEdgeIdDetails.stream() + .map(pathDetail -> (String) pathDetail.getValue()) + .collect(Collectors.toList()); + // allStableIds.addAll(pathEdgeIds); + + // Merge all path IDs into String to use as value for gtfs link map + String pathStableEdgeIdString = pathEdgeIds.stream().collect(Collectors.joining(",")); + gtfsLinkMappings.put(stopPairString, pathStableEdgeIdString); + } + processedTripCount++; + } + logger.info("Done processing GTFS feed " + feedId + "; " + tripIdToStopsInTrip.keySet().size() + + " total trips processed; " + nonUniqueODPairs + "/" + odStopCount + + " O/D stop pairs were non-unique; routes for " + routeNotFoundCount + "/" + odStopCount + + " stop->stop pairs were not found"); + } + db.commit(); + db.close(); + logger.info("Done creating GTFS link mappings for " + gtfsFeedMap.size() + " GTFS feeds"); + + // For testing + logger.info("All stable edge IDs: "); + logger.info(allStableIds.stream().collect(Collectors.joining(","))); + + + } + + + + + public void setGtfsLinkMappings() { logger.info("Starting GTFS link mapping process"); GtfsStorage gtfsStorage = ((GraphHopperGtfs) graphHopper).getGtfsStorage(); diff --git a/web/src/main/java/com/graphhopper/http/cli/GtfsLinkMapperCommand.java b/web/src/main/java/com/graphhopper/http/cli/GtfsLinkMapperCommand.java index a36ff771168..69573268521 100644 --- a/web/src/main/java/com/graphhopper/http/cli/GtfsLinkMapperCommand.java +++ b/web/src/main/java/com/graphhopper/http/cli/GtfsLinkMapperCommand.java @@ -20,7 +20,7 @@ protected void run(Bootstrap bootstrap, Namespac GraphHopper gh = graphHopper.getGraphHopper(); gh.load(gh.getGraphHopperLocation()); GtfsLinkMapper gtfsLinkMapper = new GtfsLinkMapper(gh); - gtfsLinkMapper.setGtfsLinkMappings(); + gtfsLinkMapper.setGtfsLinkMappingsMapMatching(); gh.close(); } } From ccdbb47317bd2f7afdba42fcde228c72f364a1a9 Mon Sep 17 00:00:00 2001 From: Mark Idleman Date: Fri, 25 Sep 2020 16:45:31 -0700 Subject: [PATCH 2/4] cleanup + comments --- .../graphhopper/replica/GtfsLinkMapper.java | 23 +++++-------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/web-bundle/src/main/java/com/graphhopper/replica/GtfsLinkMapper.java b/web-bundle/src/main/java/com/graphhopper/replica/GtfsLinkMapper.java index 5b4cc543a67..d2250787d4c 100644 --- a/web-bundle/src/main/java/com/graphhopper/replica/GtfsLinkMapper.java +++ b/web-bundle/src/main/java/com/graphhopper/replica/GtfsLinkMapper.java @@ -4,7 +4,6 @@ import com.conveyal.gtfs.model.Route; import com.conveyal.gtfs.model.Stop; import com.conveyal.gtfs.model.StopTime; -import com.conveyal.gtfs.model.Trip; import com.google.common.collect.HashMultimap; import com.google.common.collect.Lists; import com.google.common.collect.SetMultimap; @@ -18,10 +17,7 @@ import com.graphhopper.matching.MatchResult; import com.graphhopper.matching.Observation; import com.graphhopper.routing.ev.BooleanEncodedValue; -import com.graphhopper.routing.ev.SimpleBooleanEncodedValue; import com.graphhopper.routing.util.AllEdgesIterator; -import com.graphhopper.routing.util.CarFlagEncoder; -import com.graphhopper.routing.util.EncodingManager; import com.graphhopper.util.PMap; import com.graphhopper.util.details.PathDetail; import com.graphhopper.util.shapes.GHPoint; @@ -54,11 +50,13 @@ public void setGtfsLinkMappingsMapMatching() { Map gtfsFeedMap = gtfsStorage.getGtfsFeeds(); final Set STREET_BASED_ROUTE_TYPES = Sets.newHashSet(0, 3, 5); + // Set up map matcher and Set to hold matched edge IDs Set matchedEdgeSet = Sets.newHashSet(); PMap hints = new PMap(); hints.putObject("profile", "car"); MapMatching matching = new MapMatching(graphHopper, hints); + // For each feed, perform map matching against geo of each trip, and store all matched edges for (String feedId : gtfsFeedMap.keySet()) { GTFSFeed feed = gtfsFeedMap.get(feedId); @@ -77,7 +75,7 @@ public void setGtfsLinkMappingsMapMatching() { for (String tripid : tripsForStreetBasedRoutes) { LineString tripGeometry = feed.getTripGeometry(tripid); - // do map matching, store results in set + // do map matching, store results in Set List pointsToMatch = Arrays.stream(tripGeometry.getCoordinates()) .map(coordinate -> new GHPoint(coordinate.x, coordinate.y)) .map(ghPoint -> new Observation(ghPoint)) @@ -89,8 +87,7 @@ public void setGtfsLinkMappingsMapMatching() { .forEach(edgeId -> matchedEdgeSet.add(edgeId)); } - //make new flag encoder; loop through set and mark each of them as accessible (all others not) - //route using this flag encoder profile + // Set all edges that were not matched against GTFS shapes as inaccessible, so they can't be routed on BooleanEncodedValue carAccessEncoder = graphHopper.getEncodingManager().getEncoder("car").getAccessEnc(); AllEdgesIterator edgesIterator = graphHopper.getGraphHopperStorage().getAllEdges(); while (edgesIterator.next()) { @@ -100,9 +97,7 @@ public void setGtfsLinkMappingsMapMatching() { } } - - // Do standard stop->stop routing on new "subgraph" - + // ----Now, perform standard link mapping procedure, routing only along still-accessible edges---- // Initialize mapdb database to store link mappings and route info logger.info("Initializing new mapdb file to store link mappings"); DB db = DBMaker.newFileDB(new File("transit_data/gtfs_link_mappings.db")).make(); @@ -219,14 +214,8 @@ public void setGtfsLinkMappingsMapMatching() { // For testing logger.info("All stable edge IDs: "); logger.info(allStableIds.stream().collect(Collectors.joining(","))); - - } - - - - - + public void setGtfsLinkMappings() { logger.info("Starting GTFS link mapping process"); GtfsStorage gtfsStorage = ((GraphHopperGtfs) graphHopper).getGtfsStorage(); From f917ff10c52944c1eb97cfba4bbc8e131cd5bb7b Mon Sep 17 00:00:00 2001 From: Mark Idleman Date: Mon, 5 Oct 2020 10:47:37 -0700 Subject: [PATCH 3/4] update map matching package version to avoid conflicts --- replica-common/pom.xml | 16 ++-------------- .../com/graphhopper/replica/GtfsLinkMapper.java | 7 +++++-- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/replica-common/pom.xml b/replica-common/pom.xml index fd0c524658c..5309291c503 100644 --- a/replica-common/pom.xml +++ b/replica-common/pom.xml @@ -24,24 +24,12 @@ com.graphhopper graphhopper-map-matching-core - 1.0 - - - * - * - - + e164705aa7a6320a29d7760294be8d353b7b21c7 com.graphhopper graphhopper-map-matching-web-bundle - 1.0 - - - * - * - - + e164705aa7a6320a29d7760294be8d353b7b21c7 diff --git a/web-bundle/src/main/java/com/graphhopper/replica/GtfsLinkMapper.java b/web-bundle/src/main/java/com/graphhopper/replica/GtfsLinkMapper.java index d2250787d4c..5741223f75c 100644 --- a/web-bundle/src/main/java/com/graphhopper/replica/GtfsLinkMapper.java +++ b/web-bundle/src/main/java/com/graphhopper/replica/GtfsLinkMapper.java @@ -37,6 +37,8 @@ import java.util.Set; import java.util.stream.Collectors; +import static com.graphhopper.util.Parameters.Routing.MAX_VISITED_NODES; + public class GtfsLinkMapper { private final Logger logger = LoggerFactory.getLogger(getClass()); private final GraphHopper graphHopper; @@ -54,6 +56,7 @@ public void setGtfsLinkMappingsMapMatching() { Set matchedEdgeSet = Sets.newHashSet(); PMap hints = new PMap(); hints.putObject("profile", "car"); + hints.putObject(MAX_VISITED_NODES, 10000); MapMatching matching = new MapMatching(graphHopper, hints); // For each feed, perform map matching against geo of each trip, and store all matched edges @@ -77,11 +80,11 @@ public void setGtfsLinkMappingsMapMatching() { // do map matching, store results in Set List pointsToMatch = Arrays.stream(tripGeometry.getCoordinates()) - .map(coordinate -> new GHPoint(coordinate.x, coordinate.y)) + .map(coordinate -> new GHPoint(coordinate.y, coordinate.x)) .map(ghPoint -> new Observation(ghPoint)) .collect(Collectors.toList()); - MatchResult result = matching.doWork(pointsToMatch); + MatchResult result = matching.match(pointsToMatch); result.getMergedPath().calcEdges().stream() .map(edge -> edge.getEdge()) .forEach(edgeId -> matchedEdgeSet.add(edgeId)); From 805d2f399b3ccbdb6bcaf1d2e9f383b31bbc1616 Mon Sep 17 00:00:00 2001 From: Mark Idleman Date: Mon, 14 Dec 2020 16:02:14 -0800 Subject: [PATCH 4/4] update code to be hybrid of map matching + routing --- .../graphhopper/replica/GtfsLinkMapper.java | 277 +++++++++++------- 1 file changed, 175 insertions(+), 102 deletions(-) diff --git a/web-bundle/src/main/java/com/graphhopper/replica/GtfsLinkMapper.java b/web-bundle/src/main/java/com/graphhopper/replica/GtfsLinkMapper.java index 0b2c1f7a66f..b5128230e2f 100644 --- a/web-bundle/src/main/java/com/graphhopper/replica/GtfsLinkMapper.java +++ b/web-bundle/src/main/java/com/graphhopper/replica/GtfsLinkMapper.java @@ -14,7 +14,11 @@ import com.graphhopper.matching.MatchResult; import com.graphhopper.matching.Observation; import com.graphhopper.routing.ev.BooleanEncodedValue; -import com.graphhopper.routing.util.AllEdgesIterator; +import com.graphhopper.routing.util.EdgeFilter; +import com.graphhopper.stableid.StableIdEncodedValues; +import com.graphhopper.storage.index.Location2IDFullWithEdgesIndex; +import com.graphhopper.storage.index.LocationIndex; +import com.graphhopper.util.EdgeIteratorState; import com.graphhopper.util.PMap; import com.graphhopper.util.details.PathDetail; import com.graphhopper.util.shapes.GHPoint; @@ -51,55 +55,14 @@ public void setGtfsLinkMappingsMapMatching() { Map gtfsFeedMap = gtfsStorage.getGtfsFeeds(); final Set STREET_BASED_ROUTE_TYPES = Sets.newHashSet(0, 3, 5); - // Set up map matcher and Set to hold matched edge IDs - Set matchedEdgeSet = Sets.newHashSet(); + // Set up map matcher PMap hints = new PMap(); hints.putObject("profile", "car"); hints.putObject(MAX_VISITED_NODES, 10000); MapMatching matching = new MapMatching(graphHopper, hints); + LocationIndex locationIndex = + new Location2IDFullWithEdgesIndex(graphHopper.getGraphHopperStorage().getBaseGraph()); - // For each feed, perform map matching against geo of each trip, and store all matched edges - for (String feedId : gtfsFeedMap.keySet()) { - GTFSFeed feed = gtfsFeedMap.get(feedId); - - // For mapping purposes, only look at routes for transit that use the street network - Set streetBasedRouteIdsForFeed = feed.routes.values().stream() - .filter(route -> STREET_BASED_ROUTE_TYPES.contains(route.route_type)) - .map(route -> route.route_id) - .collect(Collectors.toSet()); - - // Find all GTFS trips for each route - Set tripsForStreetBasedRoutes = feed.trips.values().stream() - .filter(trip -> streetBasedRouteIdsForFeed.contains(trip.route_id)) - .map(trip -> trip.trip_id) - .collect(Collectors.toSet()); - - for (String tripid : tripsForStreetBasedRoutes) { - LineString tripGeometry = feed.getTripGeometry(tripid); - - // do map matching, store results in Set - List pointsToMatch = Arrays.stream(tripGeometry.getCoordinates()) - .map(coordinate -> new GHPoint(coordinate.y, coordinate.x)) - .map(ghPoint -> new Observation(ghPoint)) - .collect(Collectors.toList()); - - MatchResult result = matching.match(pointsToMatch); - result.getMergedPath().calcEdges().stream() - .map(edge -> edge.getEdge()) - .forEach(edgeId -> matchedEdgeSet.add(edgeId)); - } - - // Set all edges that were not matched against GTFS shapes as inaccessible, so they can't be routed on - BooleanEncodedValue carAccessEncoder = graphHopper.getEncodingManager().getEncoder("car").getAccessEnc(); - AllEdgesIterator edgesIterator = graphHopper.getGraphHopperStorage().getAllEdges(); - while (edgesIterator.next()) { - if (!matchedEdgeSet.contains(edgesIterator.getEdge())) { - edgesIterator.set(carAccessEncoder, false); - } - } - } - - // ----Now, perform standard link mapping procedure, routing only along still-accessible edges---- // Initialize mapdb database to store link mappings and route info logger.info("Initializing new mapdb file to store link mappings"); DB db = DBMaker.newFileDB(new File("transit_data/gtfs_link_mappings.db")).make(); @@ -109,13 +72,15 @@ public void setGtfsLinkMappingsMapMatching() { .valueSerializer(Serializer.STRING) .make(); + StableIdEncodedValues stableIdEncodedValues = + StableIdEncodedValues.fromEncodingManager(graphHopper.getEncodingManager()); + BooleanEncodedValue carAccessEncoder = graphHopper.getEncodingManager().getEncoder("car").getAccessEnc(); Set allStableIds = Sets.newHashSet(); - // For each GTFS feed, pull out all stops for trips on GTFS routes that travel on the street network, - // and then for each trip, route via car between each stop pair in sequential order, storing the returned IDs + // For each feed, perform map matching against geo of each trip, and store all matched edges for (String feedId : gtfsFeedMap.keySet()) { + logger.info("Processing GTFS feed " + feedId + " " + feedId); GTFSFeed feed = gtfsFeedMap.get(feedId); - logger.info("Processing GTFS feed " + feedId + " " + feed.feedId); // For mapping purposes, only look at routes for transit that use the street network Set streetBasedRouteIdsForFeed = feed.routes.values().stream() @@ -143,71 +108,151 @@ public void setGtfsLinkMappingsMapMatching() { .filter(stop -> stopIdsForStreetBasedTrips.contains(stop.stop_id)) .collect(Collectors.toMap(stop -> stop.stop_id, stop -> stop)); - logger.info("There are " + streetBasedRouteIdsForFeed.size() + " GTFS routes containing " - + tripsForStreetBasedRoutes.size() + " total trips to process for this feed. Routes to be computed for " - + stopIdsForStreetBasedTrips.size() + "/" + feed.stops.values().size() + " stop->stop pairs"); - - int processedTripCount = 0; int odStopCount = 0; int nonUniqueODPairs = 0; int routeNotFoundCount = 0; - // For each trip, route with auto between all O/D stop pairs, - // and store returned stable edge IDs for each route in mapdb file - for (String tripId : tripIdToStopsInTrip.keySet()) { - if (processedTripCount % (tripIdToStopsInTrip.keySet().size() / 10) == 0) { + int tripsMatchedCount = 0; + int tripsNotMatchedCount = 0; + int matchedNotSnappedCount = 0; + int allStopsAlreadyRoutedCount = 0; + int processedTripCount = 0; + + for (String tripId : tripsForStreetBasedRoutes) { + if (tripIdToStopsInTrip.keySet().size() > 10 && + processedTripCount % (tripIdToStopsInTrip.keySet().size() / 10) == 0) { logger.info(processedTripCount + "/" + tripIdToStopsInTrip.keySet().size() + " trips for feed " - + feedId + " processed so far; " + nonUniqueODPairs + "/" + odStopCount + + feed.feedId + " processed so far; " + nonUniqueODPairs + "/" + odStopCount + " O/D stop pairs were non-unique, and were not routed between."); } + processedTripCount++; - // Fetch all sequentially-ordered stop->stop pairs for this trip - List> odStopsForTrip = getODStopsForTrip(tripIdToStopsInTrip.get(tripId), stopsForStreetBasedTrips); - - // Route a car between each stop->stop pair, and store the returned stable edge IDs in mapdb map - for (Pair odStopPair : odStopsForTrip) { - odStopCount++; + List> odStopsForTrip = + getODStopsForTrip(Sets.newHashSet(feed.getOrderedStopTimesForTrip(tripId)), stopsForStreetBasedTrips); - // Create String from the ID of each stop in pair, to use as key for map - String stopPairString = odStopPair.getLeft().stop_id + "," + odStopPair.getRight().stop_id; + // Skip processing if all stop->stop pairs in trip have already been mapped + if (stopsAlreadyRouted(odStopsForTrip, gtfsLinkMappings)) { + allStopsAlreadyRoutedCount++; + continue; + } + try { + // Attempt to map-match trip geometry to street network + LineString tripGeometry = feed.getTripGeometry(tripId); + List pointsToMatch = Arrays.stream(tripGeometry.getCoordinates()) + .map(coordinate -> new GHPoint(coordinate.y, coordinate.x)) + .map(ghPoint -> new Observation(ghPoint)) + .collect(Collectors.toList()); - // Don't route for any stop->stop pairs we've already routed between - if (gtfsLinkMappings.containsKey(stopPairString)) { - nonUniqueODPairs++; - continue; + // Match points to network and store pair of (GH edge ID, stable edge ID) for all matched edges + MatchResult result = matching.match(pointsToMatch); + List> matchedEdgeSet = result.getMergedPath().calcEdges().stream() + .map(edge -> Pair.of(edge.getEdge(), stableIdEncodedValues.getStableId(edge.getReverse(carAccessEncoder), edge))) + .collect(Collectors.toList()); + tripsMatchedCount++; + + // For each stop->stop pair in trip, snap stops to map-matched edges and store path between them + boolean needsRoadRouting = false; + for (Pair odStopPair : odStopsForTrip) { + odStopCount++; + // Create String from the ID of each stop in pair, to use as key for map + String stopPairString = odStopPair.getLeft().stop_id + "," + odStopPair.getRight().stop_id; + + // Don't route for any stop->stop pairs we've already routed between + if (gtfsLinkMappings.containsKey(stopPairString)) { + nonUniqueODPairs++; + continue; + } + + EdgeIteratorState snappedOriginEdge = snapEdge(locationIndex, matchedEdgeSet, + stopsForStreetBasedTrips.get(odStopPair.getLeft().stop_id).stop_lat, + stopsForStreetBasedTrips.get(odStopPair.getLeft().stop_id).stop_lon); + EdgeIteratorState snappedDestEdge = snapEdge(locationIndex, matchedEdgeSet, + stopsForStreetBasedTrips.get(odStopPair.getRight().stop_id).stop_lat, + stopsForStreetBasedTrips.get(odStopPair.getRight().stop_id).stop_lon); + + if (snappedOriginEdge == null || snappedDestEdge == null) { + // Ensure we try normal auto routing for this trip, because snapping failed; we may still + // match other stops in this trip via the map-matching approach, but we want to be sure + // that we still record routes between any stops that we couldn't snap successfully + matchedNotSnappedCount++; + needsRoadRouting = true; + } else { + // We can snap each stop to an edge that we matched for this trip; + // so, we walk along the matched edges between the two stops and store the resulting path + List matchedEdgeIds = matchedEdgeSet.stream().map(pair -> pair.getLeft()).collect(Collectors.toList()); + int originIndex = matchedEdgeIds.indexOf(snappedOriginEdge.getEdge()); + int destIndex = matchedEdgeIds.indexOf(snappedDestEdge.getEdge()); + + int firstIndex = Math.min(originIndex, destIndex); + int lastIndex = Math.max(originIndex, destIndex); + boolean reversed = destIndex < originIndex; + + // Form comma-separated string containing stable IDs of all edges along stop->stop path + List pathStableEdgeIds = matchedEdgeSet.subList(firstIndex, lastIndex + 1) + .stream().map(pair -> pair.getRight()).collect(Collectors.toList()); + if (reversed) { + pathStableEdgeIds = Lists.reverse(pathStableEdgeIds); + } + allStableIds.addAll(pathStableEdgeIds); + String pathStableEdgeIdString = pathStableEdgeIds.stream().collect(Collectors.joining(",")); + gtfsLinkMappings.put(stopPairString, pathStableEdgeIdString); + } } - - // Form stop->stop auto routing requests and request a route - GHRequest odRequest = new GHRequest( - odStopPair.getLeft().stop_lat, odStopPair.getLeft().stop_lon, - odStopPair.getRight().stop_lat, odStopPair.getRight().stop_lon - ); - odRequest.setProfile("car"); - odRequest.setPathDetails(Lists.newArrayList("r5_edge_id")); - GHResponse response = graphHopper.route(odRequest); - - // If stop->stop path couldn't be found by GH, don't store anything - if (response.getAll().size() == 0 || response.getAll().get(0).hasErrors()) { - routeNotFoundCount++; - continue; + if (needsRoadRouting) { + // Just meant to trigger below code block; shouldn't cause actual runtime error + throw new RuntimeException("At least one stop couldn't be snapped to map-matched edges!"); + } + } catch (Exception e) { + tripsNotMatchedCount++; + // If map matching fails, route a car between each stop->stop pair, and store the resulting paths + for (Pair odStopPair : odStopsForTrip) { + odStopCount++; + + // Create String from the ID of each stop in pair, to use as key for map + String stopPairString = odStopPair.getLeft().stop_id + "," + odStopPair.getRight().stop_id; + + // Don't route for any stop->stop pairs we've already routed between + if (gtfsLinkMappings.containsKey(stopPairString)) { + nonUniqueODPairs++; + continue; + } + + // Form stop->stop auto routing requests and request a route + GHRequest odRequest = new GHRequest( + odStopPair.getLeft().stop_lat, odStopPair.getLeft().stop_lon, + odStopPair.getRight().stop_lat, odStopPair.getRight().stop_lon + ); + odRequest.setProfile("car"); + odRequest.setPathDetails(Lists.newArrayList("stable_edge_ids")); + GHResponse response = graphHopper.route(odRequest); + + // If stop->stop path couldn't be found by GH, don't store anything + if (response.getAll().size() == 0 || response.getAll().get(0).hasErrors()) { + routeNotFoundCount++; + continue; + } + + // Parse stable IDs for each edge from response + List responsePathEdgeIdDetails = response.getAll().get(0) + .getPathDetails().get("stable_edge_ids"); + List pathEdgeIds = responsePathEdgeIdDetails.stream() + .map(pathDetail -> (String) pathDetail.getValue()) + .collect(Collectors.toList()); + allStableIds.addAll(pathEdgeIds); + + // Merge all path IDs into String to use as value for gtfs link map + String pathStableEdgeIdString = pathEdgeIds.stream().collect(Collectors.joining(",")); + gtfsLinkMappings.put(stopPairString, pathStableEdgeIdString); } - - // Parse stable IDs for each edge from response - List responsePathEdgeIdDetails = response.getAll().get(0).getPathDetails().get("r5_edge_id"); - List pathEdgeIds = responsePathEdgeIdDetails.stream() - .map(pathDetail -> (String) pathDetail.getValue()) - .collect(Collectors.toList()); - // allStableIds.addAll(pathEdgeIds); - - // Merge all path IDs into String to use as value for gtfs link map - String pathStableEdgeIdString = pathEdgeIds.stream().collect(Collectors.joining(",")); - gtfsLinkMappings.put(stopPairString, pathStableEdgeIdString); } - processedTripCount++; } + logger.info("Done processing GTFS feed " + feedId + "; " + tripIdToStopsInTrip.keySet().size() + " total trips processed; " + nonUniqueODPairs + "/" + odStopCount - + " O/D stop pairs were non-unique; routes for " + routeNotFoundCount + "/" + odStopCount - + " stop->stop pairs were not found"); + + " O/D stop pairs were non-unique; " + tripsNotMatchedCount + " trips could not be map-matched" + + " successfully; " + tripsMatchedCount + " trips were map-matched successfully; " + + allStopsAlreadyRoutedCount + " trips already had routes recorded for all stop->stop pairs ; " + + matchedNotSnappedCount + " stops were unsuccessfully snapped to map-matched edges; routes for " + + routeNotFoundCount + " stop->stop pairs were not found via auto routing."); } db.commit(); db.close(); @@ -217,6 +262,34 @@ public void setGtfsLinkMappingsMapMatching() { logger.info("All stable edge IDs: "); logger.info(allStableIds.stream().collect(Collectors.joining(","))); } + + private static EdgeIteratorState snapEdge(LocationIndex locationIndex, + List> edgeFilterSet, + double lat, double lon) { + Set edgeIdFilter = edgeFilterSet.stream().map(pair -> pair.getLeft()).collect(Collectors.toSet()); + EdgeIteratorState toReturn = locationIndex.findClosest(lat, lon, + new EdgeFilter() { + @Override + public boolean accept(EdgeIteratorState edgeIteratorState) { + return edgeIdFilter.contains(edgeIteratorState.getEdge()); + } + }).getClosestEdge(); + if (!edgeIdFilter.contains(toReturn.getEdge())) { // todo: this should never happen (?) if filter is working + return null; + } else { + return toReturn; + } + } + + private static boolean stopsAlreadyRouted(List> odStopsForTrip, Map gtfsLinkMappings) { + for (Pair odStopPair : odStopsForTrip) { + String stopPairString = odStopPair.getLeft().stop_id + "," + odStopPair.getRight().stop_id; + if (!gtfsLinkMappings.containsKey(stopPairString)) { + return false; + } + } + return true; + } public void setGtfsLinkMappings() { logger.info("Starting GTFS link mapping process"); @@ -253,7 +326,7 @@ public void setGtfsLinkMappings() { List gtfsLinkMappingCsvRows = Lists.newArrayList(); // For testing - // Set allStableIds = Sets.newHashSet(); + Set allStableIds = Sets.newHashSet(); // For each GTFS feed, pull out all stops for trips on GTFS routes that travel on the street network, // and then for each trip, route via car between each stop pair in sequential order, storing the returned IDs @@ -360,7 +433,7 @@ public void setGtfsLinkMappings() { List pathEdgeIds = responsePathEdgeIdDetails.stream() .map(pathDetail -> (String) pathDetail.getValue()) .collect(Collectors.toList()); - // allStableIds.addAll(pathEdgeIds); + allStableIds.addAll(pathEdgeIds); // Merge all path IDs into String to use as value for gtfs link map String pathStableEdgeIdString = pathEdgeIds.stream().collect(Collectors.joining(",")); @@ -382,8 +455,8 @@ public void setGtfsLinkMappings() { writeGtfsLinksToCsv(gtfsLinkMappingCsvRows, gtfsLinksCsvOutput); // For testing - // logger.info("All stable edge IDs: "); - // logger.info(allStableIds.stream().collect(Collectors.joining(","))); + logger.info("All stable edge IDs: "); + logger.info(allStableIds.stream().collect(Collectors.joining(","))); } // Given a set of StopTimes for a trip, and an overall mapping of stop IDs->Stop,