diff --git a/replica-common/pom.xml b/replica-common/pom.xml
index f37238467a2..e59567a1e3f 100644
--- a/replica-common/pom.xml
+++ b/replica-common/pom.xml
@@ -21,6 +21,16 @@
graphhopper-matrix
3.0-replica2
+
+ com.graphhopper
+ graphhopper-map-matching-core
+ e164705aa7a6320a29d7760294be8d353b7b21c7
+
+
+ com.graphhopper
+ graphhopper-map-matching-web-bundle
+ e164705aa7a6320a29d7760294be8d353b7b21c7
+
diff --git a/web-bundle/src/main/java/com/graphhopper/replica/GtfsLinkMapper.java b/web-bundle/src/main/java/com/graphhopper/replica/GtfsLinkMapper.java
index 632a06ec43d..b5128230e2f 100644
--- a/web-bundle/src/main/java/com/graphhopper/replica/GtfsLinkMapper.java
+++ b/web-bundle/src/main/java/com/graphhopper/replica/GtfsLinkMapper.java
@@ -10,16 +10,36 @@
import com.graphhopper.GraphHopper;
import com.graphhopper.gtfs.GraphHopperGtfs;
import com.graphhopper.gtfs.GtfsStorage;
+import com.graphhopper.matching.MapMatching;
+import com.graphhopper.matching.MatchResult;
+import com.graphhopper.matching.Observation;
+import com.graphhopper.routing.ev.BooleanEncodedValue;
+import com.graphhopper.routing.util.EdgeFilter;
+import com.graphhopper.stableid.StableIdEncodedValues;
+import com.graphhopper.storage.index.Location2IDFullWithEdgesIndex;
+import com.graphhopper.storage.index.LocationIndex;
+import com.graphhopper.util.EdgeIteratorState;
+import com.graphhopper.util.PMap;
import com.graphhopper.util.details.PathDetail;
+import com.graphhopper.util.shapes.GHPoint;
import org.apache.commons.lang3.tuple.Pair;
-import org.mapdb.*;
+import org.locationtech.jts.geom.LineString;
+import org.mapdb.DB;
+import org.mapdb.DBMaker;
+import org.mapdb.HTreeMap;
+import org.mapdb.Serializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
-import java.util.*;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
import java.util.stream.Collectors;
+import static com.graphhopper.util.Parameters.Routing.MAX_VISITED_NODES;
+
public class GtfsLinkMapper {
private final Logger logger = LoggerFactory.getLogger(getClass());
private final GraphHopper graphHopper;
@@ -30,6 +50,247 @@ public GtfsLinkMapper(GraphHopper graphHopper) {
this.graphHopper = graphHopper;
}
+ public void setGtfsLinkMappingsMapMatching() {
+ GtfsStorage gtfsStorage = ((GraphHopperGtfs) graphHopper).getGtfsStorage();
+ Map gtfsFeedMap = gtfsStorage.getGtfsFeeds();
+ final Set STREET_BASED_ROUTE_TYPES = Sets.newHashSet(0, 3, 5);
+
+ // Set up map matcher
+ PMap hints = new PMap();
+ hints.putObject("profile", "car");
+ hints.putObject(MAX_VISITED_NODES, 10000);
+ MapMatching matching = new MapMatching(graphHopper, hints);
+ LocationIndex locationIndex =
+ new Location2IDFullWithEdgesIndex(graphHopper.getGraphHopperStorage().getBaseGraph());
+
+ // Initialize mapdb database to store link mappings and route info
+ logger.info("Initializing new mapdb file to store link mappings");
+ DB db = DBMaker.newFileDB(new File("transit_data/gtfs_link_mappings.db")).make();
+ HTreeMap gtfsLinkMappings = db
+ .createHashMap("gtfsLinkMappings")
+ .keySerializer(Serializer.STRING)
+ .valueSerializer(Serializer.STRING)
+ .make();
+
+ StableIdEncodedValues stableIdEncodedValues =
+ StableIdEncodedValues.fromEncodingManager(graphHopper.getEncodingManager());
+ BooleanEncodedValue carAccessEncoder = graphHopper.getEncodingManager().getEncoder("car").getAccessEnc();
+ Set allStableIds = Sets.newHashSet();
+
+ // For each feed, perform map matching against geo of each trip, and store all matched edges
+ for (String feedId : gtfsFeedMap.keySet()) {
+ logger.info("Processing GTFS feed " + feedId + " " + feedId);
+ GTFSFeed feed = gtfsFeedMap.get(feedId);
+
+ // For mapping purposes, only look at routes for transit that use the street network
+ Set streetBasedRouteIdsForFeed = feed.routes.values().stream()
+ .filter(route -> STREET_BASED_ROUTE_TYPES.contains(route.route_type))
+ .map(route -> route.route_id)
+ .collect(Collectors.toSet());
+
+ // Find all GTFS trips for each route
+ Set tripsForStreetBasedRoutes = feed.trips.values().stream()
+ .filter(trip -> streetBasedRouteIdsForFeed.contains(trip.route_id))
+ .map(trip -> trip.trip_id)
+ .collect(Collectors.toSet());
+
+ // Find all stops for each trip
+ SetMultimap tripIdToStopsInTrip = HashMultimap.create();
+ feed.stop_times.values().stream()
+ .filter(stopTime -> tripsForStreetBasedRoutes.contains(stopTime.trip_id))
+ .forEach(stopTime -> tripIdToStopsInTrip.put(stopTime.trip_id, stopTime));
+
+ Set stopIdsForStreetBasedTrips = tripIdToStopsInTrip.values().stream()
+ .map(stopTime -> stopTime.stop_id)
+ .collect(Collectors.toSet());
+
+ Map stopsForStreetBasedTrips = feed.stops.values().stream()
+ .filter(stop -> stopIdsForStreetBasedTrips.contains(stop.stop_id))
+ .collect(Collectors.toMap(stop -> stop.stop_id, stop -> stop));
+
+ int odStopCount = 0;
+ int nonUniqueODPairs = 0;
+ int routeNotFoundCount = 0;
+ int tripsMatchedCount = 0;
+ int tripsNotMatchedCount = 0;
+ int matchedNotSnappedCount = 0;
+ int allStopsAlreadyRoutedCount = 0;
+ int processedTripCount = 0;
+
+ for (String tripId : tripsForStreetBasedRoutes) {
+ if (tripIdToStopsInTrip.keySet().size() > 10 &&
+ processedTripCount % (tripIdToStopsInTrip.keySet().size() / 10) == 0) {
+ logger.info(processedTripCount + "/" + tripIdToStopsInTrip.keySet().size() + " trips for feed "
+ + feed.feedId + " processed so far; " + nonUniqueODPairs + "/" + odStopCount
+ + " O/D stop pairs were non-unique, and were not routed between.");
+ }
+ processedTripCount++;
+
+ List> odStopsForTrip =
+ getODStopsForTrip(Sets.newHashSet(feed.getOrderedStopTimesForTrip(tripId)), stopsForStreetBasedTrips);
+
+ // Skip processing if all stop->stop pairs in trip have already been mapped
+ if (stopsAlreadyRouted(odStopsForTrip, gtfsLinkMappings)) {
+ allStopsAlreadyRoutedCount++;
+ continue;
+ }
+ try {
+ // Attempt to map-match trip geometry to street network
+ LineString tripGeometry = feed.getTripGeometry(tripId);
+ List pointsToMatch = Arrays.stream(tripGeometry.getCoordinates())
+ .map(coordinate -> new GHPoint(coordinate.y, coordinate.x))
+ .map(ghPoint -> new Observation(ghPoint))
+ .collect(Collectors.toList());
+
+ // Match points to network and store pair of (GH edge ID, stable edge ID) for all matched edges
+ MatchResult result = matching.match(pointsToMatch);
+ List> matchedEdgeSet = result.getMergedPath().calcEdges().stream()
+ .map(edge -> Pair.of(edge.getEdge(), stableIdEncodedValues.getStableId(edge.getReverse(carAccessEncoder), edge)))
+ .collect(Collectors.toList());
+ tripsMatchedCount++;
+
+ // For each stop->stop pair in trip, snap stops to map-matched edges and store path between them
+ boolean needsRoadRouting = false;
+ for (Pair odStopPair : odStopsForTrip) {
+ odStopCount++;
+ // Create String from the ID of each stop in pair, to use as key for map
+ String stopPairString = odStopPair.getLeft().stop_id + "," + odStopPair.getRight().stop_id;
+
+ // Don't route for any stop->stop pairs we've already routed between
+ if (gtfsLinkMappings.containsKey(stopPairString)) {
+ nonUniqueODPairs++;
+ continue;
+ }
+
+ EdgeIteratorState snappedOriginEdge = snapEdge(locationIndex, matchedEdgeSet,
+ stopsForStreetBasedTrips.get(odStopPair.getLeft().stop_id).stop_lat,
+ stopsForStreetBasedTrips.get(odStopPair.getLeft().stop_id).stop_lon);
+ EdgeIteratorState snappedDestEdge = snapEdge(locationIndex, matchedEdgeSet,
+ stopsForStreetBasedTrips.get(odStopPair.getRight().stop_id).stop_lat,
+ stopsForStreetBasedTrips.get(odStopPair.getRight().stop_id).stop_lon);
+
+ if (snappedOriginEdge == null || snappedDestEdge == null) {
+ // Ensure we try normal auto routing for this trip, because snapping failed; we may still
+ // match other stops in this trip via the map-matching approach, but we want to be sure
+ // that we still record routes between any stops that we couldn't snap successfully
+ matchedNotSnappedCount++;
+ needsRoadRouting = true;
+ } else {
+ // We can snap each stop to an edge that we matched for this trip;
+ // so, we walk along the matched edges between the two stops and store the resulting path
+ List matchedEdgeIds = matchedEdgeSet.stream().map(pair -> pair.getLeft()).collect(Collectors.toList());
+ int originIndex = matchedEdgeIds.indexOf(snappedOriginEdge.getEdge());
+ int destIndex = matchedEdgeIds.indexOf(snappedDestEdge.getEdge());
+
+ int firstIndex = Math.min(originIndex, destIndex);
+ int lastIndex = Math.max(originIndex, destIndex);
+ boolean reversed = destIndex < originIndex;
+
+ // Form comma-separated string containing stable IDs of all edges along stop->stop path
+ List pathStableEdgeIds = matchedEdgeSet.subList(firstIndex, lastIndex + 1)
+ .stream().map(pair -> pair.getRight()).collect(Collectors.toList());
+ if (reversed) {
+ pathStableEdgeIds = Lists.reverse(pathStableEdgeIds);
+ }
+ allStableIds.addAll(pathStableEdgeIds);
+ String pathStableEdgeIdString = pathStableEdgeIds.stream().collect(Collectors.joining(","));
+ gtfsLinkMappings.put(stopPairString, pathStableEdgeIdString);
+ }
+ }
+ if (needsRoadRouting) {
+ // Just meant to trigger below code block; shouldn't cause actual runtime error
+ throw new RuntimeException("At least one stop couldn't be snapped to map-matched edges!");
+ }
+ } catch (Exception e) {
+ tripsNotMatchedCount++;
+ // If map matching fails, route a car between each stop->stop pair, and store the resulting paths
+ for (Pair odStopPair : odStopsForTrip) {
+ odStopCount++;
+
+ // Create String from the ID of each stop in pair, to use as key for map
+ String stopPairString = odStopPair.getLeft().stop_id + "," + odStopPair.getRight().stop_id;
+
+ // Don't route for any stop->stop pairs we've already routed between
+ if (gtfsLinkMappings.containsKey(stopPairString)) {
+ nonUniqueODPairs++;
+ continue;
+ }
+
+ // Form stop->stop auto routing requests and request a route
+ GHRequest odRequest = new GHRequest(
+ odStopPair.getLeft().stop_lat, odStopPair.getLeft().stop_lon,
+ odStopPair.getRight().stop_lat, odStopPair.getRight().stop_lon
+ );
+ odRequest.setProfile("car");
+ odRequest.setPathDetails(Lists.newArrayList("stable_edge_ids"));
+ GHResponse response = graphHopper.route(odRequest);
+
+ // If stop->stop path couldn't be found by GH, don't store anything
+ if (response.getAll().size() == 0 || response.getAll().get(0).hasErrors()) {
+ routeNotFoundCount++;
+ continue;
+ }
+
+ // Parse stable IDs for each edge from response
+ List responsePathEdgeIdDetails = response.getAll().get(0)
+ .getPathDetails().get("stable_edge_ids");
+ List pathEdgeIds = responsePathEdgeIdDetails.stream()
+ .map(pathDetail -> (String) pathDetail.getValue())
+ .collect(Collectors.toList());
+ allStableIds.addAll(pathEdgeIds);
+
+ // Merge all path IDs into String to use as value for gtfs link map
+ String pathStableEdgeIdString = pathEdgeIds.stream().collect(Collectors.joining(","));
+ gtfsLinkMappings.put(stopPairString, pathStableEdgeIdString);
+ }
+ }
+ }
+
+ logger.info("Done processing GTFS feed " + feedId + "; " + tripIdToStopsInTrip.keySet().size() +
+ " total trips processed; " + nonUniqueODPairs + "/" + odStopCount
+ + " O/D stop pairs were non-unique; " + tripsNotMatchedCount + " trips could not be map-matched" +
+ " successfully; " + tripsMatchedCount + " trips were map-matched successfully; " +
+ allStopsAlreadyRoutedCount + " trips already had routes recorded for all stop->stop pairs ; " +
+ matchedNotSnappedCount + " stops were unsuccessfully snapped to map-matched edges; routes for " +
+ routeNotFoundCount + " stop->stop pairs were not found via auto routing.");
+ }
+ db.commit();
+ db.close();
+ logger.info("Done creating GTFS link mappings for " + gtfsFeedMap.size() + " GTFS feeds");
+
+ // For testing
+ logger.info("All stable edge IDs: ");
+ logger.info(allStableIds.stream().collect(Collectors.joining(",")));
+ }
+
+ private static EdgeIteratorState snapEdge(LocationIndex locationIndex,
+ List> edgeFilterSet,
+ double lat, double lon) {
+ Set edgeIdFilter = edgeFilterSet.stream().map(pair -> pair.getLeft()).collect(Collectors.toSet());
+ EdgeIteratorState toReturn = locationIndex.findClosest(lat, lon,
+ new EdgeFilter() {
+ @Override
+ public boolean accept(EdgeIteratorState edgeIteratorState) {
+ return edgeIdFilter.contains(edgeIteratorState.getEdge());
+ }
+ }).getClosestEdge();
+ if (!edgeIdFilter.contains(toReturn.getEdge())) { // todo: this should never happen (?) if filter is working
+ return null;
+ } else {
+ return toReturn;
+ }
+ }
+
+ private static boolean stopsAlreadyRouted(List> odStopsForTrip, Map gtfsLinkMappings) {
+ for (Pair odStopPair : odStopsForTrip) {
+ String stopPairString = odStopPair.getLeft().stop_id + "," + odStopPair.getRight().stop_id;
+ if (!gtfsLinkMappings.containsKey(stopPairString)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
public void setGtfsLinkMappings() {
logger.info("Starting GTFS link mapping process");
GtfsStorage gtfsStorage = ((GraphHopperGtfs) graphHopper).getGtfsStorage();
@@ -65,7 +326,7 @@ public void setGtfsLinkMappings() {
List gtfsLinkMappingCsvRows = Lists.newArrayList();
// For testing
- // Set allStableIds = Sets.newHashSet();
+ Set allStableIds = Sets.newHashSet();
// For each GTFS feed, pull out all stops for trips on GTFS routes that travel on the street network,
// and then for each trip, route via car between each stop pair in sequential order, storing the returned IDs
@@ -172,7 +433,7 @@ public void setGtfsLinkMappings() {
List pathEdgeIds = responsePathEdgeIdDetails.stream()
.map(pathDetail -> (String) pathDetail.getValue())
.collect(Collectors.toList());
- // allStableIds.addAll(pathEdgeIds);
+ allStableIds.addAll(pathEdgeIds);
// Merge all path IDs into String to use as value for gtfs link map
String pathStableEdgeIdString = pathEdgeIds.stream().collect(Collectors.joining(","));
@@ -194,8 +455,8 @@ public void setGtfsLinkMappings() {
writeGtfsLinksToCsv(gtfsLinkMappingCsvRows, gtfsLinksCsvOutput);
// For testing
- // logger.info("All stable edge IDs: ");
- // logger.info(allStableIds.stream().collect(Collectors.joining(",")));
+ logger.info("All stable edge IDs: ");
+ logger.info(allStableIds.stream().collect(Collectors.joining(",")));
}
// Given a set of StopTimes for a trip, and an overall mapping of stop IDs->Stop,
diff --git a/web/src/main/java/com/graphhopper/http/cli/GtfsLinkMapperCommand.java b/web/src/main/java/com/graphhopper/http/cli/GtfsLinkMapperCommand.java
index a36ff771168..69573268521 100644
--- a/web/src/main/java/com/graphhopper/http/cli/GtfsLinkMapperCommand.java
+++ b/web/src/main/java/com/graphhopper/http/cli/GtfsLinkMapperCommand.java
@@ -20,7 +20,7 @@ protected void run(Bootstrap bootstrap, Namespac
GraphHopper gh = graphHopper.getGraphHopper();
gh.load(gh.getGraphHopperLocation());
GtfsLinkMapper gtfsLinkMapper = new GtfsLinkMapper(gh);
- gtfsLinkMapper.setGtfsLinkMappings();
+ gtfsLinkMapper.setGtfsLinkMappingsMapMatching();
gh.close();
}
}