diff --git a/replica-common/pom.xml b/replica-common/pom.xml index f37238467a2..e59567a1e3f 100644 --- a/replica-common/pom.xml +++ b/replica-common/pom.xml @@ -21,6 +21,16 @@ graphhopper-matrix 3.0-replica2 + + com.graphhopper + graphhopper-map-matching-core + e164705aa7a6320a29d7760294be8d353b7b21c7 + + + com.graphhopper + graphhopper-map-matching-web-bundle + e164705aa7a6320a29d7760294be8d353b7b21c7 + diff --git a/web-bundle/src/main/java/com/graphhopper/replica/GtfsLinkMapper.java b/web-bundle/src/main/java/com/graphhopper/replica/GtfsLinkMapper.java index 632a06ec43d..b5128230e2f 100644 --- a/web-bundle/src/main/java/com/graphhopper/replica/GtfsLinkMapper.java +++ b/web-bundle/src/main/java/com/graphhopper/replica/GtfsLinkMapper.java @@ -10,16 +10,36 @@ import com.graphhopper.GraphHopper; import com.graphhopper.gtfs.GraphHopperGtfs; import com.graphhopper.gtfs.GtfsStorage; +import com.graphhopper.matching.MapMatching; +import com.graphhopper.matching.MatchResult; +import com.graphhopper.matching.Observation; +import com.graphhopper.routing.ev.BooleanEncodedValue; +import com.graphhopper.routing.util.EdgeFilter; +import com.graphhopper.stableid.StableIdEncodedValues; +import com.graphhopper.storage.index.Location2IDFullWithEdgesIndex; +import com.graphhopper.storage.index.LocationIndex; +import com.graphhopper.util.EdgeIteratorState; +import com.graphhopper.util.PMap; import com.graphhopper.util.details.PathDetail; +import com.graphhopper.util.shapes.GHPoint; import org.apache.commons.lang3.tuple.Pair; -import org.mapdb.*; +import org.locationtech.jts.geom.LineString; +import org.mapdb.DB; +import org.mapdb.DBMaker; +import org.mapdb.HTreeMap; +import org.mapdb.Serializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.*; -import java.util.*; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; +import static com.graphhopper.util.Parameters.Routing.MAX_VISITED_NODES; + public class GtfsLinkMapper { private final Logger logger = LoggerFactory.getLogger(getClass()); private final GraphHopper graphHopper; @@ -30,6 +50,247 @@ public GtfsLinkMapper(GraphHopper graphHopper) { this.graphHopper = graphHopper; } + public void setGtfsLinkMappingsMapMatching() { + GtfsStorage gtfsStorage = ((GraphHopperGtfs) graphHopper).getGtfsStorage(); + Map gtfsFeedMap = gtfsStorage.getGtfsFeeds(); + final Set STREET_BASED_ROUTE_TYPES = Sets.newHashSet(0, 3, 5); + + // Set up map matcher + PMap hints = new PMap(); + hints.putObject("profile", "car"); + hints.putObject(MAX_VISITED_NODES, 10000); + MapMatching matching = new MapMatching(graphHopper, hints); + LocationIndex locationIndex = + new Location2IDFullWithEdgesIndex(graphHopper.getGraphHopperStorage().getBaseGraph()); + + // Initialize mapdb database to store link mappings and route info + logger.info("Initializing new mapdb file to store link mappings"); + DB db = DBMaker.newFileDB(new File("transit_data/gtfs_link_mappings.db")).make(); + HTreeMap gtfsLinkMappings = db + .createHashMap("gtfsLinkMappings") + .keySerializer(Serializer.STRING) + .valueSerializer(Serializer.STRING) + .make(); + + StableIdEncodedValues stableIdEncodedValues = + StableIdEncodedValues.fromEncodingManager(graphHopper.getEncodingManager()); + BooleanEncodedValue carAccessEncoder = graphHopper.getEncodingManager().getEncoder("car").getAccessEnc(); + Set allStableIds = Sets.newHashSet(); + + // For each feed, perform map matching against geo of each trip, and store all matched edges + for (String feedId : gtfsFeedMap.keySet()) { + logger.info("Processing GTFS feed " + feedId + " " + feedId); + GTFSFeed feed = gtfsFeedMap.get(feedId); + + // For mapping purposes, only look at routes for transit that use the street network + Set streetBasedRouteIdsForFeed = feed.routes.values().stream() + .filter(route -> STREET_BASED_ROUTE_TYPES.contains(route.route_type)) + .map(route -> route.route_id) + .collect(Collectors.toSet()); + + // Find all GTFS trips for each route + Set tripsForStreetBasedRoutes = feed.trips.values().stream() + .filter(trip -> streetBasedRouteIdsForFeed.contains(trip.route_id)) + .map(trip -> trip.trip_id) + .collect(Collectors.toSet()); + + // Find all stops for each trip + SetMultimap tripIdToStopsInTrip = HashMultimap.create(); + feed.stop_times.values().stream() + .filter(stopTime -> tripsForStreetBasedRoutes.contains(stopTime.trip_id)) + .forEach(stopTime -> tripIdToStopsInTrip.put(stopTime.trip_id, stopTime)); + + Set stopIdsForStreetBasedTrips = tripIdToStopsInTrip.values().stream() + .map(stopTime -> stopTime.stop_id) + .collect(Collectors.toSet()); + + Map stopsForStreetBasedTrips = feed.stops.values().stream() + .filter(stop -> stopIdsForStreetBasedTrips.contains(stop.stop_id)) + .collect(Collectors.toMap(stop -> stop.stop_id, stop -> stop)); + + int odStopCount = 0; + int nonUniqueODPairs = 0; + int routeNotFoundCount = 0; + int tripsMatchedCount = 0; + int tripsNotMatchedCount = 0; + int matchedNotSnappedCount = 0; + int allStopsAlreadyRoutedCount = 0; + int processedTripCount = 0; + + for (String tripId : tripsForStreetBasedRoutes) { + if (tripIdToStopsInTrip.keySet().size() > 10 && + processedTripCount % (tripIdToStopsInTrip.keySet().size() / 10) == 0) { + logger.info(processedTripCount + "/" + tripIdToStopsInTrip.keySet().size() + " trips for feed " + + feed.feedId + " processed so far; " + nonUniqueODPairs + "/" + odStopCount + + " O/D stop pairs were non-unique, and were not routed between."); + } + processedTripCount++; + + List> odStopsForTrip = + getODStopsForTrip(Sets.newHashSet(feed.getOrderedStopTimesForTrip(tripId)), stopsForStreetBasedTrips); + + // Skip processing if all stop->stop pairs in trip have already been mapped + if (stopsAlreadyRouted(odStopsForTrip, gtfsLinkMappings)) { + allStopsAlreadyRoutedCount++; + continue; + } + try { + // Attempt to map-match trip geometry to street network + LineString tripGeometry = feed.getTripGeometry(tripId); + List pointsToMatch = Arrays.stream(tripGeometry.getCoordinates()) + .map(coordinate -> new GHPoint(coordinate.y, coordinate.x)) + .map(ghPoint -> new Observation(ghPoint)) + .collect(Collectors.toList()); + + // Match points to network and store pair of (GH edge ID, stable edge ID) for all matched edges + MatchResult result = matching.match(pointsToMatch); + List> matchedEdgeSet = result.getMergedPath().calcEdges().stream() + .map(edge -> Pair.of(edge.getEdge(), stableIdEncodedValues.getStableId(edge.getReverse(carAccessEncoder), edge))) + .collect(Collectors.toList()); + tripsMatchedCount++; + + // For each stop->stop pair in trip, snap stops to map-matched edges and store path between them + boolean needsRoadRouting = false; + for (Pair odStopPair : odStopsForTrip) { + odStopCount++; + // Create String from the ID of each stop in pair, to use as key for map + String stopPairString = odStopPair.getLeft().stop_id + "," + odStopPair.getRight().stop_id; + + // Don't route for any stop->stop pairs we've already routed between + if (gtfsLinkMappings.containsKey(stopPairString)) { + nonUniqueODPairs++; + continue; + } + + EdgeIteratorState snappedOriginEdge = snapEdge(locationIndex, matchedEdgeSet, + stopsForStreetBasedTrips.get(odStopPair.getLeft().stop_id).stop_lat, + stopsForStreetBasedTrips.get(odStopPair.getLeft().stop_id).stop_lon); + EdgeIteratorState snappedDestEdge = snapEdge(locationIndex, matchedEdgeSet, + stopsForStreetBasedTrips.get(odStopPair.getRight().stop_id).stop_lat, + stopsForStreetBasedTrips.get(odStopPair.getRight().stop_id).stop_lon); + + if (snappedOriginEdge == null || snappedDestEdge == null) { + // Ensure we try normal auto routing for this trip, because snapping failed; we may still + // match other stops in this trip via the map-matching approach, but we want to be sure + // that we still record routes between any stops that we couldn't snap successfully + matchedNotSnappedCount++; + needsRoadRouting = true; + } else { + // We can snap each stop to an edge that we matched for this trip; + // so, we walk along the matched edges between the two stops and store the resulting path + List matchedEdgeIds = matchedEdgeSet.stream().map(pair -> pair.getLeft()).collect(Collectors.toList()); + int originIndex = matchedEdgeIds.indexOf(snappedOriginEdge.getEdge()); + int destIndex = matchedEdgeIds.indexOf(snappedDestEdge.getEdge()); + + int firstIndex = Math.min(originIndex, destIndex); + int lastIndex = Math.max(originIndex, destIndex); + boolean reversed = destIndex < originIndex; + + // Form comma-separated string containing stable IDs of all edges along stop->stop path + List pathStableEdgeIds = matchedEdgeSet.subList(firstIndex, lastIndex + 1) + .stream().map(pair -> pair.getRight()).collect(Collectors.toList()); + if (reversed) { + pathStableEdgeIds = Lists.reverse(pathStableEdgeIds); + } + allStableIds.addAll(pathStableEdgeIds); + String pathStableEdgeIdString = pathStableEdgeIds.stream().collect(Collectors.joining(",")); + gtfsLinkMappings.put(stopPairString, pathStableEdgeIdString); + } + } + if (needsRoadRouting) { + // Just meant to trigger below code block; shouldn't cause actual runtime error + throw new RuntimeException("At least one stop couldn't be snapped to map-matched edges!"); + } + } catch (Exception e) { + tripsNotMatchedCount++; + // If map matching fails, route a car between each stop->stop pair, and store the resulting paths + for (Pair odStopPair : odStopsForTrip) { + odStopCount++; + + // Create String from the ID of each stop in pair, to use as key for map + String stopPairString = odStopPair.getLeft().stop_id + "," + odStopPair.getRight().stop_id; + + // Don't route for any stop->stop pairs we've already routed between + if (gtfsLinkMappings.containsKey(stopPairString)) { + nonUniqueODPairs++; + continue; + } + + // Form stop->stop auto routing requests and request a route + GHRequest odRequest = new GHRequest( + odStopPair.getLeft().stop_lat, odStopPair.getLeft().stop_lon, + odStopPair.getRight().stop_lat, odStopPair.getRight().stop_lon + ); + odRequest.setProfile("car"); + odRequest.setPathDetails(Lists.newArrayList("stable_edge_ids")); + GHResponse response = graphHopper.route(odRequest); + + // If stop->stop path couldn't be found by GH, don't store anything + if (response.getAll().size() == 0 || response.getAll().get(0).hasErrors()) { + routeNotFoundCount++; + continue; + } + + // Parse stable IDs for each edge from response + List responsePathEdgeIdDetails = response.getAll().get(0) + .getPathDetails().get("stable_edge_ids"); + List pathEdgeIds = responsePathEdgeIdDetails.stream() + .map(pathDetail -> (String) pathDetail.getValue()) + .collect(Collectors.toList()); + allStableIds.addAll(pathEdgeIds); + + // Merge all path IDs into String to use as value for gtfs link map + String pathStableEdgeIdString = pathEdgeIds.stream().collect(Collectors.joining(",")); + gtfsLinkMappings.put(stopPairString, pathStableEdgeIdString); + } + } + } + + logger.info("Done processing GTFS feed " + feedId + "; " + tripIdToStopsInTrip.keySet().size() + + " total trips processed; " + nonUniqueODPairs + "/" + odStopCount + + " O/D stop pairs were non-unique; " + tripsNotMatchedCount + " trips could not be map-matched" + + " successfully; " + tripsMatchedCount + " trips were map-matched successfully; " + + allStopsAlreadyRoutedCount + " trips already had routes recorded for all stop->stop pairs ; " + + matchedNotSnappedCount + " stops were unsuccessfully snapped to map-matched edges; routes for " + + routeNotFoundCount + " stop->stop pairs were not found via auto routing."); + } + db.commit(); + db.close(); + logger.info("Done creating GTFS link mappings for " + gtfsFeedMap.size() + " GTFS feeds"); + + // For testing + logger.info("All stable edge IDs: "); + logger.info(allStableIds.stream().collect(Collectors.joining(","))); + } + + private static EdgeIteratorState snapEdge(LocationIndex locationIndex, + List> edgeFilterSet, + double lat, double lon) { + Set edgeIdFilter = edgeFilterSet.stream().map(pair -> pair.getLeft()).collect(Collectors.toSet()); + EdgeIteratorState toReturn = locationIndex.findClosest(lat, lon, + new EdgeFilter() { + @Override + public boolean accept(EdgeIteratorState edgeIteratorState) { + return edgeIdFilter.contains(edgeIteratorState.getEdge()); + } + }).getClosestEdge(); + if (!edgeIdFilter.contains(toReturn.getEdge())) { // todo: this should never happen (?) if filter is working + return null; + } else { + return toReturn; + } + } + + private static boolean stopsAlreadyRouted(List> odStopsForTrip, Map gtfsLinkMappings) { + for (Pair odStopPair : odStopsForTrip) { + String stopPairString = odStopPair.getLeft().stop_id + "," + odStopPair.getRight().stop_id; + if (!gtfsLinkMappings.containsKey(stopPairString)) { + return false; + } + } + return true; + } + public void setGtfsLinkMappings() { logger.info("Starting GTFS link mapping process"); GtfsStorage gtfsStorage = ((GraphHopperGtfs) graphHopper).getGtfsStorage(); @@ -65,7 +326,7 @@ public void setGtfsLinkMappings() { List gtfsLinkMappingCsvRows = Lists.newArrayList(); // For testing - // Set allStableIds = Sets.newHashSet(); + Set allStableIds = Sets.newHashSet(); // For each GTFS feed, pull out all stops for trips on GTFS routes that travel on the street network, // and then for each trip, route via car between each stop pair in sequential order, storing the returned IDs @@ -172,7 +433,7 @@ public void setGtfsLinkMappings() { List pathEdgeIds = responsePathEdgeIdDetails.stream() .map(pathDetail -> (String) pathDetail.getValue()) .collect(Collectors.toList()); - // allStableIds.addAll(pathEdgeIds); + allStableIds.addAll(pathEdgeIds); // Merge all path IDs into String to use as value for gtfs link map String pathStableEdgeIdString = pathEdgeIds.stream().collect(Collectors.joining(",")); @@ -194,8 +455,8 @@ public void setGtfsLinkMappings() { writeGtfsLinksToCsv(gtfsLinkMappingCsvRows, gtfsLinksCsvOutput); // For testing - // logger.info("All stable edge IDs: "); - // logger.info(allStableIds.stream().collect(Collectors.joining(","))); + logger.info("All stable edge IDs: "); + logger.info(allStableIds.stream().collect(Collectors.joining(","))); } // Given a set of StopTimes for a trip, and an overall mapping of stop IDs->Stop, diff --git a/web/src/main/java/com/graphhopper/http/cli/GtfsLinkMapperCommand.java b/web/src/main/java/com/graphhopper/http/cli/GtfsLinkMapperCommand.java index a36ff771168..69573268521 100644 --- a/web/src/main/java/com/graphhopper/http/cli/GtfsLinkMapperCommand.java +++ b/web/src/main/java/com/graphhopper/http/cli/GtfsLinkMapperCommand.java @@ -20,7 +20,7 @@ protected void run(Bootstrap bootstrap, Namespac GraphHopper gh = graphHopper.getGraphHopper(); gh.load(gh.getGraphHopperLocation()); GtfsLinkMapper gtfsLinkMapper = new GtfsLinkMapper(gh); - gtfsLinkMapper.setGtfsLinkMappings(); + gtfsLinkMapper.setGtfsLinkMappingsMapMatching(); gh.close(); } }