Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Use map matching with GTFS shapes to find routes in GTFS link mapper #37

Draft
wants to merge 5 commits into
base: original-direction
Choose a base branch
from
Draft

Use map matching with GTFS shapes to find routes in GTFS link mapper #37

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
dd3d638
Use map matching with GTFS shapes to find routes in GTFS link mapper
mark-idleman Sep 25, 2020
ccdbb47
cleanup + comments
mark-idleman Sep 25, 2020
f917ff1
update map matching package version to avoid conflicts
mark-idleman Oct 5, 2020
e31040a
merge master
mark-idleman Dec 11, 2020
805d2f3
update code to be hybrid of map matching + routing
mark-idleman Dec 15, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions replica-common/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,16 @@
<artifactId>graphhopper-matrix</artifactId>
<version>3.0-replica2</version>
</dependency>
<dependency>
<groupId>com.graphhopper</groupId>
<artifactId>graphhopper-map-matching-core</artifactId>
<version>e164705aa7a6320a29d7760294be8d353b7b21c7</version>
</dependency>
<dependency>
<groupId>com.graphhopper</groupId>
<artifactId>graphhopper-map-matching-web-bundle</artifactId>
<version>e164705aa7a6320a29d7760294be8d353b7b21c7</version>
</dependency>
</dependencies>

</project>
273 changes: 267 additions & 6 deletions web-bundle/src/main/java/com/graphhopper/replica/GtfsLinkMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,36 @@
import com.graphhopper.GraphHopper;
import com.graphhopper.gtfs.GraphHopperGtfs;
import com.graphhopper.gtfs.GtfsStorage;
import com.graphhopper.matching.MapMatching;
import com.graphhopper.matching.MatchResult;
import com.graphhopper.matching.Observation;
import com.graphhopper.routing.ev.BooleanEncodedValue;
import com.graphhopper.routing.util.EdgeFilter;
import com.graphhopper.stableid.StableIdEncodedValues;
import com.graphhopper.storage.index.Location2IDFullWithEdgesIndex;
import com.graphhopper.storage.index.LocationIndex;
import com.graphhopper.util.EdgeIteratorState;
import com.graphhopper.util.PMap;
import com.graphhopper.util.details.PathDetail;
import com.graphhopper.util.shapes.GHPoint;
import org.apache.commons.lang3.tuple.Pair;
import org.mapdb.*;
import org.locationtech.jts.geom.LineString;
import org.mapdb.DB;
import org.mapdb.DBMaker;
import org.mapdb.HTreeMap;
import org.mapdb.Serializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.*;
import java.util.*;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;

import static com.graphhopper.util.Parameters.Routing.MAX_VISITED_NODES;

public class GtfsLinkMapper {
private final Logger logger = LoggerFactory.getLogger(getClass());
private final GraphHopper graphHopper;
Expand All @@ -30,6 +50,247 @@ public GtfsLinkMapper(GraphHopper graphHopper) {
this.graphHopper = graphHopper;
}

public void setGtfsLinkMappingsMapMatching() {
GtfsStorage gtfsStorage = ((GraphHopperGtfs) graphHopper).getGtfsStorage();
Map<String, GTFSFeed> gtfsFeedMap = gtfsStorage.getGtfsFeeds();
final Set<Integer> STREET_BASED_ROUTE_TYPES = Sets.newHashSet(0, 3, 5);

// Set up map matcher
PMap hints = new PMap();
hints.putObject("profile", "car");
hints.putObject(MAX_VISITED_NODES, 10000);
MapMatching matching = new MapMatching(graphHopper, hints);
LocationIndex locationIndex =
new Location2IDFullWithEdgesIndex(graphHopper.getGraphHopperStorage().getBaseGraph());

// Initialize mapdb database to store link mappings and route info
logger.info("Initializing new mapdb file to store link mappings");
DB db = DBMaker.newFileDB(new File("transit_data/gtfs_link_mappings.db")).make();
HTreeMap<String, String> gtfsLinkMappings = db
.createHashMap("gtfsLinkMappings")
.keySerializer(Serializer.STRING)
.valueSerializer(Serializer.STRING)
.make();

StableIdEncodedValues stableIdEncodedValues =
StableIdEncodedValues.fromEncodingManager(graphHopper.getEncodingManager());
BooleanEncodedValue carAccessEncoder = graphHopper.getEncodingManager().getEncoder("car").getAccessEnc();
Set<String> allStableIds = Sets.newHashSet();

// For each feed, perform map matching against geo of each trip, and store all matched edges
for (String feedId : gtfsFeedMap.keySet()) {
logger.info("Processing GTFS feed " + feedId + " " + feedId);
GTFSFeed feed = gtfsFeedMap.get(feedId);

// For mapping purposes, only look at routes for transit that use the street network
Set<String> streetBasedRouteIdsForFeed = feed.routes.values().stream()
.filter(route -> STREET_BASED_ROUTE_TYPES.contains(route.route_type))
.map(route -> route.route_id)
.collect(Collectors.toSet());

// Find all GTFS trips for each route
Set<String> tripsForStreetBasedRoutes = feed.trips.values().stream()
.filter(trip -> streetBasedRouteIdsForFeed.contains(trip.route_id))
.map(trip -> trip.trip_id)
.collect(Collectors.toSet());

// Find all stops for each trip
SetMultimap<String, StopTime> tripIdToStopsInTrip = HashMultimap.create();
feed.stop_times.values().stream()
.filter(stopTime -> tripsForStreetBasedRoutes.contains(stopTime.trip_id))
.forEach(stopTime -> tripIdToStopsInTrip.put(stopTime.trip_id, stopTime));

Set<String> stopIdsForStreetBasedTrips = tripIdToStopsInTrip.values().stream()
.map(stopTime -> stopTime.stop_id)
.collect(Collectors.toSet());

Map<String, Stop> stopsForStreetBasedTrips = feed.stops.values().stream()
.filter(stop -> stopIdsForStreetBasedTrips.contains(stop.stop_id))
.collect(Collectors.toMap(stop -> stop.stop_id, stop -> stop));

int odStopCount = 0;
int nonUniqueODPairs = 0;
int routeNotFoundCount = 0;
int tripsMatchedCount = 0;
int tripsNotMatchedCount = 0;
int matchedNotSnappedCount = 0;
int allStopsAlreadyRoutedCount = 0;
int processedTripCount = 0;

for (String tripId : tripsForStreetBasedRoutes) {
if (tripIdToStopsInTrip.keySet().size() > 10 &&
processedTripCount % (tripIdToStopsInTrip.keySet().size() / 10) == 0) {
logger.info(processedTripCount + "/" + tripIdToStopsInTrip.keySet().size() + " trips for feed "
+ feed.feedId + " processed so far; " + nonUniqueODPairs + "/" + odStopCount
+ " O/D stop pairs were non-unique, and were not routed between.");
}
processedTripCount++;

List<Pair<Stop, Stop>> odStopsForTrip =
getODStopsForTrip(Sets.newHashSet(feed.getOrderedStopTimesForTrip(tripId)), stopsForStreetBasedTrips);

// Skip processing if all stop->stop pairs in trip have already been mapped
if (stopsAlreadyRouted(odStopsForTrip, gtfsLinkMappings)) {
allStopsAlreadyRoutedCount++;
continue;
}
try {
// Attempt to map-match trip geometry to street network
LineString tripGeometry = feed.getTripGeometry(tripId);
List<Observation> pointsToMatch = Arrays.stream(tripGeometry.getCoordinates())
.map(coordinate -> new GHPoint(coordinate.y, coordinate.x))
.map(ghPoint -> new Observation(ghPoint))
.collect(Collectors.toList());

// Match points to network and store pair of (GH edge ID, stable edge ID) for all matched edges
MatchResult result = matching.match(pointsToMatch);
List<Pair<Integer, String>> matchedEdgeSet = result.getMergedPath().calcEdges().stream()
.map(edge -> Pair.of(edge.getEdge(), stableIdEncodedValues.getStableId(edge.getReverse(carAccessEncoder), edge)))
.collect(Collectors.toList());
tripsMatchedCount++;

// For each stop->stop pair in trip, snap stops to map-matched edges and store path between them
boolean needsRoadRouting = false;
for (Pair<Stop, Stop> odStopPair : odStopsForTrip) {
odStopCount++;
// Create String from the ID of each stop in pair, to use as key for map
String stopPairString = odStopPair.getLeft().stop_id + "," + odStopPair.getRight().stop_id;

// Don't route for any stop->stop pairs we've already routed between
if (gtfsLinkMappings.containsKey(stopPairString)) {
nonUniqueODPairs++;
continue;
}

EdgeIteratorState snappedOriginEdge = snapEdge(locationIndex, matchedEdgeSet,
stopsForStreetBasedTrips.get(odStopPair.getLeft().stop_id).stop_lat,
stopsForStreetBasedTrips.get(odStopPair.getLeft().stop_id).stop_lon);
EdgeIteratorState snappedDestEdge = snapEdge(locationIndex, matchedEdgeSet,
stopsForStreetBasedTrips.get(odStopPair.getRight().stop_id).stop_lat,
stopsForStreetBasedTrips.get(odStopPair.getRight().stop_id).stop_lon);

if (snappedOriginEdge == null || snappedDestEdge == null) {
// Ensure we try normal auto routing for this trip, because snapping failed; we may still
// match other stops in this trip via the map-matching approach, but we want to be sure
// that we still record routes between any stops that we couldn't snap successfully
matchedNotSnappedCount++;
needsRoadRouting = true;
} else {
// We can snap each stop to an edge that we matched for this trip;
// so, we walk along the matched edges between the two stops and store the resulting path
List<Integer> matchedEdgeIds = matchedEdgeSet.stream().map(pair -> pair.getLeft()).collect(Collectors.toList());
int originIndex = matchedEdgeIds.indexOf(snappedOriginEdge.getEdge());
int destIndex = matchedEdgeIds.indexOf(snappedDestEdge.getEdge());

int firstIndex = Math.min(originIndex, destIndex);
int lastIndex = Math.max(originIndex, destIndex);
boolean reversed = destIndex < originIndex;

// Form comma-separated string containing stable IDs of all edges along stop->stop path
List<String> pathStableEdgeIds = matchedEdgeSet.subList(firstIndex, lastIndex + 1)
.stream().map(pair -> pair.getRight()).collect(Collectors.toList());
if (reversed) {
pathStableEdgeIds = Lists.reverse(pathStableEdgeIds);
}
allStableIds.addAll(pathStableEdgeIds);
String pathStableEdgeIdString = pathStableEdgeIds.stream().collect(Collectors.joining(","));
gtfsLinkMappings.put(stopPairString, pathStableEdgeIdString);
}
}
if (needsRoadRouting) {
// Just meant to trigger below code block; shouldn't cause actual runtime error
throw new RuntimeException("At least one stop couldn't be snapped to map-matched edges!");
}
} catch (Exception e) {
tripsNotMatchedCount++;
// If map matching fails, route a car between each stop->stop pair, and store the resulting paths
for (Pair<Stop, Stop> odStopPair : odStopsForTrip) {
odStopCount++;

// Create String from the ID of each stop in pair, to use as key for map
String stopPairString = odStopPair.getLeft().stop_id + "," + odStopPair.getRight().stop_id;

// Don't route for any stop->stop pairs we've already routed between
if (gtfsLinkMappings.containsKey(stopPairString)) {
nonUniqueODPairs++;
continue;
}

// Form stop->stop auto routing requests and request a route
GHRequest odRequest = new GHRequest(
odStopPair.getLeft().stop_lat, odStopPair.getLeft().stop_lon,
odStopPair.getRight().stop_lat, odStopPair.getRight().stop_lon
);
odRequest.setProfile("car");
odRequest.setPathDetails(Lists.newArrayList("stable_edge_ids"));
GHResponse response = graphHopper.route(odRequest);

// If stop->stop path couldn't be found by GH, don't store anything
if (response.getAll().size() == 0 || response.getAll().get(0).hasErrors()) {
routeNotFoundCount++;
continue;
}

// Parse stable IDs for each edge from response
List<PathDetail> responsePathEdgeIdDetails = response.getAll().get(0)
.getPathDetails().get("stable_edge_ids");
List<String> pathEdgeIds = responsePathEdgeIdDetails.stream()
.map(pathDetail -> (String) pathDetail.getValue())
.collect(Collectors.toList());
allStableIds.addAll(pathEdgeIds);

// Merge all path IDs into String to use as value for gtfs link map
String pathStableEdgeIdString = pathEdgeIds.stream().collect(Collectors.joining(","));
gtfsLinkMappings.put(stopPairString, pathStableEdgeIdString);
}
}
}

logger.info("Done processing GTFS feed " + feedId + "; " + tripIdToStopsInTrip.keySet().size() +
" total trips processed; " + nonUniqueODPairs + "/" + odStopCount
+ " O/D stop pairs were non-unique; " + tripsNotMatchedCount + " trips could not be map-matched" +
" successfully; " + tripsMatchedCount + " trips were map-matched successfully; " +
allStopsAlreadyRoutedCount + " trips already had routes recorded for all stop->stop pairs ; " +
matchedNotSnappedCount + " stops were unsuccessfully snapped to map-matched edges; routes for " +
routeNotFoundCount + " stop->stop pairs were not found via auto routing.");
}
db.commit();
db.close();
logger.info("Done creating GTFS link mappings for " + gtfsFeedMap.size() + " GTFS feeds");

// For testing
logger.info("All stable edge IDs: ");
logger.info(allStableIds.stream().collect(Collectors.joining(",")));
}

private static EdgeIteratorState snapEdge(LocationIndex locationIndex,
List<Pair<Integer, String>> edgeFilterSet,
double lat, double lon) {
Set<Integer> edgeIdFilter = edgeFilterSet.stream().map(pair -> pair.getLeft()).collect(Collectors.toSet());
EdgeIteratorState toReturn = locationIndex.findClosest(lat, lon,
new EdgeFilter() {
@Override
public boolean accept(EdgeIteratorState edgeIteratorState) {
return edgeIdFilter.contains(edgeIteratorState.getEdge());
}
}).getClosestEdge();
if (!edgeIdFilter.contains(toReturn.getEdge())) { // todo: this should never happen (?) if filter is working
return null;
} else {
return toReturn;
}
}

private static boolean stopsAlreadyRouted(List<Pair<Stop, Stop>> odStopsForTrip, Map<String, String> gtfsLinkMappings) {
for (Pair<Stop, Stop> odStopPair : odStopsForTrip) {
String stopPairString = odStopPair.getLeft().stop_id + "," + odStopPair.getRight().stop_id;
if (!gtfsLinkMappings.containsKey(stopPairString)) {
return false;
}
}
return true;
}

public void setGtfsLinkMappings() {
logger.info("Starting GTFS link mapping process");
GtfsStorage gtfsStorage = ((GraphHopperGtfs) graphHopper).getGtfsStorage();
Expand Down Expand Up @@ -65,7 +326,7 @@ public void setGtfsLinkMappings() {
List<String> gtfsLinkMappingCsvRows = Lists.newArrayList();

// For testing
// Set<String> allStableIds = Sets.newHashSet();
Set<String> allStableIds = Sets.newHashSet();

// For each GTFS feed, pull out all stops for trips on GTFS routes that travel on the street network,
// and then for each trip, route via car between each stop pair in sequential order, storing the returned IDs
Expand Down Expand Up @@ -172,7 +433,7 @@ public void setGtfsLinkMappings() {
List<String> pathEdgeIds = responsePathEdgeIdDetails.stream()
.map(pathDetail -> (String) pathDetail.getValue())
.collect(Collectors.toList());
// allStableIds.addAll(pathEdgeIds);
allStableIds.addAll(pathEdgeIds);

// Merge all path IDs into String to use as value for gtfs link map
String pathStableEdgeIdString = pathEdgeIds.stream().collect(Collectors.joining(","));
Expand All @@ -194,8 +455,8 @@ public void setGtfsLinkMappings() {
writeGtfsLinksToCsv(gtfsLinkMappingCsvRows, gtfsLinksCsvOutput);

// For testing
// logger.info("All stable edge IDs: ");
// logger.info(allStableIds.stream().collect(Collectors.joining(",")));
logger.info("All stable edge IDs: ");
logger.info(allStableIds.stream().collect(Collectors.joining(",")));
}

// Given a set of StopTimes for a trip, and an overall mapping of stop IDs->Stop,
Expand Down
2 changes: 1 addition & 1 deletion web/src/main/java/com/graphhopper/http/cli/GtfsLinkMapperCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ protected void run(Bootstrap<GraphHopperServerConfiguration> bootstrap, Namespac
GraphHopper gh = graphHopper.getGraphHopper();
gh.load(gh.getGraphHopperLocation());
GtfsLinkMapper gtfsLinkMapper = new GtfsLinkMapper(gh);
gtfsLinkMapper.setGtfsLinkMappings();
gtfsLinkMapper.setGtfsLinkMappingsMapMatching();
gh.close();
}
}