Examples with ExternalServices org.onebusaway.cloud.api.ExternalServices used on opensource projects

Example 11 with ExternalServices

use of org.onebusaway.cloud.api.ExternalServices in project onebusaway-gtfs-modules by OneBusAway.

the class ExtrapolateRidershipData method run.

@Override
public void run(TransformContext context, GtfsMutableRelationalDao dao) {
    File controlFile = new File((String) context.getParameter("controlFile"));
    String feed = CloudContextService.getLikelyFeedName(dao);
    ExternalServices es = new ExternalServicesBridgeFactory().getExternalServices();
    if (!controlFile.exists()) {
        es.publishMultiDimensionalMetric(CloudContextService.getNamespace(), "MissingControlFiles", new String[] { "feed", "controlFileName" }, new String[] { feed, controlFile.getName() }, 1);
        throw new IllegalStateException("Control file does not exist: " + controlFile.getName());
    }
    AgencyAndId agencyAndId = dao.getAllTrips().iterator().next().getId();
    List<String> controlLines = new InputLibrary().readList((String) context.getParameter("controlFile"));
    // String is the TripId, ridershipData is the data from the Access DB where it is a list of all the entries for that TripId, it should
    // correspond to a list of the stops (and ridership data) for that trip
    HashMap<String, ArrayList<RidershipData>> ridershipMap = new HashMap<>();
    for (String controlLine : controlLines) {
        String[] controlArray = controlLine.split(",");
        if (controlArray == null || controlArray.length < 2) {
            _log.info("bad control line {}", controlLine);
            continue;
        }
        String routeId = controlArray[ROUTE_ID];
        String tripId = controlArray[TRIP_ID];
        String stopId = controlArray[STOP_ID];
        String boardings = controlArray[AVG_ON];
        String alightings = controlArray[AVG_OFF];
        String avgLoad = controlArray[AVG_LOAD];
        String stopSequence = controlArray[STOP_SEQ];
        String passTime = controlArray[PASS_TIME];
        RidershipData ridership = new RidershipData();
        // why is agency and id separate for ridership where all others in model are AgencyAndId ?
        ridership.setAgencyId(agencyAndId.getAgencyId());
        ridership.setRouteId(routeId);
        ridership.setTripId(tripId);
        ridership.setRsTripId(tripId);
        ridership.setStopId(stopId);
        ridership.setPassingTime(convertToTime(passTime));
        try {
            ridership.setAverageLoad(Double.valueOf(avgLoad));
        } catch (NumberFormatException e) {
            _log.error("NFE on avgLoad {}", controlLine);
            continue;
        }
        try {
            Double board = (Double.valueOf(boardings));
            ridership.setTotalBoardings(board.intValue());
        } catch (NumberFormatException e) {
            _log.error("NFE on boardings {}", controlLine);
            continue;
        }
        try {
            Double alight = (Double.valueOf(alightings));
            ridership.setTotalAlightings(alight.intValue());
        } catch (NumberFormatException e) {
            _log.error("NFE on alightings {}", controlLine);
            continue;
        }
        try {
            Double stopSequence1 = (Double.valueOf(stopSequence));
            ridership.setStopSequence(stopSequence1.intValue());
        } catch (NumberFormatException e) {
            _log.error("NFE on stop sequence {}", controlLine);
            continue;
        }
        if (ridershipMap.containsKey(ridership.getTripId())) {
            ridershipMap.get(ridership.getTripId()).add(ridership);
        } else {
            ArrayList<RidershipData> riderships = new ArrayList<>();
            riderships.add(ridership);
            ridershipMap.put(ridership.getTripId(), riderships);
        }
    }
    _log.error("Number of trips in ridership data {}", ridershipMap.size());
    // String is the GTFS AgencyAndId TripId, treeMap is a map of StopSequence, StopId
    HashMap<AgencyAndId, TreeMap<Integer, String>> daoTrips = new HashMap<>();
    // String is the Access DB TripId, treeMap is a map of StopSequence, StopId
    HashMap<String, TreeMap<Integer, String>> ridershipTrips = new HashMap<>();
    for (Trip trip : dao.getAllTrips()) {
        TreeMap<Integer, String> daoStops = new TreeMap<>();
        for (StopTime stoptime : dao.getStopTimesForTrip(trip)) {
            daoStops.put(stoptime.getStopSequence(), stoptime.getStop().getId().getId());
        }
        daoTrips.put(trip.getId(), daoStops);
    }
    for (HashMap.Entry<String, ArrayList<RidershipData>> ridershipEntry : ridershipMap.entrySet()) {
        TreeMap<Integer, String> ridershipStops = new TreeMap<>();
        ArrayList<RidershipData> ridershipList = (ArrayList<RidershipData>) ridershipEntry.getValue();
        for (RidershipData ridership : ridershipList) {
            ridershipStops.put(ridership.getStopSequence(), ridership.getStopId());
        }
        ridershipTrips.put(ridershipEntry.getKey().toString(), ridershipStops);
    }
    _log.error("Number of trips in GTFS: {}", daoTrips.size());
    // used for metrics, each ridership trip_id that is added to dao ridership.txt
    Set<String> ridershipIds = new HashSet<String>();
    // on the same route each day
    for (HashMap.Entry<String, TreeMap<Integer, String>> trip : ridershipTrips.entrySet()) {
        TreeMap<Integer, String> ridershipStops = trip.getValue();
        // for each ridership trip, we now have a tree list of the stops on that trip.  Compare to each dao trip
        List<String> ridershipStopList = new ArrayList<>(ridershipStops.values());
        for (HashMap.Entry<AgencyAndId, TreeMap<Integer, String>> daoTrip : daoTrips.entrySet()) {
            // daoStops is a treemap of stops on that trip, stop sequence: stop id
            TreeMap<Integer, String> daoStopsTree = daoTrip.getValue();
            // daoStopList is the stop ids on that trip in order, from the sequences being in order
            List<String> daoStopList = new ArrayList<>(daoStopsTree.values());
            boolean equal = ridershipStopList.equals(daoStopList);
            // the Access trip id is: trip.getKey(), the GTFS trip id is: daoTrip.getKey().getId
            if (equal) {
                // get the first stop time for the trip
                Trip GTFStrip = dao.getTripForId(daoTrip.getKey());
                StopTime firstStopTime = dao.getStopTimesForTrip(GTFStrip).get(0);
                DateTimeFormatter dtf = DateTimeFormatter.ofPattern("HH:mm:ss");
                String GTFStime = StopTimeFieldMappingFactory.getSecondsAsString(firstStopTime.getArrivalTime());
                // a lot of stop times are 'invalid' hours of 24, 25, 26.  For 24 hour clock, hour s/b 0-23
                GTFStime = ConvertToValidHours(GTFStime);
                LocalTime GTFSArrival = LocalTime.parse(GTFStime, dtf);
                // We have the stop sequence and the stop id.  From the stop id, get the stop time
                Entry<Integer, String> riderStop = ridershipStops.firstEntry();
                String firstStopId = riderStop.getValue();
                LocalTime passTime = null;
                // get the stop time for the first stop on this trip
                ArrayList<RidershipData> ridershipData = ridershipMap.get(trip.getKey());
                for (RidershipData rd : ridershipData) {
                    if (rd.getStopId() == firstStopId) {
                        passTime = rd.getPassingTime();
                    }
                }
                long minutesDifference = 15;
                if (passTime != null) {
                    minutesDifference = Duration.between(GTFSArrival, passTime).toMinutes();
                }
                if (minutesDifference < 4 && minutesDifference > -4) {
                    // _log.error("GTFS trip {}, time {}, Ridership trip {}, time {}, diff: {} ", trip.getKey().getId(), GTFSArrival, ridershipStops.getKey(),passTime, minutesDifference);
                    // we have HashMap<String, ArrayList<RidershipData>> ridershipMap = new HashMap<>();
                    // and we need to set all of the ridershipData ids to the GTFS id, from the ridership id
                    ArrayList<RidershipData> rsList = ridershipMap.get(trip.getKey());
                    // so, update the Trip id to go from Ridership_id to Trip_id
                    for (RidershipData rs : rsList) {
                        // get trips added ids
                        // if its already been added, compare differences and use lowest?
                        rs.setRsTripId(trip.getKey());
                        rs.setGtfsTripId(GTFStrip.getId().getId());
                        rs.setTripId(GTFStrip.getId().getId());
                        rs.setMinutesDifference(minutesDifference);
                        rs.setRouteId(dao.getTripForId(GTFStrip.getId()).getRoute().getId().getId());
                        saveRidership(dao, rs);
                        ridershipIds.add(rs.getRsTripId());
                    }
                // Do we need to handle when two ridership stops match to one GTFS? Compare and take the one that is less?
                // or what about the same ridership matching to multiple GTFS stops
                }
            }
        }
    }
    // the algorithm above takes each ridership trip and tries to find a GTFS trip to match it to.  The algorithm below
    // takes each GTFS trip and tries to find a ridership trip to match it to.  The numbers of matched trips is the same either way.
    // Try to find trips that match by comparing the stops on that trip.  We will get a lot of matches as identical trips run
    // on the same route each day
    /*        for (HashMap.Entry<AgencyAndId, TreeMap<Integer, String>> trip : daoTrips.entrySet()) {
            TreeMap<Integer, String> daoStops = (TreeMap<Integer, String>) trip.getValue();
            //for each dao trip, we now have a tree list of the stops on that trip.  Compare to each trip from ridership data
            List<String> daoStopList = new ArrayList<>(daoStops.values());
            for (HashMap.Entry<String, TreeMap<Integer, String>> ridershipStops : ridershipTrips.entrySet()) {
                TreeMap<Integer, String> ridersStops = (TreeMap<Integer, String>) ridershipStops.getValue();
                List<String> riderStopList = new ArrayList<>(ridersStops.values());
                boolean equal = daoStopList.equals(riderStopList);
                //The trips are 'equal' in that they have the same stops, narrow it down by time
                //the GTFS trip id is: trip.getKey().getId, the Access trip id is: ridershipStops.getKey()
                if (equal) {
                    //get the first stop time for the trip
                    Trip GTFStrip = dao.getTripForId(trip.getKey());
                    StopTime firstStopTime = dao.getStopTimesForTrip(GTFStrip).get(0);
                    DateTimeFormatter dtf = DateTimeFormatter.ofPattern("HH:mm:ss");

                    String GTFStime = StopTimeFieldMappingFactory.getSecondsAsString(firstStopTime.getArrivalTime());
                    //a lot of stop times are 'invalid' hours of 24, 25, 26.  For 24 hour clock, hour s/b 0-23
                    GTFStime = ConvertToValidHours(GTFStime);

                    LocalTime GTFSArrival = LocalTime.parse(GTFStime, dtf);

                    //We have the stop sequence and the stop id.  From the stop id, get the stop time
                    Entry<Integer, String> riderStop = ridersStops.firstEntry();
                    String firstStopId = riderStop.getValue();
                    LocalTime passTime = null;

                    //get the stop time for the first stop on this trip
                    ArrayList<RidershipData> ridershipData = ridershipMap.get(ridershipStops.getKey());
                    for (RidershipData rd : ridershipData) {
                        if (rd.getStopId() == firstStopId) {
                            passTime = rd.getPassingTime();
                        }
                    }
                    long minutesDifference = 15;

                    if (passTime != null) {
                        minutesDifference = Duration.between(GTFSArrival, passTime).toMinutes();
                    }
                    if (minutesDifference < 5 && minutesDifference > -5) {
                        //_log.error("GTFS trip {}, time {}, Ridership trip {}, time {}, diff: {} ", trip.getKey().getId(), GTFSArrival, ridershipStops.getKey(),passTime, minutesDifference);

                        //we have
                        // HashMap<String, ArrayList<RidershipData>> ridershipMap = new HashMap<>();
                        // and we need to set all of the ridershipData ids to the GTFS id, from the ridership id

                        ArrayList<RidershipData> rsList = ridershipMap.get(ridershipStops.getKey());
                        //this is the list of data where the Trip ids are all the same, and each entry is a stop
                        //so, update the Trip id to go from Ridership_id to Trip_id
                        for (RidershipData rs : rsList) {

                            //get trips added ids
                            //if its already been added, compare differences and use lowest?
                            rs.setGtfsTripId(trip.getKey().getId());
                            rs.setTripId(trip.getKey().getId());
                            rs.setMinutesDifference(minutesDifference);
                            rs.setRouteId(dao.getTripForId(trip.getKey()).getRoute().getId().getId());
                            saveRidership(dao, rs);
                            ridershipIds.add(rs.getRsTripId());
                        }

                        //Do we need to handle when two ridership stops match to one GTFS? Compare and take the one that is less?
                        //or what about the same ridership matching to multiple GTFS stops

                    }
                }
            }
        }*/
    List<Ridership> riderships2 = new ArrayList<>(dao.getAllRiderships());
    _log.error("Ridership size: {}", dao.getAllRiderships().size());
    _log.error("Ridership size: {}", riderships2.size());
    int in = 0;
    int out = 0;
    // iterate over list of ridership data. For each ridershipData, is it in ridership.txt?
    for (String ridership_trip_id : ridershipMap.keySet()) {
        if (ridershipIds.contains(ridership_trip_id)) {
            in++;
        } else {
            out++;
        }
    }
    _log.error("Trips ids in ridership.txt: {}, trip ids abandoned: {}", in, out);
}

Example 12 with ExternalServices

use of org.onebusaway.cloud.api.ExternalServices in project onebusaway-gtfs-modules by OneBusAway.

the class CountAndTestSubway method run.

@Override
public void run(TransformContext context, GtfsMutableRelationalDao dao) {
    GtfsMutableRelationalDao reference = (GtfsMutableRelationalDao) context.getReferenceReader().getEntityStore();
    String agency = dao.getAllAgencies().iterator().next().getId();
    String name = dao.getAllAgencies().iterator().next().getName();
    HashMap<String, Route> referenceRoutes = new HashMap<>();
    for (Route route : reference.getAllRoutes()) {
        referenceRoutes.put(route.getId().getId(), route);
    }
    HashMap<String, Trip> referenceTrips = new HashMap<>();
    for (Trip trip : reference.getAllTrips()) {
        referenceTrips.put(trip.getId().getId(), trip);
    }
    HashMap<String, Stop> referenceStops = new HashMap<>();
    for (Stop stop : reference.getAllStops()) {
        referenceStops.put(stop.getId().getId(), stop);
    }
    int matches = 0;
    for (Route route : dao.getAllRoutes()) {
        if (referenceRoutes.containsKey(route.getId().getId())) {
            matches++;
        } else {
            _log.info("ATIS route {} doesn't have match in reference", route.getId().getId());
        }
    }
    _log.info("ATIS Routes: {}, References: {}, ATIS match to reference: {}", dao.getAllRoutes().size(), reference.getAllRoutes().size(), matches);
    int countSt = 0;
    int countCd = 0;
    int countNoSt = 0;
    int countNoCd = 0;
    int countNoHs = 0;
    int curSerTrips = 0;
    AgencyAndId serviceAgencyAndId = new AgencyAndId();
    matches = 0;
    for (Trip trip : dao.getAllTrips()) {
        if (referenceTrips.containsKey(trip.getId().getId())) {
            matches++;
        }
        if (dao.getStopTimesForTrip(trip).size() == 0) {
            countNoSt++;
        } else {
            countSt++;
        }
        serviceAgencyAndId = trip.getServiceId();
        if (dao.getCalendarDatesForServiceId(serviceAgencyAndId).size() == 0) {
            countNoCd++;
        } else {
            countCd++;
        }
        if (trip.getTripHeadsign() == null) {
            countNoHs++;
        }
        // check for current service
        for (ServiceCalendarDate calDate : dao.getCalendarDatesForServiceId(trip.getServiceId())) {
            Date date = constructDate(calDate.getDate());
            Date today = removeTime(new Date());
            if (calDate.getExceptionType() == 1 && date.equals(today)) {
                curSerTrips++;
                break;
            }
        }
    }
    _log.info("ATIS Trips: {}, Reference: {}, match: {}, Current Service: {}", dao.getAllTrips().size(), reference.getAllTrips().size(), matches, curSerTrips);
    _log.info("Total stop times {}, Trips w/ st: {}, Trips w/out st: {}", dao.getAllStopTimes().size(), countSt, countNoSt);
    _log.info("Total calendar dates {}, Trips w/cd {}, Trips w/out cd: {}", dao.getAllCalendarDates().size(), countCd, countNoCd);
    _log.info("Total trips w/out headsign: {}", countNoHs);
    matches = 0;
    for (Stop stop : dao.getAllStops()) {
        if (referenceStops.containsKey(stop.getId().getId())) {
            matches++;
        }
    }
    _log.info("ATIS Stops: {}, Reference: {}, ATIS match to reference: {}", dao.getAllStops().size(), reference.getAllStops().size(), matches);
    ExternalServices es = new ExternalServicesBridgeFactory().getExternalServices();
    String feed = CloudContextService.getLikelyFeedName(dao);
    es.publishMetric(CloudContextService.getNamespace(), "SubwayTripsInServiceToday", "feed", feed, curSerTrips);
    if (countNoHs > 0) {
        _log.error("There are trips with no headsign");
    }
    es.publishMetric(CloudContextService.getNamespace(), "TripsWithoutHeadsigns", "feed", feed, countNoHs);
    HashSet<String> ids = new HashSet<String>();
    for (Stop stop : dao.getAllStops()) {
        // check for duplicate stop ids.
        if (ids.contains(stop.getId().getId())) {
            _log.error("Duplicate stop ids! Agency {} stop id {}", agency, stop.getId().getId());
            es.publishMultiDimensionalMetric(CloudContextService.getNamespace(), "DuplicateStopIds", new String[] { "feed", "stopId" }, new String[] { feed, stop.getId().toString() }, 1);
            throw new IllegalStateException("There are duplicate stop ids!");
        } else {
            ids.add(stop.getId().getId());
        }
    }
}

Example 13 with ExternalServices

use of org.onebusaway.cloud.api.ExternalServices in project onebusaway-gtfs-modules by OneBusAway.

the class CheckForFutureService method run.

@Override
public void run(TransformContext context, GtfsMutableRelationalDao dao) {
    int tripsToday = 0;
    int tripsTomorrow = 0;
    int tripsNextDay = 0;
    int tripsDayAfterNext = 0;
    Date today = removeTime(new Date());
    Date tomorrow = removeTime(addDays(new Date(), 1));
    Date nextDay = removeTime(addDays(new Date(), 2));
    Date dayAfterNext = removeTime(addDays(new Date(), 3));
    String feed = CloudContextService.getLikelyFeedName(dao);
    ExternalServices es = new ExternalServicesBridgeFactory().getExternalServices();
    String agency = dao.getAllAgencies().iterator().next().getId();
    String agencyName = dao.getAllAgencies().iterator().next().getName();
    tripsToday = hasServiceForDate(dao, today);
    tripsTomorrow = hasServiceForDate(dao, tomorrow);
    tripsNextDay = hasServiceForDate(dao, nextDay);
    tripsDayAfterNext = hasServiceForDate(dao, dayAfterNext);
    _log.info("Feed for metrics: {}, agency id: {}", feed, agencyName);
    es.publishMetric(CloudContextService.getNamespace(), "TripsToday", "feed", feed, tripsToday);
    es.publishMetric(CloudContextService.getNamespace(), "TripsTomorrow", "feed", feed, tripsTomorrow);
    es.publishMetric(CloudContextService.getNamespace(), "TripsIn2Days", "feed", feed, tripsNextDay);
    es.publishMetric(CloudContextService.getNamespace(), "TripsIn3Days", "feed", feed, tripsDayAfterNext);
    _log.info("TripsToday: {}, feed: {}, namespace: {}", tripsToday, feed, CloudContextService.getNamespace());
    _log.info("TripsTomorrow: {}, feed: {}, namespace: {}", tripsTomorrow, feed, CloudContextService.getNamespace());
    _log.info("TripsIn2Days: {}, feed: {}, namespace: {}", tripsNextDay, feed, CloudContextService.getNamespace());
    _log.info("TripsIn3Days: {}, feed: {}, namespace: {}", tripsDayAfterNext, feed, CloudContextService.getNamespace());
    if (tripsToday == 0) {
        _log.error("Agency {} {} is missing service for today {}", agency, agencyName, tomorrow);
    }
    if (tripsTomorrow == 0) {
        _log.error("Agency {} {} is missing service for tomorrow {}", agency, agencyName, tomorrow);
    }
    if (tripsNextDay == 0) {
        _log.error("Agency {} {} is missing service for the day after tomorrow {}", agency, agencyName, nextDay);
    }
    if (tripsDayAfterNext == 0) {
        _log.error("Agency {} {} is missing service in 3 days {}", agency, agencyName, dayAfterNext);
    }
}

Example 14 with ExternalServices

use of org.onebusaway.cloud.api.ExternalServices in project onebusaway-gtfs-modules by OneBusAway.

the class CheckForPlausibleStopTimes method run.

@Override
public void run(TransformContext context, GtfsMutableRelationalDao dao) {
    String feed = CloudContextService.getLikelyFeedName(dao);
    ExternalServices es = new ExternalServicesBridgeFactory().getExternalServices();
    RemoveEntityLibrary removeEntityLibrary = new RemoveEntityLibrary();
    SimpleDateFormat sdf = new SimpleDateFormat("HH:mm:ss");
    Set<Trip> stopsWarn = new HashSet<Trip>();
    Set<Trip> stopsRemove = new HashSet<Trip>();
    String collectedWarnString = "";
    String collectedRemoveString = "";
    for (Trip trip : dao.getAllTrips()) {
        StopTime oldTime = new StopTime();
        boolean starting = true;
        stopLoop: for (StopTime newTime : dao.getStopTimesForTrip(trip)) {
            if (starting) {
                starting = false;
                oldTime = newTime;
            }
            // check if the bus takes more than five hours between stops
            int timeDelta = newTime.getArrivalTime() - oldTime.getDepartureTime();
            if (timeDelta > 1 * MINUTES_PER_HOUR * SECONDS_PER_MINUTE) {
                Date departure = new Date(oldTime.getDepartureTime() * 1000);
                Date arrival = new Date(newTime.getArrivalTime() * 1000);
                String message = "Trip " + trip.getId().getId() + " on Route " + trip.getRoute().getId() + " is scheduled for unrealistic transit time (>1hr) when traveling between stoptime" + oldTime.getId() + " at " + sdf.format(departure) + ", and stoptime" + newTime.getId() + " at " + sdf.format(arrival);
                _log.warn(message);
                stopsWarn.add(trip);
                collectedWarnString += ", " + trip.toString() + "at " + sdf.format(departure) + "and " + sdf.format(arrival);
            }
            if (timeDelta > 3 * MINUTES_PER_HOUR * SECONDS_PER_MINUTE) {
                Date departure = new Date(oldTime.getDepartureTime() * 1000);
                Date arrival = new Date(newTime.getArrivalTime() * 1000);
                String message = "Trip " + trip.getId().getId() + " on Route " + trip.getRoute().getId() + " is scheduled for unrealistic transit time (>3hr) when traveling between stoptime" + oldTime.getId() + " at " + sdf.format(departure) + ", and stoptime" + newTime.getId() + " at " + sdf.format(arrival) + ". This trip will be deleted.";
                _log.error(message);
                collectedRemoveString += ", " + trip.toString();
                stopsRemove.add(trip);
                break stopLoop;
            }
            oldTime = newTime;
        }
    }
    if (stopsWarn.size() > 0) {
        collectedWarnString = "Total number of trips with transit times of greater than one hour: " + stopsWarn.size() + ".\n Here are the trips and stops: " + collectedWarnString.substring(2);
        _log.info(collectedWarnString);
    }
    es.publishMetric(CloudContextService.getNamespace(), "TripsWith1-3HrTransitTime", "feed", feed, stopsWarn.size());
    if (stopsRemove.size() > 0) {
        collectedRemoveString = "Total number of trips with transit times of greater than three hours: " + stopsRemove.size() + ".\n These trips are being removed. \nTrips being removed: " + collectedRemoveString.substring(2);
        _log.info(collectedRemoveString);
    }
    es.publishMetric(CloudContextService.getNamespace(), "TripsWithRemovedForTransitTime", "feed", feed, stopsRemove.size());
    for (Trip trip : stopsRemove) {
        removeEntityLibrary.removeTrip(dao, trip);
    }
}

Example 15 with ExternalServices

use of org.onebusaway.cloud.api.ExternalServices in project onebusaway-gtfs-modules by OneBusAway.

the class UpdateStopIdById method run.

@Override
public void run(TransformContext context, GtfsMutableRelationalDao dao) {
    GtfsMutableRelationalDao reference = (GtfsMutableRelationalDao) context.getReferenceReader().getEntityStore();
    RemoveEntityLibrary removeEntityLibrary = new RemoveEntityLibrary();
    HashMap<String, Stop> referenceStops = new HashMap<>();
    for (Stop stop : reference.getAllStops()) {
        referenceStops.put(stop.getId().getId(), stop);
    }
    ArrayList<Stop> stopsToDelete = new ArrayList<>();
    ArrayList<String> existingStops = new ArrayList<>();
    String feed = CloudContextService.getLikelyFeedName(dao);
    ExternalServices es = new ExternalServicesBridgeFactory().getExternalServices();
    String agency = dao.getAllAgencies().iterator().next().getId();
    String name = dao.getAllAgencies().iterator().next().getName();
    for (Stop stop : dao.getAllStops()) {
        if (stop.getMtaStopId() != null) {
            if (existingStops.contains(stop.getMtaStopId())) {
                _log.info("There is another stop with the same mta_id. This stop will be removed: Agency {} {} ATIS stop id: {} MTA stop id: {}", agency, name, stop.getId().getId(), stop.getMtaStopId());
                stopsToDelete.add(stop);
            } else {
                existingStops.add(stop.getMtaStopId());
                Stop refStop = referenceStops.get(stop.getMtaStopId());
                if (refStop != null) {
                    stop.setName(refStop.getName());
                }
                stop.setId(new AgencyAndId(stop.getId().getAgencyId(), stop.getMtaStopId()));
            }
        }
    }
    es.publishMetric(CloudContextService.getNamespace(), "DuplicateStopCount", "feed", feed, stopsToDelete.size());
    for (Stop stop : stopsToDelete) {
        removeEntityLibrary.removeStop(dao, stop);
    }
}