Examples with GtfsMutableRelationalDao org.onebusaway.gtfs.services.GtfsMutableRelationalDao used on opensource projects

Example 51 with GtfsMutableRelationalDao

use of org.onebusaway.gtfs.services.GtfsMutableRelationalDao in project onebusaway-gtfs-modules by OneBusAway.

the class UpdateTripsForSdon method run.

@Override
public void run(TransformContext context, GtfsMutableRelationalDao dao) {
    GtfsMutableRelationalDao reference = (GtfsMutableRelationalDao) context.getReferenceReader().getEntityStore();
    CalendarService refCalendarService = CalendarServiceDataFactoryImpl.createService(reference);
    String agency = dao.getAllTrips().iterator().next().getId().getAgencyId();
    HashMap<String, Trip> referenceTrips = new HashMap<>();
    for (Trip trip : reference.getAllTrips()) {
        referenceTrips.put(trip.getId().getId(), trip);
    }
    HashMap<String, Trip> atisTrips = new HashMap<>();
    for (Trip trip : dao.getAllTrips()) {
        atisTrips.put(trip.getMtaTripId(), trip);
    }
    // Unique list of calendar dates to add where the key = new service id
    HashMap<String, ArrayList<ServiceDate>> serviceDates = new HashMap<>();
    // for each reference trip, if it contails SDon
    // create a nonSdon string
    // is that id in ATIS?
    // no, do nothing
    // yes - ok, for each calendar date, we need to do something...
    // if the calendar_date is "in" the SDON, then move it there
    // if the calendar_date isn't in SDON, then keep it as it is
    // NO!! can't move them.  The numbers might be used somewhere else.  Arg.  Create new.
    // Don't be sloppy and create duplicates
    // all the stop times for this trip, create new identical ones for the new Sdon trip
    int service_id = getNextServiceId(dao);
    int countdao = 0;
    for (HashMap.Entry<String, Trip> referenceTrip : referenceTrips.entrySet()) {
        Trip refTrip = referenceTrip.getValue();
        if (refTrip.getId().getId().contains("SDon")) {
            String refId = refTrip.getId().getId();
            String refIdMinusSDon = refId.replace("-SDon", "");
            if (atisTrips.containsKey(refIdMinusSDon)) {
                Trip atisTrip = atisTrips.get(refIdMinusSDon);
                // we have an atis trip that 'matches' a reference Sdon trip
                // create a new trip with this id and copy over everything about the trip
                Trip sdonAtisTrip = createTrip(dao, refTrip, atisTrip);
                // create a list of dates that are the service dates for this Sdon trip
                Set<ServiceDate> refServiceDates = refCalendarService.getServiceDatesForServiceId(refTrip.getServiceId());
                // then, for each calendar date from the ATIS trip, if it matches the Sdon trip,
                // create a new list of calendar dates for this service id
                ArrayList<ServiceDate> sdForTrip = new ArrayList<>();
                List<ServiceCalendarDate> atisCalendarDates = dao.getCalendarDatesForServiceId(atisTrip.getServiceId());
                for (ServiceCalendarDate atisScd : atisCalendarDates) {
                    ServiceDate atisDate = atisScd.getDate();
                    if (refServiceDates.contains(atisDate)) {
                        // _log.info("Date match. Ref id: {}, Atis id {}, Atis date {}", refTrip.getId(), atisTrip.getId(), atisDate.toString());
                        sdForTrip.add(atisDate);
                    }
                }
                // see if this list of sdForTrip is unique
                // have a hashmap of service dates, where the service id is the key and the value is the list of dates
                // does the  hashmap have this list of dates?
                // yes, great, take the id and use that for the trip
                // no? ok, create the calendar dates
                AgencyAndId newServiceId = new AgencyAndId(agency, Integer.toString(service_id));
                if (serviceDates.isEmpty()) {
                    serviceDates.put(Integer.toString(service_id), sdForTrip);
                    newServiceId = new AgencyAndId(agency, Integer.toString(service_id));
                    service_id++;
                } else {
                    boolean addNewServiceDate = true;
                    for (HashMap.Entry<String, ArrayList<ServiceDate>> serviceDate : serviceDates.entrySet()) {
                        ArrayList<ServiceDate> scds = (ArrayList<ServiceDate>) serviceDate.getValue();
                        // scds is a unique list of service dates in the map
                        if (new HashSet<ServiceDate>(sdForTrip).equals(new HashSet<ServiceDate>(scds))) {
                            // we already have a list of the same dates.  Re-use the service id
                            addNewServiceDate = false;
                            // set the service date id to be this one
                            newServiceId = new AgencyAndId(agency, serviceDate.getKey());
                            sdonAtisTrip.setServiceId(newServiceId);
                        }
                    }
                    // there was no match, update the date map and add new serviceId
                    if (addNewServiceDate) {
                        // dates don't exist, add new entry
                        serviceDates.put(Integer.toString(service_id), sdForTrip);
                        newServiceId = new AgencyAndId(agency, Integer.toString(service_id));
                        service_id++;
                    }
                }
                sdonAtisTrip.setServiceId(newServiceId);
                for (StopTime stopTime : dao.getStopTimesForTrip(atisTrip)) {
                    StopTime st = new StopTime();
                    st.setTrip(sdonAtisTrip);
                    st.setStop(stopTime.getStop());
                    st.setArrivalTime(stopTime.getArrivalTime());
                    st.setDepartureTime(stopTime.getDepartureTime());
                    st.setStopSequence(stopTime.getStopSequence());
                    st.setDropOffType(stopTime.getDropOffType());
                    st.setDepartureBuffer(stopTime.getDepartureBuffer());
                    dao.saveOrUpdateEntity(st);
                    countdao++;
                }
            }
        }
    }
    _log.info("Adding {} service date ids. Next id {}", serviceDates.size(), getNextServiceId(dao));
    int serviceIds = 0;
    // Now the list is compete, add the new service id and dates
    for (HashMap.Entry<String, ArrayList<ServiceDate>> serviceDatesToAdd : serviceDates.entrySet()) {
        AgencyAndId newServiceId = new AgencyAndId(agency, serviceDatesToAdd.getKey());
        ArrayList<ServiceDate> scds = serviceDatesToAdd.getValue();
        // need a list of the service cal dates, iterate, add
        for (ServiceDate sd : scds) {
            serviceIds++;
            ServiceCalendarDate newScd = new ServiceCalendarDate();
            newScd.setServiceId(newServiceId);
            newScd.setDate(sd);
            // add
            newScd.setExceptionType(1);
            dao.saveOrUpdateEntity(newScd);
        }
    }
    _log.error("Dao count: {}.  Added Service dates: {}", countdao, serviceIds);
}

Example 52 with GtfsMutableRelationalDao

use of org.onebusaway.gtfs.services.GtfsMutableRelationalDao in project onebusaway-gtfs-modules by OneBusAway.

the class UpdateCalendarDatesForDuplicateTrips method run.

@Override
public void run(TransformContext context, GtfsMutableRelationalDao dao) {
    RemoveEntityLibrary removeEntityLibrary = new RemoveEntityLibrary();
    String agency = dao.getAllTrips().iterator().next().getId().getAgencyId();
    // map of each mta_trip_id and list of trips
    HashMap<String, ArrayList<Trip>> tripsMap = new HashMap<>();
    // List of DuplicateTrips
    ArrayList<DuplicateTrips> duplicateTripData = new ArrayList<>();
    // they are only duplicates if the stop times match as well.
    // if the stop times match, then we can move forward with merging trips
    // if not, then we can't merge and we leave the trips alone
    // set all the trips that are duplicates based on mta_trip_id
    int mtaIdNull = 0;
    for (Trip trip : dao.getAllTrips()) {
        if (trip.getMtaTripId() != null) {
            if (tripsMap.containsKey(trip.getMtaTripId())) {
                ArrayList<Trip> trips = tripsMap.get(trip.getMtaTripId());
                trips.add(trip);
                tripsMap.put(trip.getMtaTripId(), trips);
            } else {
                ArrayList<Trip> trips = new ArrayList<>();
                trips.add(trip);
                tripsMap.put(trip.getMtaTripId(), trips);
            }
        } else {
            _log.info("trip {} mta_trip_id is null", trip.getId());
            mtaIdNull++;
        }
    }
    GtfsMutableRelationalDao reference = (GtfsMutableRelationalDao) context.getReferenceReader().getEntityStore();
    HashMap<String, Trip> referenceTrips = new HashMap<>();
    for (Trip trip : reference.getAllTrips()) {
        referenceTrips.put(trip.getId().getId(), trip);
    }
    // this is just for logging if dups are in reference, delete when ready
    /*       Iterator entries2 = tripsMap.entrySet().iterator();
        while (entries2.hasNext()) {
            HashMap.Entry entry = (HashMap.Entry) entries2.next();
            ArrayList<Trip> trips = (ArrayList<Trip>) entry.getValue();
            if (trips.size() > 1) {
                //these are duplicates
                if (referenceTrips.containsKey(entry.getKey())) {
                    //_log.info("Duplicate trip id {} is in reference", entry.getKey());
                }
            }
        }
*/
    int orStopTimes = dao.getAllStopTimes().size();
    _log.info("Routes: {} Trips: {} Stops: {} Stop times: {} CalDatess: {} ", dao.getAllRoutes().size(), dao.getAllTrips().size(), dao.getAllStops().size(), dao.getAllStopTimes().size(), dao.getAllCalendarDates().size());
    int countUnique = 0;
    int countCombine = 0;
    int countDoNothing = 0;
    int countToday = 0;
    Iterator entries = tripsMap.entrySet().iterator();
    int service_id = getNextServiceId(dao);
    while (entries.hasNext()) {
        HashMap.Entry entry = (HashMap.Entry) entries.next();
        ArrayList<Trip> trips = (ArrayList<Trip>) entry.getValue();
        if (trips.size() > 1) {
            Boolean equals = true;
            // If not, leave the trip alone.  Do nothing.
            trip_loop: for (int i = 0; i < trips.size(); i++) {
                for (int j = i + 1; j < trips.size(); j++) {
                    // they won't be equal because a stop time has a trip id and the trip ids are different
                    if (!stopTimesEqual(dao.getStopTimesForTrip(trips.get(i)), dao.getStopTimesForTrip(trips.get(j)))) {
                        // _log.info("The stop times for {} and {} are not equal", trips.get(i).getId().getId(), trips.get(j).getId().getId());
                        equals = false;
                        // so at this point the stop times don't equal.  Do I check if its just one or just throw the whole thing out?
                        // For now if one doesn't match then none do and I'm going to ignore.
                        countDoNothing = countDoNothing + trips.size();
                        // so at least one trip will get the right id
                        if (checkForServiceToday(trips, dao)) {
                            countToday++;
                        }
                        break trip_loop;
                    }
                }
            }
            if (equals) {
                // _log.info("EQUALS!");
                // for each mta_id that is a duplicate, we need to ultimately delete those duplicates
                // First, get all the corresponding serviceDates for all the trips with that mta_id, then create new service ids
                // and add entries with that new id that correspond to all the service dates for all the trips
                DuplicateTrips dup = new DuplicateTrips((String) entry.getKey(), Integer.toString(service_id), trips);
                duplicateTripData.add(dup);
                service_id++;
                countCombine = countCombine + trips.size();
            }
        } else {
            // trips.size is not > 1 so these trips are unique.  Copy over mta_trip_id
            Trip trip = trips.get(0);
            trip.setId(new AgencyAndId(trip.getId().getAgencyId(), trip.getMtaTripId()));
            countUnique++;
        }
    }
    _log.info("Mta_trip_ids: null {}, unique {}, do nothing {}, today {}, combine {}, total {}", mtaIdNull, countUnique, countDoNothing, countToday, countCombine, mtaIdNull + countUnique + countDoNothing + countCombine);
    // now we have a list of DuplicateTrips and we need to fill in the calendar dates
    for (DuplicateTrips dts : duplicateTripData) {
        for (Trip trip : dts.getTrips()) {
            // for each trip, get the calendar dates
            for (ServiceCalendarDate calDate : dao.getCalendarDatesForServiceId(trip.getServiceId())) {
                dts.addServiceDate(calDate);
            }
        }
    }
    // now we have a list of DuplicateTrips and their calendar dates
    // a lot of the DuplicateTrips will have the same list of calendar dates.  Don't create duplicate calendar entries unnecessarily
    // Create a unique list of calendar dates to add
    HashMap<String, ArrayList<ServiceCalendarDate>> dateMap = new HashMap<>();
    // for each duplicateTrips in the list, get the list of caldate entries
    // if the caldate entries is in the dateMap, change the Service Id for the duplicate trip
    // if its not in there, then add it
    int newDates = 0;
    for (DuplicateTrips dts : duplicateTripData) {
        // first time through, populate dateMap
        if (dateMap.isEmpty()) {
            dateMap.put(dts.getServiceId(), dts.getDates());
        } else {
            boolean addNewDateMap = true;
            for (HashMap.Entry<String, ArrayList<ServiceCalendarDate>> calDate : dateMap.entrySet()) {
                ArrayList<ServiceCalendarDate> scds = (ArrayList<ServiceCalendarDate>) calDate.getValue();
                // scds is a unique list of service calendar dates in the map
                if (new HashSet<ServiceCalendarDate>(dts.getDates()).equals(new HashSet<ServiceCalendarDate>(scds))) {
                    // we already have a list of the same dates.  Re-use the service id
                    addNewDateMap = false;
                    // set the service date id in DuplicateTrips to be this one
                    dts.setServiceId(calDate.getKey());
                    break;
                }
            }
            // there was no match, update the date map and add new serviceId
            if (addNewDateMap) {
                // dates don't exist, add new entry to date map and add service id
                dateMap.put(dts.getServiceId(), dts.getDates());
                newDates = newDates + dts.getDates().size();
            }
        }
    }
    int serviceIds = 0;
    // Now the list is compete, add the new service id and dates
    for (HashMap.Entry<String, ArrayList<ServiceCalendarDate>> calDateId : dateMap.entrySet()) {
        AgencyAndId newServiceId = new AgencyAndId(agency, calDateId.getKey());
        ArrayList<ServiceCalendarDate> scds = calDateId.getValue();
        // need a list of the service cal dates, iterate, add
        for (ServiceCalendarDate calDate : scds) {
            serviceIds++;
            // for each date, create a new calendar_dates entry with the new service_id
            ServiceCalendarDate newScd = new ServiceCalendarDate();
            newScd.setServiceId(newServiceId);
            newScd.setDate(calDate.getDate());
            newScd.setExceptionType(calDate.getExceptionType());
            dao.saveOrUpdateEntity(newScd);
        }
    }
    // trips updated, array of mta_ids that we've updated
    HashMap<String, Trip> tripsUpdated = new HashMap<>();
    ArrayList<Trip> tripsToRemove = new ArrayList<>();
    // update the trips with the new service_id
    for (DuplicateTrips dts : duplicateTripData) {
        AgencyAndId newServiceId = new AgencyAndId(agency, dts.getServiceId());
        for (Trip trip : dts.getTrips()) {
            // for each trip, set the new service id
            trip.setServiceId(newServiceId);
            // and then there needs to be a seperate method for all the deletions.
            if (trip.getMtaTripId() != null) {
                if (tripsUpdated.containsKey(trip.getMtaTripId())) {
                    tripsToRemove.add(trip);
                } else {
                    tripsUpdated.put(trip.getMtaTripId(), trip);
                    trip.setId(new AgencyAndId(trip.getId().getAgencyId(), trip.getMtaTripId()));
                }
            }
        }
    }
    int stopsTimesToRemove = 0;
    int remove = 0;
    for (Trip tripToRemove : tripsToRemove) {
        stopsTimesToRemove = stopsTimesToRemove + dao.getStopTimesForTrip(tripToRemove).size();
        removeEntityLibrary.removeTrip(dao, tripToRemove);
        remove++;
    }
    _log.info("Added Service Cal dates: {}, Removed trips: {}, Removed stoptimes: {}", serviceIds, remove, stopsTimesToRemove);
    _log.info("Routes: {} Trips: {} Stops: {} Stop times: {} CalDates: {} ", dao.getAllRoutes().size(), dao.getAllTrips().size(), dao.getAllStops().size(), dao.getAllStopTimes().size(), dao.getAllCalendarDates().size());
}

Example 53 with GtfsMutableRelationalDao

use of org.onebusaway.gtfs.services.GtfsMutableRelationalDao in project onebusaway-gtfs-modules by OneBusAway.

the class UpdateStopIdById method run.

@Override
public void run(TransformContext context, GtfsMutableRelationalDao dao) {
    GtfsMutableRelationalDao reference = (GtfsMutableRelationalDao) context.getReferenceReader().getEntityStore();
    RemoveEntityLibrary removeEntityLibrary = new RemoveEntityLibrary();
    HashMap<String, Stop> referenceStops = new HashMap<>();
    for (Stop stop : reference.getAllStops()) {
        referenceStops.put(stop.getId().getId(), stop);
    }
    ArrayList<Stop> stopsToDelete = new ArrayList<>();
    ArrayList<String> existingStops = new ArrayList<>();
    String feed = CloudContextService.getLikelyFeedName(dao);
    ExternalServices es = new ExternalServicesBridgeFactory().getExternalServices();
    String agency = dao.getAllAgencies().iterator().next().getId();
    String name = dao.getAllAgencies().iterator().next().getName();
    for (Stop stop : dao.getAllStops()) {
        if (stop.getMtaStopId() != null) {
            if (existingStops.contains(stop.getMtaStopId())) {
                _log.info("There is another stop with the same mta_id. This stop will be removed: Agency {} {} ATIS stop id: {} MTA stop id: {}", agency, name, stop.getId().getId(), stop.getMtaStopId());
                stopsToDelete.add(stop);
            } else {
                existingStops.add(stop.getMtaStopId());
                Stop refStop = referenceStops.get(stop.getMtaStopId());
                if (refStop != null) {
                    stop.setName(refStop.getName());
                }
                stop.setId(new AgencyAndId(stop.getId().getAgencyId(), stop.getMtaStopId()));
            }
        }
    }
    es.publishMetric(CloudContextService.getNamespace(), "DuplicateStopCount", "feed", feed, stopsToDelete.size());
    for (Stop stop : stopsToDelete) {
        removeEntityLibrary.removeStop(dao, stop);
    }
}

Example 54 with GtfsMutableRelationalDao

use of org.onebusaway.gtfs.services.GtfsMutableRelationalDao in project onebusaway-gtfs-modules by OneBusAway.

the class VerifyBusService method run.

@Override
public void run(TransformContext context, GtfsMutableRelationalDao dao) {
    GtfsMutableRelationalDao reference = (GtfsMutableRelationalDao) context.getReferenceReader().getEntityStore();
    ExternalServices es = new ExternalServicesBridgeFactory().getExternalServices();
    String feed = CloudContextService.getLikelyFeedName(dao);
    CalendarService refCalendarService = CalendarServiceDataFactoryImpl.createService(reference);
    AgencyAndId refAgencyAndId = reference.getAllTrips().iterator().next().getId();
    int curSerRoute = 0;
    int alarmingRoutes = 0;
    Date today = removeTime(new Date());
    // list of all routes in ATIS
    Set<String> ATISrouteIds = new HashSet<>();
    // check for route specific current service
    for (Route route : dao.getAllRoutes()) {
        ATISrouteIds.add(route.getId().getId());
        _log.info("Adding route: {}", route.getId().getId());
        curSerRoute = 0;
        triploop: for (Trip trip1 : dao.getTripsForRoute(route)) {
            for (ServiceCalendarDate calDate : dao.getCalendarDatesForServiceId(trip1.getServiceId())) {
                Date date = constructDate(calDate.getDate());
                if (calDate.getExceptionType() == 1 && date.equals(today)) {
                    _log.info("ATIS has current service for route: {}", route.getId().getId());
                    curSerRoute++;
                    break triploop;
                }
            }
        }
        if (curSerRoute == 0) {
            _log.error("No current service for {}", route.getId().getId());
            // if there is no current service, check that it should have service
            // there are certain routes that don't run on the weekend or won't have service in reference
            ServiceDate sToday = createServiceDate(today);
            Route refRoute = reference.getRouteForId(new AgencyAndId(refAgencyAndId.getAgencyId(), route.getId().getId()));
            reftriploop: for (Trip refTrip : reference.getTripsForRoute(refRoute)) {
                Set<ServiceDate> activeDates = refCalendarService.getServiceDatesForServiceId(refTrip.getServiceId());
                if (activeDates.contains(sToday)) {
                    _log.info("Reference has service for this bus route today but ATIS does not: {}", route.getId());
                    // This would be the site to add to a bulk metric, missingBus:
                    alarmingRoutes++;
                }
            }
        }
    }
    es.publishMetric(CloudContextService.getNamespace(), "RoutesMissingTripsFromAtisButInRefToday", "feed", feed, alarmingRoutes);
    es.publishMetric(CloudContextService.getNamespace(), "RoutesContainingTripsToday", "feed", feed, curSerRoute);
}

Example 55 with GtfsMutableRelationalDao

use of org.onebusaway.gtfs.services.GtfsMutableRelationalDao in project onebusaway-gtfs-modules by OneBusAway.

the class UpdateTripHeadsignByReference method run.

@Override
public void run(TransformContext context, GtfsMutableRelationalDao dao) {
    GtfsMutableRelationalDao reference = (GtfsMutableRelationalDao) context.getReferenceReader().getEntityStore();
    String agency = reference.getAllTrips().iterator().next().getId().getAgencyId();
    ArrayList<String> missingStops = new ArrayList<>();
    for (Trip trip : dao.getAllTrips()) {
        List<StopTime> stopTimes = dao.getStopTimesForTrip(trip);
        if (stopTimes != null && stopTimes.size() > 0) {
            Stop stop = (Stop) stopTimes.get(stopTimes.size() - 1).getStop();
            Stop gtfsStop = reference.getStopForId(new AgencyAndId(getReferenceAgencyId(reference), stop.getMtaStopId()));
            if (gtfsStop == null && !missingStops.contains(stop.getMtaStopId())) {
                _log.info("Stop {} is missing reference stop {} for agency {}", stop.getId(), stop.getMtaStopId(), getReferenceAgencyId(reference));
                missingStops.add(stop.getMtaStopId());
            }
            if (gtfsStop != null) {
                // then set the headsign from the reference stop
                String tripHeadSign = gtfsStop.getName();
                if (tripHeadSign != null) {
                    trip.setTripHeadsign(tripHeadSign);
                } else {
                    // TODO reference GTFS has no headsign, add publish message for this error
                    _log.error("No reference trip headsign {}", gtfsStop.getId());
                    fallbackSetHeadsign(trip, stop);
                }
            } else {
                fallbackSetHeadsign(trip, stop);
            }
        } else {
            _log.error("No stoptimes for trip {} mta id", trip.toString(), trip.getMtaTripId());
            if (trip.getTripHeadsign() == null && trip.getRouteShortName() == null) {
                // if trip has no headsign, no stoptimes and no shortname, remove it
                _log.error("Removing trip {}", trip.getId());
                dao.removeEntity(trip);
            } else {
                genericSetHeadsign(trip);
            }
        }
    }
}