Examples with ErrorListHelperModel edu.usf.cutr.gtfsrtvalidator.lib.model.helper.ErrorListHelperModel used on opensource projects

Example 6 with ErrorListHelperModel

use of edu.usf.cutr.gtfsrtvalidator.lib.model.helper.ErrorListHelperModel in project gtfs-realtime-validator by CUTR-at-USF.

the class BackgroundTask method validateEntity.

private StringBuffer validateEntity(long currentTimeMillis, GtfsRealtime.FeedMessage currentFeedMessage, GtfsRealtime.FeedMessage previousFeedMessage, GtfsRealtime.FeedMessage combinedFeedMessage, GtfsDaoImpl gtfsData, GtfsMetadata gtfsMetadata, GtfsRtFeedIterationModel feedIteration, FeedEntityValidator feedEntityValidator) {
    StringBuffer consoleLine = new StringBuffer();
    long startTimeNanos = System.nanoTime();
    List<ErrorListHelperModel> errorLists = feedEntityValidator.validate(currentTimeMillis, gtfsData, gtfsMetadata, currentFeedMessage, previousFeedMessage, combinedFeedMessage);
    consoleLine.append("\n" + feedEntityValidator.getClass().getSimpleName() + " - rule = " + getElapsedTimeString(getElapsedTime(startTimeNanos, System.nanoTime())));
    if (errorLists != null) {
        startTimeNanos = System.nanoTime();
        for (ErrorListHelperModel errorList : errorLists) {
            if (!errorList.getOccurrenceList().isEmpty()) {
                // Set iteration Id
                errorList.getErrorMessage().setGtfsRtFeedIterationModel(feedIteration);
                // Save the captured errors to the database
                DBHelper.saveError(errorList);
            }
        }
        consoleLine.append(", database = " + getElapsedTimeString(getElapsedTime(startTimeNanos, System.nanoTime())));
    }
    return consoleLine;
}

Example 7 with ErrorListHelperModel

use of edu.usf.cutr.gtfsrtvalidator.lib.model.helper.ErrorListHelperModel in project gtfs-realtime-validator by CUTR-at-USF.

the class StopTimeUpdateValidator method validate.

@Override
public List<ErrorListHelperModel> validate(long currentTimeMillis, GtfsDaoImpl gtfsData, GtfsMetadata gtfsMetadata, GtfsRealtime.FeedMessage feedMessage, GtfsRealtime.FeedMessage previousFeedMessage, GtfsRealtime.FeedMessage combinedFeedMessage) {
    List<GtfsRealtime.FeedEntity> entityList = feedMessage.getEntityList();
    List<OccurrenceModel> e002List = new ArrayList<>();
    List<OccurrenceModel> e009List = new ArrayList<>();
    List<OccurrenceModel> e036List = new ArrayList<>();
    List<OccurrenceModel> e037List = new ArrayList<>();
    List<OccurrenceModel> e040List = new ArrayList<>();
    List<OccurrenceModel> e041List = new ArrayList<>();
    List<OccurrenceModel> e042List = new ArrayList<>();
    List<OccurrenceModel> e043List = new ArrayList<>();
    List<OccurrenceModel> e044List = new ArrayList<>();
    List<OccurrenceModel> e045List = new ArrayList<>();
    List<OccurrenceModel> e046List = new ArrayList<>();
    List<OccurrenceModel> e051List = new ArrayList<>();
    for (GtfsRealtime.FeedEntity entity : entityList) {
        if (entity.hasTripUpdate()) {
            GtfsRealtime.TripUpdate tripUpdate = entity.getTripUpdate();
            checkE041(entity, tripUpdate, e041List);
            List<StopTime> gtfsStopTimes = null;
            int gtfsStopTimeIndex = 0;
            String tripId = null;
            if (tripUpdate.hasTrip() && tripUpdate.getTrip().hasTripId()) {
                tripId = tripUpdate.getTrip().getTripId();
                gtfsStopTimes = gtfsMetadata.getTripStopTimes().get(tripId);
            }
            List<GtfsRealtime.TripUpdate.StopTimeUpdate> rtStopTimeUpdateList = tripUpdate.getStopTimeUpdateList();
            List<Integer> rtStopSequenceList = new ArrayList<>();
            List<String> rtStopIdList = new ArrayList<>();
            Integer previousRtStopSequence = null;
            String previousRtStopId = null;
            boolean foundE009error = false;
            boolean addedStopSequenceFromStopId = false;
            Map<String, List<String>> tripWithMultiStop = gtfsMetadata.getTripsWithMultiStops();
            boolean unknownRtStopSequence = false;
            for (GtfsRealtime.TripUpdate.StopTimeUpdate stopTimeUpdate : rtStopTimeUpdateList) {
                if (!foundE009error && tripId != null && tripWithMultiStop.containsKey(tripId) && !stopTimeUpdate.hasStopSequence()) {
                    // E009 - GTFS-rt stop_sequence isn't provided for trip that visits same stop_id more than once
                    List<String> stopIds = tripWithMultiStop.get(tripId);
                    RuleUtils.addOccurrence(ValidationRules.E009, "trip_id " + tripId + " visits stop_id " + stopIds.toString(), e009List, _log);
                    // Only log error once for this trip
                    foundE009error = true;
                }
                if (previousRtStopSequence != null) {
                    checkE036(entity, previousRtStopSequence, stopTimeUpdate, e036List);
                }
                if (previousRtStopId != null) {
                    checkE037(entity, previousRtStopId, stopTimeUpdate, e037List);
                }
                previousRtStopSequence = stopTimeUpdate.getStopSequence();
                previousRtStopId = stopTimeUpdate.getStopId();
                if (stopTimeUpdate.hasStopSequence()) {
                    rtStopSequenceList.add(stopTimeUpdate.getStopSequence());
                }
                if (stopTimeUpdate.hasStopId()) {
                    rtStopIdList.add(stopTimeUpdate.getStopId());
                }
                if (gtfsStopTimes != null) {
                    // Loop through GTFS stop_time.txt to try and find a matching GTFS stop
                    while (gtfsStopTimeIndex < gtfsStopTimes.size()) {
                        int gtfsStopSequence = gtfsStopTimes.get(gtfsStopTimeIndex).getStopSequence();
                        Stop gtfsStop = gtfsStopTimes.get(gtfsStopTimeIndex).getStop();
                        boolean foundStopSequence = false;
                        boolean foundStopId = false;
                        if (stopTimeUpdate.hasStopSequence()) {
                            if (gtfsStopSequence == stopTimeUpdate.getStopSequence()) {
                                // Found a matching stop_sequence from GTFS stop_times.txt
                                checkE045(entity, tripUpdate, stopTimeUpdate, gtfsStopSequence, gtfsStop, e045List);
                                checkE046(entity, tripUpdate, stopTimeUpdate, gtfsStopTimes.get(gtfsStopTimeIndex), e046List);
                                foundStopSequence = true;
                            }
                        }
                        if (stopTimeUpdate.hasStopId()) {
                            if (gtfsStop.getId().getId().equals(stopTimeUpdate.getStopId())) {
                                /**
                                 * Found a matching stop_id - note that there could be loops in routes, so unlike
                                 * stop_sequence this isn't a definitive match between this stopTimeUpdate and a GTFS stop_times.txt entry
                                 */
                                foundStopId = true;
                            }
                        }
                        gtfsStopTimeIndex++;
                        if (foundStopSequence) {
                            // We caught up with the stop_sequence in GTFS data - stop so we can pick up from here in next WHILE loop
                            break;
                        } else {
                            if (stopTimeUpdate.hasStopSequence() && gtfsStopTimeIndex == gtfsStopTimes.size()) {
                                // For E051 - we've reached the last GTFS stop_times.txt record for the GTFS-rt stop_time_update and haven't found stop_sequence (#261)
                                unknownRtStopSequence = true;
                            }
                            if (!stopTimeUpdate.hasStopSequence() && foundStopId) {
                                // For E002 - in the case when stop_sequence is missing from the GTFS-rt feed, add the GTFS stop_sequence (See #159)
                                if (!stopTimeUpdate.hasStopSequence()) {
                                    rtStopSequenceList.add(gtfsStopSequence);
                                    addedStopSequenceFromStopId = true;
                                }
                                // E046 hasn't been checked yet if a stop_sequence doesn't exist - check now
                                checkE046(entity, tripUpdate, stopTimeUpdate, gtfsStopTimes.get(gtfsStopTimeIndex - 1), e046List);
                                // Note that for routes with loops, we could potentially be stopping prematurely
                                break;
                            }
                        }
                    }
                }
                checkE040(entity, tripUpdate, stopTimeUpdate, e040List);
                checkE042(entity, tripUpdate, stopTimeUpdate, e042List);
                checkE043(entity, tripUpdate, stopTimeUpdate, e043List);
                checkE044(entity, tripUpdate, stopTimeUpdate, e044List);
                if (unknownRtStopSequence) {
                    // E051 - GTFS-rt stop_sequence not found in GTFS data
                    RuleUtils.addOccurrence(ValidationRules.E051, "GTFS-rt " + GtfsUtils.getTripId(entity, tripUpdate) + " contains stop_sequence " + stopTimeUpdate.getStopSequence(), e051List, _log);
                    // the remaining stop_time_updates for this GTFS-rt trip will be validated.
                    break;
                }
            }
            boolean sorted = Ordering.natural().isStrictlyOrdered(rtStopSequenceList);
            if (!sorted) {
                // E002 - stop_time_updates for a given trip_id must be sorted by increasing stop_sequence
                String id = GtfsUtils.getTripId(entity, tripUpdate);
                RuleUtils.addOccurrence(ValidationRules.E002, id + " stop_sequence " + rtStopSequenceList.toString(), e002List, _log);
            } else if (addedStopSequenceFromStopId) {
                // TripUpdate was missing at least one stop_sequence
                if (rtStopSequenceList.size() < rtStopTimeUpdateList.size()) {
                    // We didn't find all of the stop_time_updates in GTFS using stop_id, so stop_time_updates are
                    // out of sequence
                    // E002 - stop_time_updates for a given trip_id must be sorted by increasing stop_sequence
                    String id = GtfsUtils.getTripId(entity, tripUpdate);
                    RuleUtils.addOccurrence(ValidationRules.E002, id + " stop_sequence for stop_ids " + rtStopIdList.toString(), e002List, _log);
                }
            }
        }
    }
    List<ErrorListHelperModel> errors = new ArrayList<>();
    if (!e002List.isEmpty()) {
        errors.add(new ErrorListHelperModel(new MessageLogModel(ValidationRules.E002), e002List));
    }
    if (!e009List.isEmpty()) {
        errors.add(new ErrorListHelperModel(new MessageLogModel(ValidationRules.E009), e009List));
    }
    if (!e036List.isEmpty()) {
        errors.add(new ErrorListHelperModel(new MessageLogModel(ValidationRules.E036), e036List));
    }
    if (!e037List.isEmpty()) {
        errors.add(new ErrorListHelperModel(new MessageLogModel(ValidationRules.E037), e037List));
    }
    if (!e040List.isEmpty()) {
        errors.add(new ErrorListHelperModel(new MessageLogModel(ValidationRules.E040), e040List));
    }
    if (!e041List.isEmpty()) {
        errors.add(new ErrorListHelperModel(new MessageLogModel(ValidationRules.E041), e041List));
    }
    if (!e042List.isEmpty()) {
        errors.add(new ErrorListHelperModel(new MessageLogModel(ValidationRules.E042), e042List));
    }
    if (!e043List.isEmpty()) {
        errors.add(new ErrorListHelperModel(new MessageLogModel(ValidationRules.E043), e043List));
    }
    if (!e044List.isEmpty()) {
        errors.add(new ErrorListHelperModel(new MessageLogModel(ValidationRules.E044), e044List));
    }
    if (!e045List.isEmpty()) {
        errors.add(new ErrorListHelperModel(new MessageLogModel(ValidationRules.E045), e045List));
    }
    if (!e046List.isEmpty()) {
        errors.add(new ErrorListHelperModel(new MessageLogModel(ValidationRules.E046), e046List));
    }
    if (!e051List.isEmpty()) {
        errors.add(new ErrorListHelperModel(new MessageLogModel(ValidationRules.E051), e051List));
    }
    return errors;
}

Example 8 with ErrorListHelperModel

use of edu.usf.cutr.gtfsrtvalidator.lib.model.helper.ErrorListHelperModel in project gtfs-realtime-validator by CUTR-at-USF.

the class TimestampValidator method validate.

@Override
public List<ErrorListHelperModel> validate(long currentTimeMillis, GtfsDaoImpl gtfsData, GtfsMetadata gtfsMetadata, GtfsRealtime.FeedMessage feedMessage, GtfsRealtime.FeedMessage previousFeedMessage, GtfsRealtime.FeedMessage combinedFeedMessage) {
    if (feedMessage.equals(previousFeedMessage)) {
        throw new IllegalArgumentException("feedMessage and previousFeedMessage must not be the same");
    }
    List<OccurrenceModel> w001List = new ArrayList<>();
    List<OccurrenceModel> w007List = new ArrayList<>();
    List<OccurrenceModel> w008List = new ArrayList<>();
    List<OccurrenceModel> e001List = new ArrayList<>();
    List<OccurrenceModel> e012List = new ArrayList<>();
    List<OccurrenceModel> e017List = new ArrayList<>();
    List<OccurrenceModel> e018List = new ArrayList<>();
    List<OccurrenceModel> e022List = new ArrayList<>();
    List<OccurrenceModel> e025List = new ArrayList<>();
    List<OccurrenceModel> e048List = new ArrayList<>();
    List<OccurrenceModel> e050List = new ArrayList<>();
    String currentTimeText = TimestampUtils.posixToClock(TimeUnit.MILLISECONDS.toSeconds(currentTimeMillis), gtfsMetadata.getTimeZone());
    /**
     * Validate FeedHeader timestamp
     */
    long headerTimestamp = feedMessage.getHeader().getTimestamp();
    if (headerTimestamp == 0) {
        boolean isV2orHigher = true;
        try {
            isV2orHigher = GtfsUtils.isV2orHigher(feedMessage.getHeader());
        } catch (Exception e) {
            _log.error("Error checking header version for E048/W001, logging as E048: " + e);
        }
        if (isV2orHigher) {
            // E048 - header timestamp not populated
            RuleUtils.addOccurrence(E048, "", e048List, _log);
        } else {
            // W001 - Timestamp not populated
            RuleUtils.addOccurrence(W001, "header", w001List, _log);
        }
    } else {
        if (!isPosix(headerTimestamp)) {
            // E001 - Not in POSIX time
            RuleUtils.addOccurrence(E001, "header.timestamp", e001List, _log);
        } else {
            long ageMillis = getAge(currentTimeMillis, headerTimestamp);
            long ageMinutes = TimeUnit.MILLISECONDS.toMinutes(ageMillis);
            long ageSeconds = TimeUnit.MILLISECONDS.toSeconds(ageMillis);
            if (ageMillis > TimeUnit.SECONDS.toMillis(MAX_AGE_SECONDS)) {
                // W008 - Header timestamp is older than 65 seconds
                RuleUtils.addOccurrence(W008, "header.timestamp is " + ageMinutes + " min " + ageSeconds % 60 + " sec", w008List, _log);
            }
            if (TimestampUtils.isInFuture(currentTimeMillis, headerTimestamp, IN_FUTURE_TOLERANCE_SECONDS)) {
                // E050 - timestamp is in the future
                String headerTimestampText = TimestampUtils.posixToClock(headerTimestamp, gtfsMetadata.getTimeZone());
                RuleUtils.addOccurrence(E050, "header.timestamp " + headerTimestampText + " (" + headerTimestamp + ") is " + Math.abs(ageMinutes) + " min " + Math.abs(ageSeconds) % 60 + " sec greater than " + currentTimeText + " (" + currentTimeMillis + ")", e050List, _log);
            }
        }
        if (previousFeedMessage != null && previousFeedMessage.getHeader().getTimestamp() != 0) {
            long previousTimestamp = previousFeedMessage.getHeader().getTimestamp();
            long interval = headerTimestamp - previousTimestamp;
            if (headerTimestamp == previousTimestamp) {
                // E017 - GTFS-rt content changed but has the same timestamp
                RuleUtils.addOccurrence(E017, "header.timestamp of " + headerTimestamp, e017List, _log);
            } else if (headerTimestamp < previousTimestamp) {
                // E018 - GTFS-rt header timestamp decreased between two sequential iterations
                String prefix = "header.timestamp of " + headerTimestamp + " is less than the header.timestamp of " + previousFeedMessage.getHeader().getTimestamp();
                RuleUtils.addOccurrence(E018, prefix, e018List, _log);
            } else if (interval > MINIMUM_REFRESH_INTERVAL_SECONDS) {
                // W007 - Refresh interval is more than 35 seconds
                RuleUtils.addOccurrence(W007, interval + " second interval between consecutive header.timestamps", w007List, _log);
            }
        }
    }
    for (GtfsRealtime.FeedEntity entity : feedMessage.getEntityList()) {
        if (entity.hasTripUpdate()) {
            GtfsRealtime.TripUpdate tripUpdate = entity.getTripUpdate();
            long tripUpdateTimestamp = tripUpdate.getTimestamp();
            /**
             * Validate TripUpdate timestamps
             */
            String id = GtfsUtils.getTripId(entity, tripUpdate);
            if (tripUpdateTimestamp == 0) {
                // W001 - Timestamp not populated
                RuleUtils.addOccurrence(W001, id, w001List, _log);
            } else {
                if (headerTimestamp != 0 && tripUpdateTimestamp > headerTimestamp) {
                    // E012 - Header timestamp should be greater than or equal to all other timestamps
                    RuleUtils.addOccurrence(E012, id + " timestamp " + tripUpdateTimestamp, e012List, _log);
                }
                if (!isPosix(tripUpdateTimestamp)) {
                    // E001 - Not in POSIX time
                    RuleUtils.addOccurrence(E001, id + " timestamp " + tripUpdateTimestamp, e001List, _log);
                } else {
                    if (TimestampUtils.isInFuture(currentTimeMillis, tripUpdateTimestamp, IN_FUTURE_TOLERANCE_SECONDS)) {
                        // E050 - timestamp is in the future
                        long ageMillis = getAge(currentTimeMillis, tripUpdateTimestamp);
                        long ageMinutes = Math.abs(TimeUnit.MILLISECONDS.toMinutes(ageMillis));
                        long ageSeconds = Math.abs(TimeUnit.MILLISECONDS.toSeconds(ageMillis));
                        String tripUpdateTimestampText = TimestampUtils.posixToClock(tripUpdateTimestamp, gtfsMetadata.getTimeZone());
                        RuleUtils.addOccurrence(E050, id + " timestamp " + tripUpdateTimestampText + " (" + tripUpdateTimestamp + ") is " + ageMinutes + " min " + ageSeconds % 60 + " sec greater than " + currentTimeText + " (" + currentTimeMillis + ")", e050List, _log);
                    }
                }
            }
            /**
             * Validate TripUpdate StopTimeUpdate times
             */
            List<GtfsRealtime.TripUpdate.StopTimeUpdate> stopTimeUpdates = tripUpdate.getStopTimeUpdateList();
            if (stopTimeUpdates != null) {
                Long previousArrivalTime = null;
                String previousArrivalTimeText = null;
                Long previousDepartureTime = null;
                String previousDepartureTimeText = null;
                for (GtfsRealtime.TripUpdate.StopTimeUpdate stopTimeUpdate : stopTimeUpdates) {
                    String stopDescription = stopTimeUpdate.hasStopSequence() ? " stop_sequence " + stopTimeUpdate.getStopSequence() : " stop_id " + stopTimeUpdate.getStopId();
                    Long arrivalTime = null;
                    String arrivalTimeText;
                    Long departureTime = null;
                    String departureTimeText;
                    if (stopTimeUpdate.hasArrival()) {
                        if (stopTimeUpdate.getArrival().hasTime()) {
                            arrivalTime = stopTimeUpdate.getArrival().getTime();
                            arrivalTimeText = TimestampUtils.posixToClock(arrivalTime, gtfsMetadata.getTimeZone());
                            if (!isPosix(arrivalTime)) {
                                // E001 - Not in POSIX time
                                RuleUtils.addOccurrence(E001, id + stopDescription + " arrival_time " + arrivalTime, e001List, _log);
                            }
                            if (previousArrivalTime != null && arrivalTime < previousArrivalTime) {
                                // E022 - this stop arrival time is < previous stop arrival time
                                String prefix = id + stopDescription + " arrival_time " + arrivalTimeText + " (" + arrivalTime + ") is less than previous stop arrival_time " + previousArrivalTimeText + " (" + previousArrivalTime + ")";
                                RuleUtils.addOccurrence(E022, prefix, e022List, _log);
                            }
                            if (previousArrivalTime != null && Objects.equals(arrivalTime, previousArrivalTime)) {
                                // E022 - this stop arrival time is == previous stop arrival time
                                String prefix = id + stopDescription + " arrival_time " + arrivalTimeText + " (" + arrivalTime + ") is equal to previous stop arrival_time " + previousArrivalTimeText + " (" + previousArrivalTime + ")";
                                RuleUtils.addOccurrence(E022, prefix, e022List, _log);
                            }
                            if (previousDepartureTime != null && arrivalTime < previousDepartureTime) {
                                // E022 - this stop arrival time is < previous stop departure time
                                String prefix = id + stopDescription + " arrival_time " + arrivalTimeText + " (" + arrivalTime + ") is less than previous stop departure_time " + previousDepartureTimeText + " (" + previousDepartureTime + ")";
                                RuleUtils.addOccurrence(E022, prefix, e022List, _log);
                            }
                            if (previousDepartureTime != null && Objects.equals(arrivalTime, previousDepartureTime)) {
                                // E022 - this stop arrival time is == previous stop departure time
                                String prefix = id + stopDescription + " arrival_time " + arrivalTimeText + " (" + arrivalTime + ") is equal to previous stop departure_time " + previousDepartureTimeText + " (" + previousDepartureTime + ")";
                                RuleUtils.addOccurrence(E022, prefix, e022List, _log);
                            }
                        }
                    }
                    if (stopTimeUpdate.hasDeparture()) {
                        if (stopTimeUpdate.getDeparture().hasTime()) {
                            departureTime = stopTimeUpdate.getDeparture().getTime();
                            departureTimeText = TimestampUtils.posixToClock(departureTime, gtfsMetadata.getTimeZone());
                            if (!isPosix(departureTime)) {
                                // E001 - Not in POSIX time
                                RuleUtils.addOccurrence(E001, id + stopDescription + " departure_time " + departureTime, e001List, _log);
                            }
                            if (previousDepartureTime != null && departureTime < previousDepartureTime) {
                                // E022 - this stop departure time is < previous stop departure time
                                String prefix = id + stopDescription + " departure_time " + departureTimeText + " (" + departureTime + ") is less than previous stop departure_time " + previousDepartureTimeText + " (" + previousDepartureTime + ")";
                                RuleUtils.addOccurrence(E022, prefix, e022List, _log);
                            }
                            if (previousDepartureTime != null && Objects.equals(departureTime, previousDepartureTime)) {
                                // E022 - this stop departure time is == previous stop departure time
                                String prefix = id + stopDescription + " departure_time " + departureTimeText + " (" + departureTime + ") is equal to previous stop departure_time " + previousDepartureTimeText + " (" + previousDepartureTime + ")";
                                RuleUtils.addOccurrence(E022, prefix, e022List, _log);
                            }
                            if (previousArrivalTime != null && departureTime < previousArrivalTime) {
                                // E022 - this stop departure time is < previous stop arrival time
                                String prefix = id + stopDescription + " departure_time " + departureTimeText + " (" + departureTime + ") is less than previous stop arrival_time " + previousArrivalTimeText + " (" + previousArrivalTime + ")";
                                RuleUtils.addOccurrence(E022, prefix, e022List, _log);
                            }
                            if (previousArrivalTime != null && Objects.equals(departureTime, previousArrivalTime)) {
                                // E022 - this stop departure time is == previous stop arrival time
                                String prefix = id + stopDescription + " departure_time " + departureTimeText + " (" + departureTime + ") is equal to previous stop arrival_time " + previousArrivalTimeText + " (" + previousArrivalTime + ")";
                                RuleUtils.addOccurrence(E022, prefix, e022List, _log);
                            }
                            if (stopTimeUpdate.getArrival().hasTime() && departureTime < stopTimeUpdate.getArrival().getTime()) {
                                // E025 - stop_time_update departure time is before arrival time
                                String prefix = id + stopDescription + " departure_time " + departureTimeText + " (" + departureTime + ") is less than the same stop arrival_time " + TimestampUtils.posixToClock(stopTimeUpdate.getArrival().getTime(), gtfsMetadata.getTimeZone()) + " (" + stopTimeUpdate.getArrival().getTime() + ")";
                                RuleUtils.addOccurrence(E025, prefix, e025List, _log);
                            }
                        }
                    }
                    if (arrivalTime != null) {
                        previousArrivalTime = arrivalTime;
                        previousArrivalTimeText = TimestampUtils.posixToClock(previousArrivalTime, gtfsMetadata.getTimeZone());
                    }
                    if (departureTime != null) {
                        previousDepartureTime = departureTime;
                        previousDepartureTimeText = TimestampUtils.posixToClock(previousDepartureTime, gtfsMetadata.getTimeZone());
                    }
                }
            }
        }
        if (entity.hasVehicle()) {
            GtfsRealtime.VehiclePosition vehiclePosition = entity.getVehicle();
            long vehicleTimestamp = vehiclePosition.getTimestamp();
            if (vehicleTimestamp == 0) {
                // W001 - Timestamp not populated
                RuleUtils.addOccurrence(W001, "vehicle_id " + vehiclePosition.getVehicle().getId(), w001List, _log);
            } else {
                String prefix = "vehicle_id " + vehiclePosition.getVehicle().getId() + " timestamp " + vehicleTimestamp;
                if (headerTimestamp != 0 && vehicleTimestamp > headerTimestamp) {
                    // E012 - Header timestamp should be greater than or equal to all other timestamps
                    RuleUtils.addOccurrence(E012, prefix, e012List, _log);
                }
                if (!isPosix(vehicleTimestamp)) {
                    // E001 - Not in POSIX time
                    RuleUtils.addOccurrence(E001, prefix, e001List, _log);
                } else {
                    if (TimestampUtils.isInFuture(currentTimeMillis, vehicleTimestamp, IN_FUTURE_TOLERANCE_SECONDS)) {
                        // E050 - timestamp is in the future
                        long ageMillis = getAge(currentTimeMillis, vehicleTimestamp);
                        long ageMinutes = Math.abs(TimeUnit.MILLISECONDS.toMinutes(ageMillis));
                        long ageSeconds = Math.abs(TimeUnit.MILLISECONDS.toSeconds(ageMillis));
                        String vehicleTimestampText = TimestampUtils.posixToClock(vehicleTimestamp, gtfsMetadata.getTimeZone());
                        RuleUtils.addOccurrence(E050, "vehicle_id " + vehiclePosition.getVehicle().getId() + " timestamp " + vehicleTimestampText + " (" + vehicleTimestamp + ") is " + ageMinutes + " min " + ageSeconds % 60 + " sec greater than " + currentTimeText + " (" + currentTimeMillis + ")", e050List, _log);
                    }
                }
            }
        }
        if (entity.hasAlert()) {
            checkAlertE001(entity, e001List);
        }
    }
    List<ErrorListHelperModel> errors = new ArrayList<>();
    if (!w001List.isEmpty()) {
        errors.add(new ErrorListHelperModel(new MessageLogModel(W001), w001List));
    }
    if (!w007List.isEmpty()) {
        errors.add(new ErrorListHelperModel(new MessageLogModel(W007), w007List));
    }
    if (!w008List.isEmpty()) {
        errors.add(new ErrorListHelperModel(new MessageLogModel(W008), w008List));
    }
    if (!e001List.isEmpty()) {
        errors.add(new ErrorListHelperModel(new MessageLogModel(E001), e001List));
    }
    if (!e012List.isEmpty()) {
        errors.add(new ErrorListHelperModel(new MessageLogModel(E012), e012List));
    }
    if (!e017List.isEmpty()) {
        errors.add(new ErrorListHelperModel(new MessageLogModel(E017), e017List));
    }
    if (!e018List.isEmpty()) {
        errors.add(new ErrorListHelperModel(new MessageLogModel(E018), e018List));
    }
    if (!e022List.isEmpty()) {
        errors.add(new ErrorListHelperModel(new MessageLogModel(E022), e022List));
    }
    if (!e025List.isEmpty()) {
        errors.add(new ErrorListHelperModel(new MessageLogModel(E025), e025List));
    }
    if (!e048List.isEmpty()) {
        errors.add(new ErrorListHelperModel(new MessageLogModel(E048), e048List));
    }
    if (!e050List.isEmpty()) {
        errors.add(new ErrorListHelperModel(new MessageLogModel(E050), e050List));
    }
    return errors;
}

Example 9 with ErrorListHelperModel

use of edu.usf.cutr.gtfsrtvalidator.lib.model.helper.ErrorListHelperModel in project gtfs-realtime-validator by CUTR-at-USF.

the class BatchProcessor method processFeeds.

/**
 * Process the GTFS and GTFS-realtime feeds provided in the constructor. If setReturnStatistics() is set to true,
 * the method will return a list of IterationStatistics (one per GTFS-rt file) for performance in the batch
 * validation.  By default this method will return null to avoid memory issues with extremely large batch processes.
 *
 * @return If setReturnStatistics() is set to true, it will return a list of IterationStatistics (one per GTFS-rt
 * file) for performance in the batch validation.  By default this method will return null to avoid memory issues
 * when processing an extremely large number of files.
 * @throws NoSuchAlgorithmException If the MD5 hash algorithm (used to determine feed uniqueness) is not available on the machine executing the code
 * @throws IOException              If the GTFS or GTFS-realtime files cannot be read or the results cannot be written to disk
 */
public List<IterationStatistics> processFeeds() throws NoSuchAlgorithmException, IOException {
    // Read GTFS data into a GtfsDaoImpl
    _log.info("Starting batch processor...");
    if (mReturnStatistics) {
        mIterationStatistics = new ArrayList<>();
    }
    String timeZoneText = null;
    double gtfsReadTime = readGtfsData();
    Collection<Agency> agencies = mGtfsData.getAllAgencies();
    for (Agency agency : agencies) {
        timeZoneText = agency.getTimezone();
        break;
    }
    mGtfsMetadata = new GtfsMetadata(mPathToGtfsFile.getAbsolutePath(), TimeZone.getTimeZone(timeZoneText), mGtfsData, mIgnoreShapes);
    // Initialize validation rules
    synchronized (mValidationRules) {
        if (mValidationRules.isEmpty()) {
            mValidationRules.add(new CrossFeedDescriptorValidator());
            mValidationRules.add(new VehicleValidator());
            mValidationRules.add(new TimestampValidator());
            mValidationRules.add(new StopTimeUpdateValidator());
            mValidationRules.add(new TripDescriptorValidator());
            mValidationRules.add(new StopValidator());
            mValidationRules.add(new FrequencyTypeZeroValidator());
            mValidationRules.add(new FrequencyTypeOneValidator());
            mValidationRules.add(new HeaderValidator());
        }
    }
    // Configure output
    ObjectMapper mapper = new ObjectMapper();
    mapper.enable(SerializationFeature.INDENT_OUTPUT);
    _log.info("Sorting GTFS-rt files by " + mSortBy.name() + "...");
    // Read GTFS-rt protobuf files from provided directory
    List<Path> paths = Files.walk(Paths.get(mPathToGtfsRealtime)).filter(Files::isRegularFile).sorted((o1, o2) -> {
        if (mSortBy.equals(SortBy.DATE_MODIFIED)) {
            try {
                // Sort by date modified (ascending) (it seems more consistent cross-platform than "date created")
                return SortUtils.compareByDateModified(o1, o2);
            } catch (IOException e) {
                _log.error("Can't sort GTFS-rt files by date - assuming dates are equal: " + e);
            }
            // Assume file dates are equal if we get an exception
            return 0;
        } else {
            // Sort by name (ascending)
            return SortUtils.compareByFileName(o1, o2);
        }
    }).collect(Collectors.toList());
    MessageDigest md = MessageDigest.getInstance("MD5");
    GtfsRealtime.FeedMessage prevMessage = null;
    byte[] prevHash = null;
    for (Path path : paths) {
        IterationStatistics stats = null;
        if (mReturnStatistics) {
            stats = new IterationStatistics();
            stats.setGtfsReadTime(gtfsReadTime);
        }
        long startTimeNanos = System.nanoTime();
        long startToByteArray = System.nanoTime();
        byte[] protobuf;
        try {
            protobuf = IOUtils.toByteArray(Files.newInputStream(path));
        } catch (IOException e) {
            _log.error("Error reading GTFS-rt file to byte array, skipping to next file: " + e);
            continue;
        }
        double toByteArray = getElapsedTime(startToByteArray, System.nanoTime());
        _log.info("Read " + path.getFileName() + " to byte array in " + getElapsedTimeString(toByteArray));
        if (mReturnStatistics) {
            stats.setToByteArrayTime(toByteArray);
        }
        byte[] currentHash = md.digest(protobuf);
        if (MessageDigest.isEqual(currentHash, prevHash)) {
            // This feed file is a duplicate of the last one - skip to next file
            continue;
        }
        long timestamp;
        if (mSortBy.equals(SortBy.DATE_MODIFIED)) {
            // Use file last modified date as "current" timestamp
            timestamp = Files.getLastModifiedTime(path).toMillis();
        } else {
            // Use time parsed from file name as "current" timestamp
            try {
                timestamp = TimestampUtils.getTimestampFromFileName(path.toFile().getName());
            } catch (DateTimeParseException | StringIndexOutOfBoundsException e) {
                _log.error("Couldn't parse timestamp from file name '" + path.toFile().getName() + "' - using date modified instead: " + e);
                timestamp = Files.getLastModifiedTime(path).toMillis();
            }
        }
        long startProtobufDecode = System.nanoTime();
        GtfsRealtime.FeedMessage message;
        try {
            message = GtfsRealtime.FeedMessage.parseFrom(protobuf);
        } catch (InvalidProtocolBufferException e) {
            _log.error("Error reading GTFS-rt message from byte array, skipping to next file: " + e);
            continue;
        }
        double pbDecode = getElapsedTime(startProtobufDecode, System.nanoTime());
        _log.info("Decoded " + path.getFileName() + " protobuf in " + getElapsedTimeString(pbDecode));
        if (mReturnStatistics) {
            stats.setDecodeProtobufTime(pbDecode);
        }
        GtfsRealtime.FeedMessage combinedMessage = null;
        // See if more than one entity type exists in this feed
        if (GtfsUtils.isCombinedFeed(message)) {
            // Run CrossFeedDescriptorValidator on this message
            combinedMessage = message;
        }
        List<ErrorListHelperModel> allErrorLists = new ArrayList<>();
        StringBuilder consoleOutput = new StringBuilder();
        List<RuleStatistics> ruleStatistics = null;
        if (mReturnStatistics) {
            ruleStatistics = new ArrayList<>();
        }
        for (FeedEntityValidator rule : mValidationRules) {
            long startRuleNanos = System.nanoTime();
            List<ErrorListHelperModel> errorLists = rule.validate(timestamp, mGtfsData, mGtfsMetadata, message, prevMessage, combinedMessage);
            allErrorLists.addAll(errorLists);
            double ruleExecutionTime = getElapsedTime(startRuleNanos, System.nanoTime());
            consoleOutput.append("\n" + rule.getClass().getSimpleName() + " - rule = " + getElapsedTimeString(ruleExecutionTime));
            if (mReturnStatistics) {
                RuleStatistics ruleStat = new RuleStatistics();
                ruleStat.setRuleExecutionTime(ruleExecutionTime);
                ruleStat.setValidator(rule.getClass().getSimpleName());
                ruleStatistics.add(ruleStat);
            }
        }
        double totalIterationTime = getElapsedTime(startTimeNanos, System.nanoTime());
        consoleOutput.append("\nProcessed " + path.getFileName() + " in " + getElapsedTimeString(totalIterationTime));
        consoleOutput.append("\n---------------------");
        _log.info(consoleOutput.toString());
        if (mReturnStatistics) {
            stats.setRuleStatistics(ruleStatistics);
            stats.setTotalIterationTime(totalIterationTime);
        }
        // Write validation results for this file to JSON
        writeResults(mapper, path, allErrorLists);
        if (mPlainTextExtension != null) {
            // Write plain text version of protocol buffer
            writePlainText(message, mapper, path);
        }
        if (mReturnStatistics) {
            mIterationStatistics.add(stats);
        }
        prevHash = currentHash;
        prevMessage = message;
    }
    return mIterationStatistics;
}

Example 10 with ErrorListHelperModel

use of edu.usf.cutr.gtfsrtvalidator.lib.model.helper.ErrorListHelperModel in project gtfs-realtime-validator by CUTR-at-USF.

the class UtilTest method testAssertResultsThrowExceptionMoreActual.

@Test(expected = AssertionError.class)
public void testAssertResultsThrowExceptionMoreActual() {
    // Make sure we fail if we have actual results that weren't expected
    List<ErrorListHelperModel> results = new ArrayList<>();
    MessageLogModel modelE001 = new MessageLogModel(E001);
    OccurrenceModel errorE001 = new OccurrenceModel(String.valueOf(MIN_POSIX_TIME));
    List<OccurrenceModel> errorListE001 = new ArrayList<>();
    errorListE001.add(errorE001);
    results.add(new ErrorListHelperModel(modelE001, errorListE001));
    Map<ValidationRule, Integer> expected = new HashMap<>();
    // No expected results included for E001, but there is one actual error for E001 - this should throw an AssertionError
    TestUtils.assertResults(expected, results);
}