Search in sources :

Example 1 with GtfsRtFeedModel

use of edu.usf.cutr.gtfsrtvalidator.lib.model.GtfsRtFeedModel in project gtfs-realtime-validator by CUTR-at-USF.

the class BackgroundTask method run.

@Override
public void run() {
    try {
        long startTimeNanos = System.nanoTime();
        GtfsRealtime.FeedMessage currentFeedMessage;
        GtfsRealtime.FeedMessage previousFeedMessage = null;
        GtfsDaoImpl gtfsData;
        GtfsMetadata gtfsMetadata;
        // Holds data needed in the database under each iteration
        GtfsRtFeedIterationModel feedIteration;
        StringBuffer consoleOutput = new StringBuffer();
        // Get the GTFS feed from the GtfsDaoMap using the gtfsFeedId of the current feed.
        gtfsData = GtfsFeed.GtfsDaoMap.get(mCurrentGtfsRtFeed.getGtfsFeedModel().getFeedId());
        // Create the GTFS metadata if it doesn't already exist
        // TODO - read ignoreShapes from website checkbox - see https://github.com/CUTR-at-USF/gtfs-realtime-validator/issues/286
        gtfsMetadata = mGtfsMetadata.computeIfAbsent(mCurrentGtfsRtFeed.getGtfsFeedModel().getFeedId(), k -> new GtfsMetadata(mCurrentGtfsRtFeed.getGtfsFeedModel().getGtfsUrl(), TimeZone.getTimeZone(mCurrentGtfsRtFeed.getGtfsFeedModel().getAgency()), gtfsData, mCurrentGtfsRtFeed.getEnableShapes()));
        // Read the GTFS-rt feed from the feed URL
        URL gtfsRtFeedUrl;
        Session session;
        try {
            gtfsRtFeedUrl = new URL(mCurrentGtfsRtFeed.getGtfsRtUrl());
        } catch (MalformedURLException e) {
            _log.error("Malformed Url: " + mCurrentGtfsRtFeed.getGtfsRtUrl(), e);
            e.printStackTrace();
            return;
        }
        try {
            // Get the GTFS-RT feedMessage for this method
            long startHttpRequest = System.nanoTime();
            InputStream in = gtfsRtFeedUrl.openStream();
            consoleOutput.append("\n" + mCurrentGtfsRtFeed.getGtfsRtUrl() + " gtfsRtFeedUrl.openStream() in " + getElapsedTimeString(getElapsedTime(startHttpRequest, System.nanoTime())));
            long startToByteArray = System.nanoTime();
            byte[] gtfsRtProtobuf = IOUtils.toByteArray(in);
            consoleOutput.append("\n" + mCurrentGtfsRtFeed.getGtfsRtUrl() + " IOUtils.toByteArray(in) in " + getElapsedTimeString(getElapsedTime(startToByteArray, System.nanoTime())));
            boolean isUniqueFeed = true;
            MessageDigest md = MessageDigest.getInstance("MD5");
            byte[] prevFeedDigest = null;
            byte[] currentFeedDigest = md.digest(gtfsRtProtobuf);
            session = GTFSDB.initSessionBeginTrans();
            feedIteration = (GtfsRtFeedIterationModel) session.createQuery("FROM GtfsRtFeedIterationModel" + " WHERE rtFeedId = :gtfsRtId" + " ORDER BY IterationId DESC").setParameter("gtfsRtId", mCurrentGtfsRtFeed.getGtfsRtId()).setMaxResults(1).uniqueResult();
            if (feedIteration != null) {
                prevFeedDigest = feedIteration.getFeedHash();
            }
            if (MessageDigest.isEqual(currentFeedDigest, prevFeedDigest)) {
                // If previous feed digest and newly fetched/current feed digest are equal means, we received the same feed again.
                isUniqueFeed = false;
            }
            long startProtobufDecode = System.nanoTime();
            currentFeedMessage = GtfsRealtime.FeedMessage.parseFrom(gtfsRtProtobuf);
            consoleOutput.append("\n" + mCurrentGtfsRtFeed.getGtfsRtUrl() + " protobuf decode in " + getElapsedTimeString(getElapsedTime(startProtobufDecode, System.nanoTime())));
            _log.info(consoleOutput.toString());
            // Clear the buffer for the next set of log statements
            consoleOutput.setLength(0);
            long feedTimestamp = TimeUnit.SECONDS.toMillis(currentFeedMessage.getHeader().getTimestamp());
            // Create new feedIteration object and save the iteration to the database
            if (isUniqueFeed) {
                if (feedIteration != null && feedIteration.getFeedprotobuf() != null) {
                    // Get the previous feed message
                    InputStream previousIs = new ByteArrayInputStream(feedIteration.getFeedprotobuf());
                    previousFeedMessage = GtfsRealtime.FeedMessage.parseFrom(previousIs);
                }
                feedIteration = new GtfsRtFeedIterationModel(System.currentTimeMillis(), feedTimestamp, gtfsRtProtobuf, mCurrentGtfsRtFeed, currentFeedDigest);
            } else {
                feedIteration = new GtfsRtFeedIterationModel(System.currentTimeMillis(), feedTimestamp, null, mCurrentGtfsRtFeed, currentFeedDigest);
            }
            session.save(feedIteration);
            GTFSDB.commitAndCloseSession(session);
            if (!isUniqueFeed) {
                return;
            }
        } catch (Exception e) {
            _log.error("The URL '" + gtfsRtFeedUrl + "' does not contain valid Gtfs-Rt data", e);
            return;
        }
        // Read all GTFS-rt entities for the current feed
        mGtfsRtFeedMap.put(feedIteration.getGtfsRtFeedModel().getGtfsRtId(), currentFeedMessage);
        session = GTFSDB.initSessionBeginTrans();
        List<GtfsRealtime.FeedEntity> allEntitiesArrayList = new ArrayList<>();
        List<GtfsRtFeedModel> gtfsRtFeedModelList;
        gtfsRtFeedModelList = session.createQuery("FROM GtfsRtFeedModel" + " WHERE gtfsFeedID = :feedID").setParameter("feedID", mCurrentGtfsRtFeed.getGtfsFeedModel().getFeedId()).list();
        GTFSDB.closeSession(session);
        while (!mGtfsRtFeedMap.keySet().containsAll(gtfsRtFeedModelList.stream().map(GtfsRtFeedModel::getGtfsRtId).collect(Collectors.toSet()))) {
            Thread.sleep(200);
        }
        GtfsRealtime.FeedHeader header = null;
        if (gtfsRtFeedModelList.size() < 1) {
            _log.error("The URL '" + gtfsRtFeedUrl + "' is not stored properly into the database");
            return;
        }
        GtfsRealtime.FeedMessage combinedFeed = null;
        if (gtfsRtFeedModelList.size() == 1) {
            // See if more than one entity type exists in this feed
            GtfsRealtime.FeedMessage message = mGtfsRtFeedMap.get(gtfsRtFeedModelList.get(0).getGtfsRtId());
            if (GtfsUtils.isCombinedFeed(message)) {
                // Run CrossFeedDescriptorValidator on this message
                combinedFeed = message;
            }
        }
        if (gtfsRtFeedModelList.size() > 1) {
            // We're monitoring multiple GTFS-rt feeds for the same GTFS data - create a combined feed message include all entities for all of those GTFS-rt feeds
            _log.debug("Creating combined feed message for " + gtfsRtFeedModelList.toString());
            for (GtfsRtFeedModel gtfsRtFeedModel : gtfsRtFeedModelList) {
                GtfsRealtime.FeedMessage message = mGtfsRtFeedMap.get(gtfsRtFeedModel.getGtfsRtId());
                if (header == null) {
                    // Save one header to use in our combined feed below
                    header = message.getHeader();
                } else {
                    if (message.getHeader() != null && message.getHeader().getTimestamp() > header.getTimestamp()) {
                        // Use largest header timestamp with multiple feeds - see #239
                        header = message.getHeader();
                    }
                }
                if (message != null) {
                    allEntitiesArrayList.addAll(message.getEntityList());
                }
            }
            GtfsRealtime.FeedMessage.Builder feedMessageBuilder = GtfsRealtime.FeedMessage.newBuilder();
            feedMessageBuilder.setHeader(header);
            feedMessageBuilder.addAllEntity(allEntitiesArrayList);
            combinedFeed = feedMessageBuilder.build();
        }
        // Use the same current time for all rules for consistency
        long currentTimeMillis = System.currentTimeMillis();
        // Run validation rules
        for (FeedEntityValidator rule : mValidationRules) {
            consoleOutput.append(validateEntity(currentTimeMillis, currentFeedMessage, previousFeedMessage, combinedFeed, gtfsData, gtfsMetadata, feedIteration, rule));
        }
        consoleOutput.append("\nProcessed " + mCurrentGtfsRtFeed.getGtfsRtUrl() + " in " + getElapsedTimeString(getElapsedTime(startTimeNanos, System.nanoTime())));
        consoleOutput.append("\n---------------------");
        _log.info(consoleOutput.toString());
    } catch (Exception ex) {
        ex.printStackTrace();
    }
}
Also used : GtfsUtils(edu.usf.cutr.gtfsrtvalidator.lib.util.GtfsUtils) edu.usf.cutr.gtfsrtvalidator.lib.validation.rules(edu.usf.cutr.gtfsrtvalidator.lib.validation.rules) ErrorListHelperModel(edu.usf.cutr.gtfsrtvalidator.lib.model.helper.ErrorListHelperModel) DBHelper(edu.usf.cutr.gtfsrtvalidator.helper.DBHelper) URL(java.net.URL) MessageDigest(java.security.MessageDigest) LoggerFactory(org.slf4j.LoggerFactory) Session(org.hibernate.Session) TimestampUtils.getElapsedTime(edu.usf.cutr.gtfsrtvalidator.lib.util.TimestampUtils.getElapsedTime) GtfsRtFeedIterationModel(edu.usf.cutr.gtfsrtvalidator.lib.model.GtfsRtFeedIterationModel) GTFSDB(edu.usf.cutr.gtfsrtvalidator.db.GTFSDB) ArrayList(java.util.ArrayList) ByteArrayInputStream(java.io.ByteArrayInputStream) Map(java.util.Map) GtfsRealtime(com.google.transit.realtime.GtfsRealtime) FeedEntityValidator(edu.usf.cutr.gtfsrtvalidator.lib.validation.interfaces.FeedEntityValidator) MalformedURLException(java.net.MalformedURLException) TimeZone(java.util.TimeZone) GtfsDaoImpl(org.onebusaway.gtfs.impl.GtfsDaoImpl) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Collectors(java.util.stream.Collectors) GtfsMetadata(edu.usf.cutr.gtfsrtvalidator.lib.validation.GtfsMetadata) TimeUnit(java.util.concurrent.TimeUnit) IOUtils(org.apache.commons.io.IOUtils) TimestampUtils.getElapsedTimeString(edu.usf.cutr.gtfsrtvalidator.lib.util.TimestampUtils.getElapsedTimeString) List(java.util.List) GtfsRtFeedModel(edu.usf.cutr.gtfsrtvalidator.lib.model.GtfsRtFeedModel) GtfsFeed(edu.usf.cutr.gtfsrtvalidator.api.resource.GtfsFeed) InputStream(java.io.InputStream) MalformedURLException(java.net.MalformedURLException) GtfsMetadata(edu.usf.cutr.gtfsrtvalidator.lib.validation.GtfsMetadata) ArrayList(java.util.ArrayList) URL(java.net.URL) GtfsRtFeedModel(edu.usf.cutr.gtfsrtvalidator.lib.model.GtfsRtFeedModel) GtfsDaoImpl(org.onebusaway.gtfs.impl.GtfsDaoImpl) MessageDigest(java.security.MessageDigest) FeedEntityValidator(edu.usf.cutr.gtfsrtvalidator.lib.validation.interfaces.FeedEntityValidator) ByteArrayInputStream(java.io.ByteArrayInputStream) InputStream(java.io.InputStream) MalformedURLException(java.net.MalformedURLException) GtfsRealtime(com.google.transit.realtime.GtfsRealtime) GtfsRtFeedIterationModel(edu.usf.cutr.gtfsrtvalidator.lib.model.GtfsRtFeedIterationModel) ByteArrayInputStream(java.io.ByteArrayInputStream) Session(org.hibernate.Session)

Aggregations

GtfsRealtime (com.google.transit.realtime.GtfsRealtime)1 GtfsFeed (edu.usf.cutr.gtfsrtvalidator.api.resource.GtfsFeed)1 GTFSDB (edu.usf.cutr.gtfsrtvalidator.db.GTFSDB)1 DBHelper (edu.usf.cutr.gtfsrtvalidator.helper.DBHelper)1 GtfsRtFeedIterationModel (edu.usf.cutr.gtfsrtvalidator.lib.model.GtfsRtFeedIterationModel)1 GtfsRtFeedModel (edu.usf.cutr.gtfsrtvalidator.lib.model.GtfsRtFeedModel)1 ErrorListHelperModel (edu.usf.cutr.gtfsrtvalidator.lib.model.helper.ErrorListHelperModel)1 GtfsUtils (edu.usf.cutr.gtfsrtvalidator.lib.util.GtfsUtils)1 TimestampUtils.getElapsedTime (edu.usf.cutr.gtfsrtvalidator.lib.util.TimestampUtils.getElapsedTime)1 TimestampUtils.getElapsedTimeString (edu.usf.cutr.gtfsrtvalidator.lib.util.TimestampUtils.getElapsedTimeString)1 GtfsMetadata (edu.usf.cutr.gtfsrtvalidator.lib.validation.GtfsMetadata)1 FeedEntityValidator (edu.usf.cutr.gtfsrtvalidator.lib.validation.interfaces.FeedEntityValidator)1 edu.usf.cutr.gtfsrtvalidator.lib.validation.rules (edu.usf.cutr.gtfsrtvalidator.lib.validation.rules)1 ByteArrayInputStream (java.io.ByteArrayInputStream)1 InputStream (java.io.InputStream)1 MalformedURLException (java.net.MalformedURLException)1 URL (java.net.URL)1 MessageDigest (java.security.MessageDigest)1 ArrayList (java.util.ArrayList)1 List (java.util.List)1