Examples with IntHashSet com.carrotsearch.hppc.IntHashSet used on opensource projects

Example 6 with IntHashSet

use of com.carrotsearch.hppc.IntHashSet in project crate by crate.

the class NodeFetchRequestTest method testStreaming.

@Test
public void testStreaming() throws Exception {
    IntObjectHashMap<IntContainer> toFetch = new IntObjectHashMap<>();
    IntHashSet docIds = new IntHashSet(3);
    toFetch.put(1, docIds);
    NodeFetchRequest orig = new NodeFetchRequest(UUID.randomUUID(), 1, true, toFetch);
    BytesStreamOutput out = new BytesStreamOutput();
    orig.writeTo(out);
    StreamInput in = StreamInput.wrap(out.bytes());
    NodeFetchRequest streamed = new NodeFetchRequest();
    streamed.readFrom(in);
    assertThat(orig.jobId(), is(streamed.jobId()));
    assertThat(orig.fetchPhaseId(), is(streamed.fetchPhaseId()));
    assertThat(orig.isCloseContext(), is(streamed.isCloseContext()));
    assertThat(orig.toFetch().toString(), is(streamed.toFetch().toString()));
}

Example 7 with IntHashSet

use of com.carrotsearch.hppc.IntHashSet in project elasticsearch by elastic.

the class DuelScrollIT method create.

private TestContext create(SearchType... searchTypes) throws Exception {
    assertAcked(prepareCreate("index").addMapping("type", jsonBuilder().startObject().startObject("type").startObject("properties").startObject("field1").field("type", "long").endObject().startObject("field2").field("type", "keyword").endObject().startObject("nested").field("type", "nested").startObject("properties").startObject("field3").field("type", "long").endObject().startObject("field4").field("type", "keyword").endObject().endObject().endObject().endObject().endObject().endObject()));
    int numDocs = 2 + randomInt(512);
    int scrollRequestSize = randomIntBetween(1, rarely() ? numDocs : numDocs / 2);
    boolean unevenRouting = randomBoolean();
    int numMissingDocs = scaledRandomIntBetween(0, numDocs / 100);
    IntHashSet missingDocs = new IntHashSet(numMissingDocs);
    for (int i = 0; i < numMissingDocs; i++) {
        while (!missingDocs.add(randomInt(numDocs))) {
        }
    }
    for (int i = 1; i <= numDocs; i++) {
        IndexRequestBuilder indexRequestBuilder = client().prepareIndex("index", "type", String.valueOf(i));
        if (missingDocs.contains(i)) {
            indexRequestBuilder.setSource("x", "y");
        } else {
            indexRequestBuilder.setSource(jsonBuilder().startObject().field("field1", i).field("field2", String.valueOf(i)).startObject("nested").field("field3", i).field("field4", String.valueOf(i)).endObject().endObject());
        }
        if (unevenRouting && randomInt(3) <= 2) {
            indexRequestBuilder.setRouting("a");
        }
        indexRandom(false, indexRequestBuilder);
    }
    refresh();
    final SortBuilder sort;
    if (randomBoolean()) {
        if (randomBoolean()) {
            sort = SortBuilders.fieldSort("field1").missing(1);
        } else {
            sort = SortBuilders.fieldSort("field2").missing("1");
        }
    } else {
        if (randomBoolean()) {
            sort = SortBuilders.fieldSort("nested.field3").missing(1);
        } else {
            sort = SortBuilders.fieldSort("nested.field4").missing("1");
        }
    }
    sort.order(randomBoolean() ? SortOrder.ASC : SortOrder.DESC);
    SearchType searchType = RandomPicks.randomFrom(random(), Arrays.asList(searchTypes));
    logger.info("numDocs={}, scrollRequestSize={}, sort={}, searchType={}", numDocs, scrollRequestSize, sort, searchType);
    return new TestContext(numDocs, scrollRequestSize, sort, searchType);
}

Example 8 with IntHashSet

use of com.carrotsearch.hppc.IntHashSet in project lucene-solr by apache.

the class ExpandComponent method process.

@SuppressWarnings("unchecked")
@Override
public void process(ResponseBuilder rb) throws IOException {
    if (!rb.doExpand) {
        return;
    }
    SolrQueryRequest req = rb.req;
    SolrParams params = req.getParams();
    String field = params.get(ExpandParams.EXPAND_FIELD);
    String hint = null;
    if (field == null) {
        List<Query> filters = rb.getFilters();
        if (filters != null) {
            for (Query q : filters) {
                if (q instanceof CollapsingQParserPlugin.CollapsingPostFilter) {
                    CollapsingQParserPlugin.CollapsingPostFilter cp = (CollapsingQParserPlugin.CollapsingPostFilter) q;
                    field = cp.getField();
                    hint = cp.hint;
                }
            }
        }
    }
    if (field == null) {
        throw new IOException("Expand field is null.");
    }
    String sortParam = params.get(ExpandParams.EXPAND_SORT);
    String[] fqs = params.getParams(ExpandParams.EXPAND_FQ);
    String qs = params.get(ExpandParams.EXPAND_Q);
    int limit = params.getInt(ExpandParams.EXPAND_ROWS, 5);
    Sort sort = null;
    if (sortParam != null) {
        sort = SortSpecParsing.parseSortSpec(sortParam, rb.req).getSort();
    }
    Query query;
    if (qs == null) {
        query = rb.getQuery();
    } else {
        try {
            QParser parser = QParser.getParser(qs, req);
            query = parser.getQuery();
        } catch (Exception e) {
            throw new IOException(e);
        }
    }
    List<Query> newFilters = new ArrayList<>();
    if (fqs == null) {
        List<Query> filters = rb.getFilters();
        if (filters != null) {
            for (Query q : filters) {
                if (!(q instanceof CollapsingQParserPlugin.CollapsingPostFilter)) {
                    newFilters.add(q);
                }
            }
        }
    } else {
        try {
            for (String fq : fqs) {
                if (fq != null && fq.trim().length() != 0 && !fq.equals("*:*")) {
                    QParser fqp = QParser.getParser(fq, req);
                    newFilters.add(fqp.getQuery());
                }
            }
        } catch (Exception e) {
            throw new IOException(e);
        }
    }
    SolrIndexSearcher searcher = req.getSearcher();
    LeafReader reader = searcher.getSlowAtomicReader();
    SchemaField schemaField = searcher.getSchema().getField(field);
    FieldType fieldType = schemaField.getType();
    SortedDocValues values = null;
    long nullValue = 0L;
    if (fieldType instanceof StrField) {
        //Get The Top Level SortedDocValues
        if (CollapsingQParserPlugin.HINT_TOP_FC.equals(hint)) {
            Map<String, UninvertingReader.Type> mapping = new HashMap();
            mapping.put(field, UninvertingReader.Type.SORTED);
            UninvertingReader uninvertingReader = new UninvertingReader(new ReaderWrapper(searcher.getSlowAtomicReader(), field), mapping);
            values = uninvertingReader.getSortedDocValues(field);
        } else {
            values = DocValues.getSorted(reader, field);
        }
    } else {
        //Get the nullValue for the numeric collapse field
        String defaultValue = searcher.getSchema().getField(field).getDefaultValue();
        final NumberType numType = fieldType.getNumberType();
        // we don't need to handle invalid 64-bit field types here.
        if (defaultValue != null) {
            if (numType == NumberType.INTEGER) {
                nullValue = Long.parseLong(defaultValue);
            } else if (numType == NumberType.FLOAT) {
                nullValue = Float.floatToIntBits(Float.parseFloat(defaultValue));
            }
        } else if (NumberType.FLOAT.equals(numType)) {
            // Integer case already handled by nullValue defaulting to 0
            nullValue = Float.floatToIntBits(0.0f);
        }
    }
    FixedBitSet groupBits = null;
    LongHashSet groupSet = null;
    DocList docList = rb.getResults().docList;
    IntHashSet collapsedSet = new IntHashSet(docList.size() * 2);
    //Gather the groups for the current page of documents
    DocIterator idit = docList.iterator();
    int[] globalDocs = new int[docList.size()];
    int docsIndex = -1;
    while (idit.hasNext()) {
        globalDocs[++docsIndex] = idit.nextDoc();
    }
    Arrays.sort(globalDocs);
    Query groupQuery = null;
    /*
    * This code gathers the group information for the current page.
    */
    List<LeafReaderContext> contexts = searcher.getTopReaderContext().leaves();
    if (contexts.size() == 0) {
        //When no context is available we can skip the expanding
        return;
    }
    int currentContext = 0;
    int currentDocBase = contexts.get(currentContext).docBase;
    int nextDocBase = (currentContext + 1) < contexts.size() ? contexts.get(currentContext + 1).docBase : Integer.MAX_VALUE;
    IntObjectHashMap<BytesRef> ordBytes = null;
    if (values != null) {
        groupBits = new FixedBitSet(values.getValueCount());
        MultiDocValues.OrdinalMap ordinalMap = null;
        SortedDocValues[] sortedDocValues = null;
        LongValues segmentOrdinalMap = null;
        SortedDocValues currentValues = null;
        if (values instanceof MultiDocValues.MultiSortedDocValues) {
            ordinalMap = ((MultiDocValues.MultiSortedDocValues) values).mapping;
            sortedDocValues = ((MultiDocValues.MultiSortedDocValues) values).values;
            currentValues = sortedDocValues[currentContext];
            segmentOrdinalMap = ordinalMap.getGlobalOrds(currentContext);
        }
        int count = 0;
        ordBytes = new IntObjectHashMap<>();
        for (int i = 0; i < globalDocs.length; i++) {
            int globalDoc = globalDocs[i];
            while (globalDoc >= nextDocBase) {
                currentContext++;
                currentDocBase = contexts.get(currentContext).docBase;
                nextDocBase = (currentContext + 1) < contexts.size() ? contexts.get(currentContext + 1).docBase : Integer.MAX_VALUE;
                if (ordinalMap != null) {
                    currentValues = sortedDocValues[currentContext];
                    segmentOrdinalMap = ordinalMap.getGlobalOrds(currentContext);
                }
            }
            int contextDoc = globalDoc - currentDocBase;
            if (ordinalMap != null) {
                if (contextDoc > currentValues.docID()) {
                    currentValues.advance(contextDoc);
                }
                if (contextDoc == currentValues.docID()) {
                    int ord = currentValues.ordValue();
                    ++count;
                    BytesRef ref = currentValues.lookupOrd(ord);
                    ord = (int) segmentOrdinalMap.get(ord);
                    ordBytes.put(ord, BytesRef.deepCopyOf(ref));
                    groupBits.set(ord);
                    collapsedSet.add(globalDoc);
                }
            } else {
                if (globalDoc > values.docID()) {
                    values.advance(globalDoc);
                }
                if (globalDoc == values.docID()) {
                    int ord = values.ordValue();
                    ++count;
                    BytesRef ref = values.lookupOrd(ord);
                    ordBytes.put(ord, BytesRef.deepCopyOf(ref));
                    groupBits.set(ord);
                    collapsedSet.add(globalDoc);
                }
            }
        }
        if (count > 0 && count < 200) {
            try {
                groupQuery = getGroupQuery(field, count, ordBytes);
            } catch (Exception e) {
                throw new IOException(e);
            }
        }
    } else {
        groupSet = new LongHashSet(docList.size());
        NumericDocValues collapseValues = contexts.get(currentContext).reader().getNumericDocValues(field);
        int count = 0;
        for (int i = 0; i < globalDocs.length; i++) {
            int globalDoc = globalDocs[i];
            while (globalDoc >= nextDocBase) {
                currentContext++;
                currentDocBase = contexts.get(currentContext).docBase;
                nextDocBase = currentContext + 1 < contexts.size() ? contexts.get(currentContext + 1).docBase : Integer.MAX_VALUE;
                collapseValues = contexts.get(currentContext).reader().getNumericDocValues(field);
            }
            int contextDoc = globalDoc - currentDocBase;
            int valueDocID = collapseValues.docID();
            if (valueDocID < contextDoc) {
                valueDocID = collapseValues.advance(contextDoc);
            }
            long value;
            if (valueDocID == contextDoc) {
                value = collapseValues.longValue();
            } else {
                value = 0;
            }
            if (value != nullValue) {
                ++count;
                groupSet.add(value);
                collapsedSet.add(globalDoc);
            }
        }
        if (count > 0 && count < 200) {
            if (fieldType.isPointField()) {
                groupQuery = getPointGroupQuery(schemaField, count, groupSet);
            } else {
                groupQuery = getGroupQuery(field, fieldType, count, groupSet);
            }
        }
    }
    Collector collector;
    if (sort != null)
        sort = sort.rewrite(searcher);
    Collector groupExpandCollector = null;
    if (values != null) {
        //Get The Top Level SortedDocValues again so we can re-iterate:
        if (CollapsingQParserPlugin.HINT_TOP_FC.equals(hint)) {
            Map<String, UninvertingReader.Type> mapping = new HashMap();
            mapping.put(field, UninvertingReader.Type.SORTED);
            UninvertingReader uninvertingReader = new UninvertingReader(new ReaderWrapper(searcher.getSlowAtomicReader(), field), mapping);
            values = uninvertingReader.getSortedDocValues(field);
        } else {
            values = DocValues.getSorted(reader, field);
        }
        groupExpandCollector = new GroupExpandCollector(values, groupBits, collapsedSet, limit, sort);
    } else {
        groupExpandCollector = new NumericGroupExpandCollector(field, nullValue, groupSet, collapsedSet, limit, sort);
    }
    if (groupQuery != null) {
        //Limits the results to documents that are in the same group as the documents in the page.
        newFilters.add(groupQuery);
    }
    SolrIndexSearcher.ProcessedFilter pfilter = searcher.getProcessedFilter(null, newFilters);
    if (pfilter.postFilter != null) {
        pfilter.postFilter.setLastDelegate(groupExpandCollector);
        collector = pfilter.postFilter;
    } else {
        collector = groupExpandCollector;
    }
    if (pfilter.filter == null) {
        searcher.search(query, collector);
    } else {
        Query q = new BooleanQuery.Builder().add(query, Occur.MUST).add(pfilter.filter, Occur.FILTER).build();
        searcher.search(q, collector);
    }
    LongObjectMap<Collector> groups = ((GroupCollector) groupExpandCollector).getGroups();
    NamedList outMap = new SimpleOrderedMap();
    CharsRefBuilder charsRef = new CharsRefBuilder();
    for (LongObjectCursor<Collector> cursor : groups) {
        long groupValue = cursor.key;
        TopDocsCollector<?> topDocsCollector = TopDocsCollector.class.cast(cursor.value);
        TopDocs topDocs = topDocsCollector.topDocs();
        ScoreDoc[] scoreDocs = topDocs.scoreDocs;
        if (scoreDocs.length > 0) {
            int[] docs = new int[scoreDocs.length];
            float[] scores = new float[scoreDocs.length];
            for (int i = 0; i < docs.length; i++) {
                ScoreDoc scoreDoc = scoreDocs[i];
                docs[i] = scoreDoc.doc;
                scores[i] = scoreDoc.score;
            }
            DocSlice slice = new DocSlice(0, docs.length, docs, scores, topDocs.totalHits, topDocs.getMaxScore());
            if (fieldType instanceof StrField) {
                final BytesRef bytesRef = ordBytes.get((int) groupValue);
                fieldType.indexedToReadable(bytesRef, charsRef);
                String group = charsRef.toString();
                outMap.add(group, slice);
            } else {
                outMap.add(numericToString(fieldType, groupValue), slice);
            }
        }
    }
    rb.rsp.add("expanded", outMap);
}

Example 9 with IntHashSet

use of com.carrotsearch.hppc.IntHashSet in project drill by apache.

the class FlattenRecordBatch method setupNewSchema.

@Override
protected boolean setupNewSchema() throws SchemaChangeException {
    this.allocationVectors = Lists.newArrayList();
    container.clear();
    final List<NamedExpression> exprs = getExpressionList();
    final ErrorCollector collector = new ErrorCollectorImpl();
    final List<TransferPair> transfers = Lists.newArrayList();
    final ClassGenerator<Flattener> cg = CodeGenerator.getRoot(Flattener.TEMPLATE_DEFINITION, context.getFunctionRegistry(), context.getOptions());
    cg.getCodeGenerator().plainJavaCapable(true);
    // Uncomment out this line to debug the generated code.
    //    cg.getCodeGenerator().saveCodeForDebugging(true);
    final IntHashSet transferFieldIds = new IntHashSet();
    final NamedExpression flattenExpr = new NamedExpression(popConfig.getColumn(), new FieldReference(popConfig.getColumn()));
    final ValueVectorReadExpression vectorRead = (ValueVectorReadExpression) ExpressionTreeMaterializer.materialize(flattenExpr.getExpr(), incoming, collector, context.getFunctionRegistry(), true);
    final FieldReference fieldReference = flattenExpr.getRef();
    final TransferPair transferPair = getFlattenFieldTransferPair(fieldReference);
    if (transferPair != null) {
        final ValueVector flattenVector = transferPair.getTo();
        // checks that list has only default ValueVector and replaces resulting ValueVector to INT typed ValueVector
        if (exprs.size() == 0 && flattenVector.getField().getType().equals(Types.LATE_BIND_TYPE)) {
            final MaterializedField outputField = MaterializedField.create(fieldReference.getAsNamePart().getName(), Types.OPTIONAL_INT);
            final ValueVector vector = TypeHelper.getNewVector(outputField, oContext.getAllocator());
            container.add(vector);
        } else {
            transfers.add(transferPair);
            container.add(flattenVector);
            transferFieldIds.add(vectorRead.getFieldId().getFieldIds()[0]);
        }
    }
    logger.debug("Added transfer for project expression.");
    ClassifierResult result = new ClassifierResult();
    for (int i = 0; i < exprs.size(); i++) {
        final NamedExpression namedExpression = exprs.get(i);
        result.clear();
        String outputName = getRef(namedExpression).getRootSegment().getPath();
        if (result != null && result.outputNames != null && result.outputNames.size() > 0) {
            for (int j = 0; j < result.outputNames.size(); j++) {
                if (!result.outputNames.get(j).equals(EMPTY_STRING)) {
                    outputName = result.outputNames.get(j);
                    break;
                }
            }
        }
        final LogicalExpression expr = ExpressionTreeMaterializer.materialize(namedExpression.getExpr(), incoming, collector, context.getFunctionRegistry(), true);
        if (collector.hasErrors()) {
            throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema.  Errors:\n %s.", collector.toErrorString()));
        }
        if (expr instanceof DrillFuncHolderExpr && ((DrillFuncHolderExpr) expr).getHolder().isComplexWriterFuncHolder()) {
            // Lazy initialization of the list of complex writers, if not done yet.
            if (complexWriters == null) {
                complexWriters = Lists.newArrayList();
            }
            // The reference name will be passed to ComplexWriter, used as the name of the output vector from the writer.
            ((DrillFuncHolderExpr) expr).getFieldReference(namedExpression.getRef());
            cg.addExpr(expr);
        } else {
            // need to do evaluation.
            final MaterializedField outputField;
            if (expr instanceof ValueVectorReadExpression) {
                final TypedFieldId id = ValueVectorReadExpression.class.cast(expr).getFieldId();
                @SuppressWarnings("resource") final ValueVector incomingVector = incoming.getValueAccessorById(id.getIntermediateClass(), id.getFieldIds()).getValueVector();
                // when the first batch will be empty.
                if (incomingVector != null) {
                    outputField = incomingVector.getField().clone();
                } else {
                    outputField = MaterializedField.create(outputName, expr.getMajorType());
                }
            } else {
                outputField = MaterializedField.create(outputName, expr.getMajorType());
            }
            @SuppressWarnings("resource") final ValueVector vector = TypeHelper.getNewVector(outputField, oContext.getAllocator());
            allocationVectors.add(vector);
            TypedFieldId fid = container.add(vector);
            ValueVectorWriteExpression write = new ValueVectorWriteExpression(fid, expr, true);
            cg.addExpr(write);
            logger.debug("Added eval for project expression.");
        }
    }
    cg.rotateBlock();
    cg.getEvalBlock()._return(JExpr.TRUE);
    container.buildSchema(SelectionVectorMode.NONE);
    try {
        this.flattener = context.getImplementationClass(cg.getCodeGenerator());
        flattener.setup(context, incoming, this, transfers);
    } catch (ClassTransformationException | IOException e) {
        throw new SchemaChangeException("Failure while attempting to load generated class", e);
    }
    return true;
}

Example 10 with IntHashSet

use of com.carrotsearch.hppc.IntHashSet in project graphhopper by graphhopper.

the class RealtimeFeed method fromProtobuf.

public static RealtimeFeed fromProtobuf(Graph graph, GtfsStorage staticGtfs, PtFlagEncoder encoder, GtfsRealtime.FeedMessage feedMessage) {
    String feedKey = "gtfs_0";
    GTFSFeed feed = staticGtfs.getGtfsFeeds().get(feedKey);
    // TODO: Require configuration of feed and agency this realtime feed is for.
    // Realtime feeds are always specific to an agency.
    Agency agency = feed.agency.values().iterator().next();
    final IntHashSet blockedEdges = new IntHashSet();
    final IntLongHashMap delaysForBoardEdges = new IntLongHashMap();
    final IntLongHashMap delaysForAlightEdges = new IntLongHashMap();
    final LinkedList<VirtualEdgeIteratorState> additionalEdges = new LinkedList<>();
    final Graph overlayGraph = new Graph() {

        int nNodes = 0;

        int firstEdge = graph.getAllEdges().getMaxId() + 1;

        final NodeAccess nodeAccess = new NodeAccess() {

            IntIntHashMap additionalNodeFields = new IntIntHashMap();

            @Override
            public int getAdditionalNodeField(int nodeId) {
                return 0;
            }

            @Override
            public void setAdditionalNodeField(int nodeId, int additionalValue) {
                additionalNodeFields.put(nodeId, additionalValue);
            }

            @Override
            public boolean is3D() {
                return false;
            }

            @Override
            public int getDimension() {
                return 0;
            }

            @Override
            public void ensureNode(int nodeId) {
            }

            @Override
            public void setNode(int nodeId, double lat, double lon) {
            }

            @Override
            public void setNode(int nodeId, double lat, double lon, double ele) {
            }

            @Override
            public double getLatitude(int nodeId) {
                return 0;
            }

            @Override
            public double getLat(int nodeId) {
                return 0;
            }

            @Override
            public double getLongitude(int nodeId) {
                return 0;
            }

            @Override
            public double getLon(int nodeId) {
                return 0;
            }

            @Override
            public double getElevation(int nodeId) {
                return 0;
            }

            @Override
            public double getEle(int nodeId) {
                return 0;
            }
        };

        @Override
        public Graph getBaseGraph() {
            return graph;
        }

        @Override
        public int getNodes() {
            return graph.getNodes() + nNodes;
        }

        @Override
        public NodeAccess getNodeAccess() {
            return nodeAccess;
        }

        @Override
        public BBox getBounds() {
            return null;
        }

        @Override
        public EdgeIteratorState edge(int a, int b) {
            return null;
        }

        @Override
        public EdgeIteratorState edge(int a, int b, double distance, boolean bothDirections) {
            int edge = firstEdge++;
            final VirtualEdgeIteratorState newEdge = new VirtualEdgeIteratorState(-1, edge, a, b, distance, 0, "", new PointList());
            final VirtualEdgeIteratorState reverseNewEdge = new VirtualEdgeIteratorState(-1, edge, b, a, distance, 0, "", new PointList());
            newEdge.setReverseEdge(reverseNewEdge);
            reverseNewEdge.setReverseEdge(newEdge);
            additionalEdges.push(newEdge);
            return newEdge;
        }

        @Override
        public EdgeIteratorState getEdgeIteratorState(int edgeId, int adjNode) {
            return null;
        }

        @Override
        public AllEdgesIterator getAllEdges() {
            return null;
        }

        @Override
        public EdgeExplorer createEdgeExplorer(EdgeFilter filter) {
            return null;
        }

        @Override
        public EdgeExplorer createEdgeExplorer() {
            return graph.createEdgeExplorer();
        }

        @Override
        public Graph copyTo(Graph g) {
            return null;
        }

        @Override
        public GraphExtension getExtension() {
            throw new RuntimeException();
        }
    };
    Map<Integer, String> routes = new HashMap<>();
    Map<GtfsStorage.Validity, Integer> operatingDayPatterns = new HashMap<>(staticGtfs.getOperatingDayPatterns());
    Map<Integer, byte[]> tripDescriptors = new HashMap<>();
    Map<Integer, Integer> stopSequences = new HashMap<>();
    Map<String, int[]> boardEdgesForTrip = new HashMap<>();
    Map<String, int[]> alightEdgesForTrip = new HashMap<>();
    Map<GtfsStorage.FeedIdWithTimezone, Integer> writableTimeZones = new HashMap<>();
    GtfsStorageI gtfsStorage = new GtfsStorageI() {

        @Override
        public Map<String, Fare> getFares() {
            return null;
        }

        @Override
        public Map<GtfsStorage.Validity, Integer> getOperatingDayPatterns() {
            return operatingDayPatterns;
        }

        @Override
        public Map<GtfsStorage.FeedIdWithTimezone, Integer> getWritableTimeZones() {
            return writableTimeZones;
        }

        @Override
        public Map<Integer, byte[]> getTripDescriptors() {
            return tripDescriptors;
        }

        @Override
        public Map<Integer, Integer> getStopSequences() {
            return stopSequences;
        }

        @Override
        public Map<String, int[]> getBoardEdgesForTrip() {
            return boardEdgesForTrip;
        }

        @Override
        public Map<String, int[]> getAlightEdgesForTrip() {
            return alightEdgesForTrip;
        }

        @Override
        public Map<String, GTFSFeed> getGtfsFeeds() {
            HashMap<String, GTFSFeed> stringGTFSFeedHashMap = new HashMap<>();
            stringGTFSFeedHashMap.put(feedKey, feed);
            return stringGTFSFeedHashMap;
        }

        @Override
        public Map<String, Transfers> getTransfers() {
            return staticGtfs.getTransfers();
        }

        @Override
        public Map<String, Integer> getStationNodes() {
            return staticGtfs.getStationNodes();
        }

        @Override
        public Map<Integer, String> getRoutes() {
            return routes;
        }
    };
    final GtfsReader gtfsReader = new GtfsReader(feedKey, overlayGraph, gtfsStorage, encoder, null);
    Instant timestamp = Instant.ofEpochSecond(feedMessage.getHeader().getTimestamp());
    // FIXME
    LocalDate dateToChange = timestamp.atZone(ZoneId.of(agency.agency_timezone)).toLocalDate();
    BitSet validOnDay = new BitSet();
    LocalDate startDate = feed.calculateStats().getStartDate();
    validOnDay.set((int) DAYS.between(startDate, dateToChange));
    feedMessage.getEntityList().stream().filter(GtfsRealtime.FeedEntity::hasTripUpdate).map(GtfsRealtime.FeedEntity::getTripUpdate).filter(tripUpdate -> tripUpdate.getTrip().getScheduleRelationship() == GtfsRealtime.TripDescriptor.ScheduleRelationship.SCHEDULED).forEach(tripUpdate -> {
        String key = GtfsStorage.tripKey(tripUpdate.getTrip().getTripId(), tripUpdate.getTrip().getStartTime());
        final int[] boardEdges = staticGtfs.getBoardEdgesForTrip().get(key);
        final int[] leaveEdges = staticGtfs.getAlightEdgesForTrip().get(key);
        if (boardEdges == null || leaveEdges == null) {
            logger.warn("Trip not found: {}", tripUpdate.getTrip());
            return;
        }
        tripUpdate.getStopTimeUpdateList().stream().filter(stopTimeUpdate -> stopTimeUpdate.getScheduleRelationship() == SKIPPED).mapToInt(GtfsRealtime.TripUpdate.StopTimeUpdate::getStopSequence).forEach(skippedStopSequenceNumber -> {
            blockedEdges.add(boardEdges[skippedStopSequenceNumber]);
            blockedEdges.add(leaveEdges[skippedStopSequenceNumber]);
        });
        GtfsReader.TripWithStopTimes tripWithStopTimes = toTripWithStopTimes(feed, agency, tripUpdate);
        tripWithStopTimes.stopTimes.forEach(stopTime -> {
            if (stopTime.stop_sequence > leaveEdges.length - 1) {
                logger.warn("Stop sequence number too high {} vs {}", stopTime.stop_sequence, leaveEdges.length);
                return;
            }
            final StopTime originalStopTime = feed.stop_times.get(new Fun.Tuple2(tripUpdate.getTrip().getTripId(), stopTime.stop_sequence));
            int arrivalDelay = stopTime.arrival_time - originalStopTime.arrival_time;
            delaysForAlightEdges.put(leaveEdges[stopTime.stop_sequence], arrivalDelay * 1000);
            int departureDelay = stopTime.departure_time - originalStopTime.departure_time;
            if (departureDelay > 0) {
                int boardEdge = boardEdges[stopTime.stop_sequence];
                int departureNode = graph.getEdgeIteratorState(boardEdge, Integer.MIN_VALUE).getAdjNode();
                int timeOffset = tripUpdate.getTrip().hasStartTime() ? LocalTime.parse(tripUpdate.getTrip().getStartTime()).toSecondOfDay() : 0;
                int delayedBoardEdge = gtfsReader.addDelayedBoardEdge(ZoneId.of(agency.agency_timezone), tripUpdate.getTrip(), stopTime.stop_sequence, stopTime.departure_time + timeOffset, departureNode, validOnDay);
                delaysForBoardEdges.put(delayedBoardEdge, departureDelay * 1000);
            }
        });
    });
    feedMessage.getEntityList().stream().filter(GtfsRealtime.FeedEntity::hasTripUpdate).map(GtfsRealtime.FeedEntity::getTripUpdate).filter(tripUpdate -> tripUpdate.getTrip().getScheduleRelationship() == GtfsRealtime.TripDescriptor.ScheduleRelationship.ADDED).forEach(tripUpdate -> {
        Trip trip = new Trip();
        trip.trip_id = tripUpdate.getTrip().getTripId();
        trip.route_id = tripUpdate.getTrip().getRouteId();
        final List<StopTime> stopTimes = tripUpdate.getStopTimeUpdateList().stream().map(stopTimeUpdate -> {
            final StopTime stopTime = new StopTime();
            stopTime.stop_sequence = stopTimeUpdate.getStopSequence();
            stopTime.stop_id = stopTimeUpdate.getStopId();
            stopTime.trip_id = trip.trip_id;
            final ZonedDateTime arrival_time = Instant.ofEpochSecond(stopTimeUpdate.getArrival().getTime()).atZone(ZoneId.of(agency.agency_timezone));
            stopTime.arrival_time = (int) Duration.between(arrival_time.truncatedTo(ChronoUnit.DAYS), arrival_time).getSeconds();
            final ZonedDateTime departure_time = Instant.ofEpochSecond(stopTimeUpdate.getArrival().getTime()).atZone(ZoneId.of(agency.agency_timezone));
            stopTime.departure_time = (int) Duration.between(departure_time.truncatedTo(ChronoUnit.DAYS), departure_time).getSeconds();
            return stopTime;
        }).collect(Collectors.toList());
        GtfsReader.TripWithStopTimes tripWithStopTimes = new GtfsReader.TripWithStopTimes(trip, stopTimes, validOnDay, Collections.emptySet(), Collections.emptySet());
        gtfsReader.addTrip(ZoneId.of(agency.agency_timezone), 0, new ArrayList<>(), tripWithStopTimes, tripUpdate.getTrip());
    });
    gtfsReader.wireUpStops();
    return new RealtimeFeed(staticGtfs, feed, agency, feedMessage, blockedEdges, delaysForBoardEdges, delaysForAlightEdges, additionalEdges, tripDescriptors, stopSequences, operatingDayPatterns);
}