Examples with MergeIterator org.apache.cassandra.utils.MergeIterator used on opensource projects

Example 1 with MergeIterator

use of org.apache.cassandra.utils.MergeIterator in project cassandra by apache.

the class UnfilteredPartitionIterators method merge.

public static UnfilteredPartitionIterator merge(final List<? extends UnfilteredPartitionIterator> iterators, final int nowInSec, final MergeListener listener) {
    assert listener != null;
    assert !iterators.isEmpty();
    final TableMetadata metadata = iterators.get(0).metadata();
    final MergeIterator<UnfilteredRowIterator, UnfilteredRowIterator> merged = MergeIterator.get(iterators, partitionComparator, new MergeIterator.Reducer<UnfilteredRowIterator, UnfilteredRowIterator>() {

        private final List<UnfilteredRowIterator> toMerge = new ArrayList<>(iterators.size());

        private DecoratedKey partitionKey;

        private boolean isReverseOrder;

        public void reduce(int idx, UnfilteredRowIterator current) {
            partitionKey = current.partitionKey();
            isReverseOrder = current.isReverseOrder();
            // Note that because the MergeListener cares about it, we want to preserve the index of the iterator.
            // Non-present iterator will thus be set to empty in getReduced.
            toMerge.set(idx, current);
        }

        protected UnfilteredRowIterator getReduced() {
            UnfilteredRowIterators.MergeListener rowListener = listener.getRowMergeListener(partitionKey, toMerge);
            // Replace nulls by empty iterators
            for (int i = 0; i < toMerge.size(); i++) if (toMerge.get(i) == null)
                toMerge.set(i, EmptyIterators.unfilteredRow(metadata, partitionKey, isReverseOrder));
            return UnfilteredRowIterators.merge(toMerge, nowInSec, rowListener);
        }

        protected void onKeyChange() {
            toMerge.clear();
            for (int i = 0; i < iterators.size(); i++) toMerge.add(null);
        }
    });
    return new AbstractUnfilteredPartitionIterator() {

        public TableMetadata metadata() {
            return metadata;
        }

        public boolean hasNext() {
            return merged.hasNext();
        }

        public UnfilteredRowIterator next() {
            return merged.next();
        }

        @Override
        public void close() {
            merged.close();
            listener.close();
        }
    };
}

Example 2 with MergeIterator

use of org.apache.cassandra.utils.MergeIterator in project cassandra by apache.

the class UnfilteredPartitionIterators method mergeLazily.

public static UnfilteredPartitionIterator mergeLazily(final List<? extends UnfilteredPartitionIterator> iterators, final int nowInSec) {
    assert !iterators.isEmpty();
    if (iterators.size() == 1)
        return iterators.get(0);
    final TableMetadata metadata = iterators.get(0).metadata();
    final MergeIterator<UnfilteredRowIterator, UnfilteredRowIterator> merged = MergeIterator.get(iterators, partitionComparator, new MergeIterator.Reducer<UnfilteredRowIterator, UnfilteredRowIterator>() {

        private final List<UnfilteredRowIterator> toMerge = new ArrayList<>(iterators.size());

        public void reduce(int idx, UnfilteredRowIterator current) {
            toMerge.add(current);
        }

        protected UnfilteredRowIterator getReduced() {
            return new LazilyInitializedUnfilteredRowIterator(toMerge.get(0).partitionKey()) {

                protected UnfilteredRowIterator initializeIterator() {
                    return UnfilteredRowIterators.merge(toMerge, nowInSec);
                }
            };
        }

        protected void onKeyChange() {
            toMerge.clear();
        }
    });
    return new AbstractUnfilteredPartitionIterator() {

        public TableMetadata metadata() {
            return metadata;
        }

        public boolean hasNext() {
            return merged.hasNext();
        }

        public UnfilteredRowIterator next() {
            return merged.next();
        }

        @Override
        public void close() {
            merged.close();
        }
    };
}

Example 3 with MergeIterator

use of org.apache.cassandra.utils.MergeIterator in project cassandra by apache.

the class UnfilteredPartitionIterators method merge.

@SuppressWarnings("resource")
public static UnfilteredPartitionIterator merge(final List<? extends UnfilteredPartitionIterator> iterators, final MergeListener listener) {
    assert !iterators.isEmpty();
    final TableMetadata metadata = iterators.get(0).metadata();
    final MergeIterator<UnfilteredRowIterator, UnfilteredRowIterator> merged = MergeIterator.get(iterators, partitionComparator, new MergeIterator.Reducer<UnfilteredRowIterator, UnfilteredRowIterator>() {

        private final List<UnfilteredRowIterator> toMerge = new ArrayList<>(iterators.size());

        private DecoratedKey partitionKey;

        private boolean isReverseOrder;

        public void reduce(int idx, UnfilteredRowIterator current) {
            partitionKey = current.partitionKey();
            isReverseOrder = current.isReverseOrder();
            // Note that because the MergeListener cares about it, we want to preserve the index of the iterator.
            // Non-present iterator will thus be set to empty in getReduced.
            toMerge.set(idx, current);
        }

        @SuppressWarnings("resource")
        protected UnfilteredRowIterator getReduced() {
            UnfilteredRowIterators.MergeListener rowListener = listener == null ? null : listener.getRowMergeListener(partitionKey, toMerge);
            // Make a single empty iterator object to merge, we don't need toMerge.size() copiess
            UnfilteredRowIterator empty = null;
            // Replace nulls by empty iterators
            for (int i = 0; i < toMerge.size(); i++) {
                if (toMerge.get(i) == null) {
                    if (null == empty)
                        empty = EmptyIterators.unfilteredRow(metadata, partitionKey, isReverseOrder);
                    toMerge.set(i, empty);
                }
            }
            return UnfilteredRowIterators.merge(toMerge, rowListener);
        }

        protected void onKeyChange() {
            toMerge.clear();
            for (int i = 0; i < iterators.size(); i++) toMerge.add(null);
        }
    });
    return new AbstractUnfilteredPartitionIterator() {

        public TableMetadata metadata() {
            return metadata;
        }

        public boolean hasNext() {
            return merged.hasNext();
        }

        public UnfilteredRowIterator next() {
            return merged.next();
        }

        @Override
        public void close() {
            merged.close();
            if (listener != null)
                listener.close();
        }
    };
}

Example 4 with MergeIterator

use of org.apache.cassandra.utils.MergeIterator in project cassandra by apache.

the class UnfilteredPartitionIterators method mergeLazily.

@SuppressWarnings("resource")
public static UnfilteredPartitionIterator mergeLazily(final List<? extends UnfilteredPartitionIterator> iterators) {
    assert !iterators.isEmpty();
    if (iterators.size() == 1)
        return iterators.get(0);
    final TableMetadata metadata = iterators.get(0).metadata();
    final MergeIterator<UnfilteredRowIterator, UnfilteredRowIterator> merged = MergeIterator.get(iterators, partitionComparator, new MergeIterator.Reducer<UnfilteredRowIterator, UnfilteredRowIterator>() {

        private final List<UnfilteredRowIterator> toMerge = new ArrayList<>(iterators.size());

        public void reduce(int idx, UnfilteredRowIterator current) {
            toMerge.add(current);
        }

        protected UnfilteredRowIterator getReduced() {
            return new LazilyInitializedUnfilteredRowIterator(toMerge.get(0).partitionKey()) {

                protected UnfilteredRowIterator initializeIterator() {
                    return UnfilteredRowIterators.merge(toMerge);
                }
            };
        }

        protected void onKeyChange() {
            toMerge.clear();
        }
    });
    return new AbstractUnfilteredPartitionIterator() {

        public TableMetadata metadata() {
            return metadata;
        }

        public boolean hasNext() {
            return merged.hasNext();
        }

        public UnfilteredRowIterator next() {
            return merged.next();
        }

        @Override
        public void close() {
            merged.close();
        }
    };
}

Example 5 with MergeIterator

use of org.apache.cassandra.utils.MergeIterator in project cassandra by apache.

the class Replay method replay.

public static void replay(String keyspace, List<String> arguments, List<String> targetHosts, List<File> resultPaths, String queryStorePath, boolean replayDDLStatements) {
    // how many fql queries should we read in to memory to be able to sort them?
    int readAhead = 200;
    List<ChronicleQueue> readQueues = null;
    List<FQLQueryIterator> iterators = null;
    List<Predicate<FQLQuery>> filters = new ArrayList<>();
    if (keyspace != null)
        filters.add(fqlQuery -> fqlQuery.keyspace() == null || fqlQuery.keyspace().equals(keyspace));
    if (!replayDDLStatements)
        filters.add(fqlQuery -> {
            boolean notDDLStatement = !fqlQuery.isDDLStatement();
            if (!notDDLStatement)
                logger.info("Excluding DDL statement from replaying: {}", ((FQLQuery.Single) fqlQuery).query);
            return notDDLStatement;
        });
    try {
        readQueues = arguments.stream().map(s -> SingleChronicleQueueBuilder.single(s).readOnly(true).build()).collect(Collectors.toList());
        iterators = readQueues.stream().map(ChronicleQueue::createTailer).map(tailer -> new FQLQueryIterator(tailer, readAhead)).collect(Collectors.toList());
        try (MergeIterator<FQLQuery, List<FQLQuery>> iter = MergeIterator.get(iterators, FQLQuery::compareTo, new Reducer());
            QueryReplayer replayer = new QueryReplayer(iter, targetHosts, resultPaths, filters, queryStorePath)) {
            replayer.replay();
        }
    } catch (Exception e) {
        throw new RuntimeException(e);
    } finally {
        if (iterators != null)
            iterators.forEach(AbstractIterator::close);
        if (readQueues != null)
            readQueues.forEach(Closeable::close);
    }
}