Examples with CloseableIterator org.apache.cassandra.utils.CloseableIterator used on opensource projects

Example 1 with CloseableIterator

use of org.apache.cassandra.utils.CloseableIterator in project eiger by wlloyd.

the class RowIteratorFactory method getIterator.

/**
 * Get a row iterator over the provided memtables and sstables, between the provided keys
 * and filtered by the queryfilter.
 * @param memtables Memtables pending flush.
 * @param sstables SStables to scan through.
 * @param startWith Start at this key
 * @param stopAt Stop and this key
 * @param filter Used to decide which columns to pull out
 * @param cfs
 * @return A row iterator following all the given restrictions
 */
public static CloseableIterator<Row> getIterator(final Iterable<Memtable> memtables, final Collection<SSTableReader> sstables, final RowPosition startWith, final RowPosition stopAt, final QueryFilter filter, final ColumnFamilyStore cfs) {
    // fetch data from current memtable, historical memtables, and SSTables in the correct order.
    final List<CloseableIterator<IColumnIterator>> iterators = new ArrayList<CloseableIterator<IColumnIterator>>();
    // memtables
    for (Memtable memtable : memtables) {
        iterators.add(new ConvertToColumnIterator(filter, memtable.getEntryIterator(startWith, stopAt)));
    }
    for (SSTableReader sstable : sstables) {
        final SSTableScanner scanner = sstable.getScanner(filter);
        scanner.seekTo(startWith);
        // otherwise we leak FDs
        assert scanner instanceof Closeable;
        iterators.add(scanner);
    }
    // reduce rows from all sources into a single row
    return MergeIterator.get(iterators, COMPARE_BY_KEY, new MergeIterator.Reducer<IColumnIterator, Row>() {

        private final int gcBefore = (int) (System.currentTimeMillis() / 1000) - cfs.metadata.getGcGraceSeconds();

        private final List<IColumnIterator> colIters = new ArrayList<IColumnIterator>();

        private DecoratedKey key;

        private ColumnFamily returnCF;

        @Override
        protected void onKeyChange() {
            this.returnCF = ColumnFamily.create(cfs.metadata);
        }

        public void reduce(IColumnIterator current) {
            this.colIters.add(current);
            this.key = current.getKey();
            this.returnCF.delete(current.getColumnFamily());
        }

        protected Row getReduced() {
            // First check if this row is in the rowCache. If it is we can skip the rest
            ColumnFamily cached = cfs.getRawCachedRow(key);
            if (cached == null)
                // not cached: collate
                filter.collateColumns(returnCF, colIters, gcBefore);
            else {
                QueryFilter keyFilter = new QueryFilter(key, filter.path, filter.filter);
                returnCF = cfs.filterColumnFamily(cached, keyFilter, gcBefore);
            }
            Row rv = new Row(key, returnCF);
            colIters.clear();
            key = null;
            return rv;
        }
    });
}

Example 2 with CloseableIterator

use of org.apache.cassandra.utils.CloseableIterator in project eiger by wlloyd.

the class CollationController method collectTimeOrderedData.

/**
 * Collects data in order of recency, using the sstable maxtimestamp data.
 * Once we have data for all requests columns that is newer than the newest remaining maxtimestamp,
 * we stop.
 */
private ColumnFamily collectTimeOrderedData() {
    logger.debug("collectTimeOrderedData");
    ISortedColumns.Factory factory = mutableColumns ? AtomicSortedColumns.factory() : TreeMapBackedSortedColumns.factory();
    ColumnFamily container = ColumnFamily.create(cfs.metadata, factory, filter.filter.isReversed());
    List<IColumnIterator> iterators = new ArrayList<IColumnIterator>();
    ColumnFamilyStore.ViewFragment view = cfs.markReferenced(filter.key);
    try {
        for (Memtable memtable : view.memtables) {
            IColumnIterator iter = filter.getMemtableColumnIterator(memtable);
            if (iter != null) {
                iterators.add(iter);
                container.delete(iter.getColumnFamily());
                while (iter.hasNext()) container.addColumn(iter.next());
            }
        }
        // avoid changing the filter columns of the original filter
        // (reduceNameFilter removes columns that are known to be irrelevant)
        TreeSet<ByteBuffer> filterColumns = new TreeSet<ByteBuffer>(((NamesQueryFilter) filter.filter).columns);
        QueryFilter reducedFilter = new QueryFilter(filter.key, filter.path, new NamesQueryFilter(filterColumns));
        /* add the SSTables on disk */
        Collections.sort(view.sstables, SSTable.maxTimestampComparator);
        // read sorted sstables
        for (SSTableReader sstable : view.sstables) {
            long currentMaxTs = sstable.getMaxTimestamp();
            reduceNameFilter(reducedFilter, container, currentMaxTs);
            if (((NamesQueryFilter) reducedFilter.filter).columns.isEmpty())
                break;
            IColumnIterator iter = reducedFilter.getSSTableColumnIterator(sstable);
            iterators.add(iter);
            if (iter.getColumnFamily() != null) {
                container.delete(iter.getColumnFamily());
                sstablesIterated++;
                while (iter.hasNext()) container.addColumn(iter.next());
            }
        }
        // and "there used to be data, but it's gone now" (we should cache the empty CF so we don't need to rebuild that slower)
        if (iterators.isEmpty())
            return null;
        // do a final collate.  toCollate is boilerplate required to provide a CloseableIterator
        final ColumnFamily c2 = container;
        CloseableIterator<IColumn> toCollate = new SimpleAbstractColumnIterator() {

            final Iterator<IColumn> iter = c2.iterator();

            protected IColumn computeNext() {
                return iter.hasNext() ? iter.next() : endOfData();
            }

            public ColumnFamily getColumnFamily() {
                return c2;
            }

            public DecoratedKey getKey() {
                return filter.key;
            }
        };
        ColumnFamily returnCF = container.cloneMeShallow();
        filter.collateColumns(returnCF, Collections.singletonList(toCollate), gcBefore);
        // "hoist up" the requested data into a more recent sstable
        if (sstablesIterated > cfs.getMinimumCompactionThreshold() && !cfs.isCompactionDisabled() && cfs.getCompactionStrategy() instanceof SizeTieredCompactionStrategy) {
            RowMutation rm = new RowMutation(cfs.table.name, new Row(filter.key, returnCF.cloneMe()));
            try {
                // skipping commitlog and index updates is fine since we're just de-fragmenting existing data
                Table.open(rm.getTable()).apply(rm, false, false);
            } catch (IOException e) {
                // log and allow the result to be returned
                logger.error("Error re-writing read results", e);
            }
        }
        // Caller is responsible for final removeDeletedCF.  This is important for cacheRow to work correctly:
        return returnCF;
    } finally {
        for (IColumnIterator iter : iterators) FileUtils.closeQuietly(iter);
        SSTableReader.releaseReferences(view.sstables);
    }
}

Example 3 with CloseableIterator

use of org.apache.cassandra.utils.CloseableIterator in project eiger by wlloyd.

the class RowRepairResolver method resolveSuperset.

static ColumnFamily resolveSuperset(Iterable<ColumnFamily> versions) {
    assert Iterables.size(versions) > 0;
    ColumnFamily resolved = null;
    for (ColumnFamily cf : versions) {
        if (cf == null)
            continue;
        if (resolved == null)
            resolved = cf.cloneMeShallow();
        else
            resolved.delete(cf);
    }
    if (resolved == null)
        return null;
    // mimic the collectCollatedColumn + removeDeleted path that getColumnFamily takes.
    // this will handle removing columns and subcolumns that are supressed by a row or
    // supercolumn tombstone.
    QueryFilter filter = new QueryFilter(null, new QueryPath(resolved.metadata().cfName), new IdentityQueryFilter());
    List<CloseableIterator<IColumn>> iters = new ArrayList<CloseableIterator<IColumn>>();
    for (ColumnFamily version : versions) {
        if (version == null)
            continue;
        iters.add(FBUtilities.closeableIterator(version.iterator()));
    }
    filter.collateColumns(resolved, iters, Integer.MIN_VALUE);
    return ColumnFamilyStore.removeDeleted(resolved, Integer.MIN_VALUE);
}