Examples with ReducingKeyIterator org.apache.cassandra.io.sstable.ReducingKeyIterator used on opensource projects

Example 6 with ReducingKeyIterator

use of org.apache.cassandra.io.sstable.ReducingKeyIterator in project cassandra by apache.

the class CustomCassandraIndex method buildBlocking.

private void buildBlocking() {
    baseCfs.forceBlockingFlush();
    try (ColumnFamilyStore.RefViewFragment viewFragment = baseCfs.selectAndReference(View.selectFunction(SSTableSet.CANONICAL));
        Refs<SSTableReader> sstables = viewFragment.refs) {
        if (sstables.isEmpty()) {
            logger.info("No SSTable data for {}.{} to build index {} from, marking empty index as built", baseCfs.metadata.keyspace, baseCfs.metadata.name, metadata.name);
            return;
        }
        logger.info("Submitting index build of {} for data in {}", metadata.name, getSSTableNames(sstables));
        SecondaryIndexBuilder builder = new CollatedViewIndexBuilder(baseCfs, Collections.singleton(this), new ReducingKeyIterator(sstables), ImmutableSet.copyOf(sstables));
        Future<?> future = CompactionManager.instance.submitIndexBuild(builder);
        FBUtilities.waitOnFuture(future);
        indexCfs.forceBlockingFlush();
    }
    logger.info("Index build of {} complete", metadata.name);
}

Example 7 with ReducingKeyIterator

use of org.apache.cassandra.io.sstable.ReducingKeyIterator in project eiger by wlloyd.

the class SecondaryIndexManager method maybeBuildSecondaryIndexes.

/**
 * Does a full, blocking rebuild of the indexes specified by columns from the sstables.
 * Does nothing if columns is empty.
 *
 * Caller must acquire and release references to the sstables used here.
 *
 * @param sstables the data to build from
 * @param columns the list of columns to index
 * @throws IOException
 */
public void maybeBuildSecondaryIndexes(Collection<SSTableReader> sstables, SortedSet<ByteBuffer> columns) throws IOException {
    if (columns.isEmpty())
        return;
    logger.info(String.format("Submitting index build of %s for data in %s", baseCfs.metadata.comparator.getString(columns), StringUtils.join(sstables, ", ")));
    SecondaryIndexBuilder builder = new SecondaryIndexBuilder(baseCfs, columns, new ReducingKeyIterator(sstables));
    Future<?> future = CompactionManager.instance.submitIndexBuild(builder);
    try {
        future.get();
        flushIndexesBlocking();
    } catch (InterruptedException e) {
        throw new AssertionError(e);
    } catch (ExecutionException e) {
        throw new RuntimeException(e);
    }
    logger.info("Index build of " + baseCfs.metadata.comparator.getString(columns) + " complete");
}

Example 8 with ReducingKeyIterator

use of org.apache.cassandra.io.sstable.ReducingKeyIterator in project cassandra by apache.

the class CancelCompactionsTest method testIndexRebuild.

/**
 * Make sure index rebuilds get cancelled
 */
@Test
public void testIndexRebuild() throws ExecutionException, InterruptedException {
    ColumnFamilyStore cfs = MockSchema.newCFS();
    List<SSTableReader> sstables = createSSTables(cfs, 5, 0);
    Index idx = new StubIndex(cfs, null);
    CountDownLatch indexBuildStarted = new CountDownLatch(1);
    CountDownLatch indexBuildRunning = new CountDownLatch(1);
    CountDownLatch compactionsStopped = new CountDownLatch(1);
    ReducingKeyIterator reducingKeyIterator = new ReducingKeyIterator(sstables) {

        @Override
        public boolean hasNext() {
            indexBuildStarted.countDown();
            try {
                indexBuildRunning.await();
            } catch (InterruptedException e) {
                throw new RuntimeException();
            }
            return false;
        }
    };
    Future<?> f = CompactionManager.instance.submitIndexBuild(new CollatedViewIndexBuilder(cfs, Collections.singleton(idx), reducingKeyIterator, ImmutableSet.copyOf(sstables)));
    // wait for hasNext to get called
    indexBuildStarted.await();
    assertEquals(1, getActiveCompactionsForTable(cfs).size());
    boolean foundCompaction = false;
    for (CompactionInfo.Holder holder : getActiveCompactionsForTable(cfs)) {
        if (holder.getCompactionInfo().getSSTables().equals(new HashSet<>(sstables))) {
            assertFalse(holder.isStopRequested());
            foundCompaction = true;
        }
    }
    assertTrue(foundCompaction);
    cfs.runWithCompactionsDisabled(() -> {
        compactionsStopped.countDown();
        return null;
    }, (sstable) -> true, false, false, true);
    // wait for the runWithCompactionsDisabled callable
    compactionsStopped.await();
    assertEquals(1, getActiveCompactionsForTable(cfs).size());
    foundCompaction = false;
    for (CompactionInfo.Holder holder : getActiveCompactionsForTable(cfs)) {
        if (holder.getCompactionInfo().getSSTables().equals(new HashSet<>(sstables))) {
            assertTrue(holder.isStopRequested());
            foundCompaction = true;
        }
    }
    assertTrue(foundCompaction);
    // signal that the index build should be finished
    indexBuildRunning.countDown();
    f.get();
    assertTrue(getActiveCompactionsForTable(cfs).isEmpty());
}

Example 9 with ReducingKeyIterator

use of org.apache.cassandra.io.sstable.ReducingKeyIterator in project eiger by wlloyd.

the class SecondaryIndex method buildIndexBlocking.

/**
 * Builds the index using the data in the underlying CFS
 * Blocks till it's complete
 */
protected void buildIndexBlocking() throws IOException {
    logger.info(String.format("Submitting index build of %s for data in %s", getIndexName(), StringUtils.join(baseCfs.getSSTables(), ", ")));
    SortedSet<ByteBuffer> columnNames = new TreeSet<ByteBuffer>();
    for (ColumnDefinition cdef : columnDefs) columnNames.add(cdef.name);
    Collection<SSTableReader> sstables = baseCfs.markCurrentSSTablesReferenced();
    SecondaryIndexBuilder builder = new SecondaryIndexBuilder(baseCfs, columnNames, new ReducingKeyIterator(sstables));
    Future<?> future = CompactionManager.instance.submitIndexBuild(builder);
    try {
        future.get();
        forceBlockingFlush();
        // Mark all indexed columns as built
        if (this instanceof PerRowSecondaryIndex) {
            for (ByteBuffer columnName : columnNames) SystemTable.setIndexBuilt(baseCfs.table.name, getIndexName() + ByteBufferUtil.string(columnName));
        } else {
            SystemTable.setIndexBuilt(baseCfs.table.name, getIndexName());
        }
    } catch (InterruptedException e) {
        throw new AssertionError(e);
    } catch (ExecutionException e) {
        throw new IOException(e);
    } finally {
        SSTableReader.releaseReferences(sstables);
    }
    logger.info("Index build of " + getIndexName() + " complete");
}