Examples with ReducingKeyIterator org.apache.cassandra.io.sstable.ReducingKeyIterator used on opensource projects

Example 1 with ReducingKeyIterator

use of org.apache.cassandra.io.sstable.ReducingKeyIterator in project cassandra by apache.

the class ViewBuilder method run.

public void run() {
    logger.trace("Running view builder for {}.{}", baseCfs.metadata.keyspace, view.name);
    UUID localHostId = SystemKeyspace.getLocalHostId();
    String ksname = baseCfs.metadata.keyspace, viewName = view.name;
    if (SystemKeyspace.isViewBuilt(ksname, viewName)) {
        if (!SystemKeyspace.isViewStatusReplicated(ksname, viewName))
            updateDistributed(ksname, viewName, localHostId);
        return;
    }
    Iterable<Range<Token>> ranges = StorageService.instance.getLocalRanges(baseCfs.metadata.keyspace);
    final Pair<Integer, Token> buildStatus = SystemKeyspace.getViewBuildStatus(ksname, viewName);
    Token lastToken;
    Function<org.apache.cassandra.db.lifecycle.View, Iterable<SSTableReader>> function;
    if (buildStatus == null) {
        baseCfs.forceBlockingFlush();
        function = org.apache.cassandra.db.lifecycle.View.selectFunction(SSTableSet.CANONICAL);
        int generation = Integer.MIN_VALUE;
        try (Refs<SSTableReader> temp = baseCfs.selectAndReference(function).refs) {
            for (SSTableReader reader : temp) {
                generation = Math.max(reader.descriptor.generation, generation);
            }
        }
        SystemKeyspace.beginViewBuild(ksname, viewName, generation);
        lastToken = null;
    } else {
        function = new Function<org.apache.cassandra.db.lifecycle.View, Iterable<SSTableReader>>() {

            @Nullable
            public Iterable<SSTableReader> apply(org.apache.cassandra.db.lifecycle.View view) {
                Iterable<SSTableReader> readers = org.apache.cassandra.db.lifecycle.View.selectFunction(SSTableSet.CANONICAL).apply(view);
                if (readers != null)
                    return Iterables.filter(readers, ssTableReader -> ssTableReader.descriptor.generation <= buildStatus.left);
                return null;
            }
        };
        lastToken = buildStatus.right;
    }
    prevToken = lastToken;
    try (Refs<SSTableReader> sstables = baseCfs.selectAndReference(function).refs;
        ReducingKeyIterator iter = new ReducingKeyIterator(sstables)) {
        SystemDistributedKeyspace.startViewBuild(ksname, viewName, localHostId);
        while (!isStopped && iter.hasNext()) {
            DecoratedKey key = iter.next();
            Token token = key.getToken();
            if (lastToken == null || lastToken.compareTo(token) < 0) {
                for (Range<Token> range : ranges) {
                    if (range.contains(token)) {
                        buildKey(key);
                        if (prevToken == null || prevToken.compareTo(token) != 0) {
                            SystemKeyspace.updateViewBuildStatus(ksname, viewName, key.getToken());
                            prevToken = token;
                        }
                    }
                }
                lastToken = null;
            }
        }
        if (!isStopped) {
            SystemKeyspace.finishViewBuildStatus(ksname, viewName);
            updateDistributed(ksname, viewName, localHostId);
        }
    } catch (Exception e) {
        ScheduledExecutors.nonPeriodicTasks.schedule(() -> CompactionManager.instance.submitViewBuilder(this), 5, TimeUnit.MINUTES);
        logger.warn("Materialized View failed to complete, sleeping 5 minutes before restarting", e);
    }
}

Example 2 with ReducingKeyIterator

use of org.apache.cassandra.io.sstable.ReducingKeyIterator in project spoon by INRIA.

the class SecondaryIndexManager method buildIndexesBlocking.

private void buildIndexesBlocking(Collection<SSTableReader> sstables, Set<Index> indexes) {
    if (indexes.isEmpty())
        return;
    logger.info("Submitting index build of {} for data in {}", indexes.stream().map(i -> i.getIndexMetadata().name).collect(Collectors.joining(",")), sstables.stream().map(SSTableReader::toString).collect(Collectors.joining(",")));
    SecondaryIndexBuilder builder = new SecondaryIndexBuilder(baseCfs, indexes, new ReducingKeyIterator(sstables));
    Future<?> future = CompactionManager.instance.submitIndexBuild(builder);
    FBUtilities.waitOnFuture(future);
    flushIndexesBlocking(indexes);
    logger.info("Index build of {} complete", indexes.stream().map(i -> i.getIndexMetadata().name).collect(Collectors.joining(",")));
}

Example 3 with ReducingKeyIterator

use of org.apache.cassandra.io.sstable.ReducingKeyIterator in project cassandra by apache.

the class CassandraIndex method buildBlocking.

@SuppressWarnings("resource")
private void buildBlocking() {
    baseCfs.forceBlockingFlush();
    try (ColumnFamilyStore.RefViewFragment viewFragment = baseCfs.selectAndReference(View.selectFunction(SSTableSet.CANONICAL));
        Refs<SSTableReader> sstables = viewFragment.refs) {
        if (sstables.isEmpty()) {
            logger.info("No SSTable data for {}.{} to build index {} from, marking empty index as built", baseCfs.metadata.keyspace, baseCfs.metadata.name, metadata.name);
            return;
        }
        logger.info("Submitting index build of {} for data in {}", metadata.name, getSSTableNames(sstables));
        SecondaryIndexBuilder builder = new CollatedViewIndexBuilder(baseCfs, Collections.singleton(this), new ReducingKeyIterator(sstables), ImmutableSet.copyOf(sstables));
        Future<?> future = CompactionManager.instance.submitIndexBuild(builder);
        FBUtilities.waitOnFuture(future);
        indexCfs.forceBlockingFlush();
    }
    logger.info("Index build of {} complete", metadata.name);
}

Example 4 with ReducingKeyIterator

use of org.apache.cassandra.io.sstable.ReducingKeyIterator in project cassandra by apache.

the class CompactionAllocationTest method testIndexingWidePartitions.

private static void testIndexingWidePartitions(String name, int numSSTable, int sstablePartitions, IndexDef... indexes) throws Throwable {
    String ksname = "ks_" + name.toLowerCase();
    SchemaLoader.createKeyspace(ksname, KeyspaceParams.simple(1), CreateTableStatement.parse("CREATE TABLE tbl (k text, c text, v1 text, v2 text, v3 text, v4 text, PRIMARY KEY (k, c))", ksname).build());
    ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(Schema.instance.getTableMetadata(ksname, "tbl").id);
    Assert.assertNotNull(cfs);
    cfs.disableAutoCompaction();
    int rowWidth = 100;
    int rowsPerPartition = 1000;
    measure(new Workload() {

        @SuppressWarnings("UnstableApiUsage")
        public void setup() {
            cfs.disableAutoCompaction();
            String insert = String.format("INSERT INTO %s.%s (k, c, v1, v2, v3, v4) VALUES (?, ?, ?, ?, ?, ?)", ksname, "tbl");
            for (int f = 0; f < numSSTable; f++) {
                for (int p = 0; p < sstablePartitions; p++) {
                    String key = String.format("%08d", (f * sstablePartitions) + p);
                    for (int r = 0; r < rowsPerPartition; r++) {
                        QueryProcessor.executeInternal(insert, key, makeRandomString(6, -1), makeRandomString(rowWidth >> 2), makeRandomString(rowWidth >> 2), makeRandomString(rowWidth >> 2), makeRandomString(rowWidth >> 2));
                    }
                }
                cfs.forceBlockingFlush();
            }
            for (IndexDef index : indexes) {
                QueryProcessor.executeInternal(String.format(index.cql, index.name, ksname, "tbl"));
                while (!cfs.indexManager.getBuiltIndexNames().contains(index.name)) Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
            }
            Assert.assertEquals(numSSTable, cfs.getLiveSSTables().size());
        }

        public ColumnFamilyStore getCfs() {
            return cfs;
        }

        public List<Runnable> getReads() {
            return new ArrayList<>();
        }

        public String name() {
            return name;
        }

        public int executeReads() {
            // return 1 to avoid divide by zero error
            return 1;
        }

        public void executeCompactions() {
            logger.info("Starting index re-build");
            try (ColumnFamilyStore.RefViewFragment viewFragment = cfs.selectAndReference(View.selectFunction(SSTableSet.CANONICAL));
                Refs<SSTableReader> sstables = viewFragment.refs) {
                Set<Index> indexes = new HashSet<>(cfs.indexManager.listIndexes());
                SecondaryIndexBuilder builder = new CollatedViewIndexBuilder(cfs, indexes, new ReducingKeyIterator(sstables), ImmutableSet.copyOf(sstables));
                builder.build();
            }
            logger.info("Index re-build complete");
        }

        public int[] getSSTableStats() {
            int numPartitions = cfs.getLiveSSTables().stream().mapToInt(sstable -> Ints.checkedCast(sstable.getSSTableMetadata().estimatedPartitionSize.count())).sum();
            int numRows = cfs.getLiveSSTables().stream().mapToInt(sstable -> Ints.checkedCast(sstable.getSSTableMetadata().totalRows)).sum();
            return new int[] { numPartitions, numRows };
        }
    });
}

Example 5 with ReducingKeyIterator

use of org.apache.cassandra.io.sstable.ReducingKeyIterator in project cassandra by apache.

the class ViewBuilderTask method call.

public Long call() {
    String ksName = baseCfs.metadata.keyspace;
    if (prevToken == null)
        logger.debug("Starting new view build for range {}", range);
    else
        logger.debug("Resuming view build for range {} from token {} with {} covered keys", range, prevToken, keysBuilt);
    /*
         * It's possible for view building to start before MV creation got propagated to other nodes. For this reason
         * we should wait for schema to converge before attempting to send any view mutations to other nodes, or else
         * face UnknownTableException upon Mutation deserialization on the nodes that haven't processed the schema change.
         */
    boolean schemaConverged = Gossiper.instance.waitForSchemaAgreement(10, TimeUnit.SECONDS, () -> this.isStopped);
    if (!schemaConverged)
        logger.warn("Failed to get schema to converge before building view {}.{}", baseCfs.keyspace.getName(), view.name);
    Function<org.apache.cassandra.db.lifecycle.View, Iterable<SSTableReader>> function;
    function = org.apache.cassandra.db.lifecycle.View.select(SSTableSet.CANONICAL, s -> range.intersects(s.getBounds()));
    try (ColumnFamilyStore.RefViewFragment viewFragment = baseCfs.selectAndReference(function);
        Refs<SSTableReader> sstables = viewFragment.refs;
        ReducingKeyIterator keyIter = new ReducingKeyIterator(sstables)) {
        PeekingIterator<DecoratedKey> iter = Iterators.peekingIterator(keyIter);
        while (!isStopped && iter.hasNext()) {
            DecoratedKey key = iter.next();
            Token token = key.getToken();
            // skip tokens already built or not present in range
            if (range.contains(token) && (prevToken == null || token.compareTo(prevToken) > 0)) {
                buildKey(key);
                ++keysBuilt;
                // build other keys sharing the same token
                while (iter.hasNext() && iter.peek().getToken().equals(token)) {
                    key = iter.next();
                    buildKey(key);
                    ++keysBuilt;
                }
                if (keysBuilt % ROWS_BETWEEN_CHECKPOINTS == 1)
                    SystemKeyspace.updateViewBuildStatus(ksName, view.name, range, token, keysBuilt);
                prevToken = token;
            }
        }
    }
    finish();
    return keysBuilt;
}