Examples with SecondaryIndexBuilder org.apache.cassandra.index.SecondaryIndexBuilder used on opensource projects

Example 1 with SecondaryIndexBuilder

use of org.apache.cassandra.index.SecondaryIndexBuilder in project cassandra by apache.

the class ActiveCompactionsTest method testActiveCompactionTrackingRaceWithIndexBuilder.

@Test
public void testActiveCompactionTrackingRaceWithIndexBuilder() throws Throwable {
    createTable("CREATE TABLE %s (pk int, ck int, a int, b int, PRIMARY KEY (pk, ck))");
    String idxName = createIndex("CREATE INDEX on %s(a)");
    getCurrentColumnFamilyStore().disableAutoCompaction();
    for (int i = 0; i < 5; i++) {
        execute("INSERT INTO %s (pk, ck, a, b) VALUES (" + i + ", 2, 3, 4)");
        getCurrentColumnFamilyStore().forceBlockingFlush();
    }
    Index idx = getCurrentColumnFamilyStore().indexManager.getIndexByName(idxName);
    Set<SSTableReader> sstables = getCurrentColumnFamilyStore().getLiveSSTables();
    ExecutorService es = Executors.newFixedThreadPool(2);
    final int loopCount = 5000;
    for (int ii = 0; ii < loopCount; ii++) {
        CountDownLatch trigger = new CountDownLatch(1);
        SecondaryIndexBuilder builder = idx.getBuildTaskSupport().getIndexBuildTask(getCurrentColumnFamilyStore(), Collections.singleton(idx), sstables);
        Future<?> f1 = es.submit(() -> {
            Uninterruptibles.awaitUninterruptibly(trigger);
            try {
                CompactionManager.instance.submitIndexBuild(builder).get();
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        });
        Future<?> f2 = es.submit(() -> {
            Uninterruptibles.awaitUninterruptibly(trigger);
            CompactionManager.instance.active.getCompactionsForSSTable(null, null);
        });
        trigger.countDown();
        FBUtilities.waitOnFutures(Arrays.asList(f1, f2));
    }
    es.shutdown();
    es.awaitTermination(1, TimeUnit.MINUTES);
}

Example 2 with SecondaryIndexBuilder

use of org.apache.cassandra.index.SecondaryIndexBuilder in project cassandra by apache.

the class ActiveCompactionsTest method testSecondaryIndexTracking.

@Test
public void testSecondaryIndexTracking() throws Throwable {
    createTable("CREATE TABLE %s (pk int, ck int, a int, b int, PRIMARY KEY (pk, ck))");
    String idxName = createIndex("CREATE INDEX on %s(a)");
    getCurrentColumnFamilyStore().disableAutoCompaction();
    for (int i = 0; i < 5; i++) {
        execute("INSERT INTO %s (pk, ck, a, b) VALUES (" + i + ", 2, 3, 4)");
        getCurrentColumnFamilyStore().forceBlockingFlush();
    }
    Index idx = getCurrentColumnFamilyStore().indexManager.getIndexByName(idxName);
    Set<SSTableReader> sstables = getCurrentColumnFamilyStore().getLiveSSTables();
    SecondaryIndexBuilder builder = idx.getBuildTaskSupport().getIndexBuildTask(getCurrentColumnFamilyStore(), Collections.singleton(idx), sstables);
    MockActiveCompactions mockActiveCompactions = new MockActiveCompactions();
    CompactionManager.instance.submitIndexBuild(builder, mockActiveCompactions).get();
    assertTrue(mockActiveCompactions.finished);
    assertNotNull(mockActiveCompactions.holder);
    assertEquals(sstables, mockActiveCompactions.holder.getCompactionInfo().getSSTables());
}

Example 3 with SecondaryIndexBuilder

use of org.apache.cassandra.index.SecondaryIndexBuilder in project cassandra by apache.

the class CompactionAllocationTest method testIndexingWidePartitions.

private static void testIndexingWidePartitions(String name, int numSSTable, int sstablePartitions, IndexDef... indexes) throws Throwable {
    String ksname = "ks_" + name.toLowerCase();
    SchemaLoader.createKeyspace(ksname, KeyspaceParams.simple(1), CreateTableStatement.parse("CREATE TABLE tbl (k text, c text, v1 text, v2 text, v3 text, v4 text, PRIMARY KEY (k, c))", ksname).build());
    ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(Schema.instance.getTableMetadata(ksname, "tbl").id);
    Assert.assertNotNull(cfs);
    cfs.disableAutoCompaction();
    int rowWidth = 100;
    int rowsPerPartition = 1000;
    measure(new Workload() {

        @SuppressWarnings("UnstableApiUsage")
        public void setup() {
            cfs.disableAutoCompaction();
            String insert = String.format("INSERT INTO %s.%s (k, c, v1, v2, v3, v4) VALUES (?, ?, ?, ?, ?, ?)", ksname, "tbl");
            for (int f = 0; f < numSSTable; f++) {
                for (int p = 0; p < sstablePartitions; p++) {
                    String key = String.format("%08d", (f * sstablePartitions) + p);
                    for (int r = 0; r < rowsPerPartition; r++) {
                        QueryProcessor.executeInternal(insert, key, makeRandomString(6, -1), makeRandomString(rowWidth >> 2), makeRandomString(rowWidth >> 2), makeRandomString(rowWidth >> 2), makeRandomString(rowWidth >> 2));
                    }
                }
                cfs.forceBlockingFlush();
            }
            for (IndexDef index : indexes) {
                QueryProcessor.executeInternal(String.format(index.cql, index.name, ksname, "tbl"));
                while (!cfs.indexManager.getBuiltIndexNames().contains(index.name)) Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
            }
            Assert.assertEquals(numSSTable, cfs.getLiveSSTables().size());
        }

        public ColumnFamilyStore getCfs() {
            return cfs;
        }

        public List<Runnable> getReads() {
            return new ArrayList<>();
        }

        public String name() {
            return name;
        }

        public int executeReads() {
            // return 1 to avoid divide by zero error
            return 1;
        }

        public void executeCompactions() {
            logger.info("Starting index re-build");
            try (ColumnFamilyStore.RefViewFragment viewFragment = cfs.selectAndReference(View.selectFunction(SSTableSet.CANONICAL));
                Refs<SSTableReader> sstables = viewFragment.refs) {
                Set<Index> indexes = new HashSet<>(cfs.indexManager.listIndexes());
                SecondaryIndexBuilder builder = new CollatedViewIndexBuilder(cfs, indexes, new ReducingKeyIterator(sstables), ImmutableSet.copyOf(sstables));
                builder.build();
            }
            logger.info("Index re-build complete");
        }

        public int[] getSSTableStats() {
            int numPartitions = cfs.getLiveSSTables().stream().mapToInt(sstable -> Ints.checkedCast(sstable.getSSTableMetadata().estimatedPartitionSize.count())).sum();
            int numRows = cfs.getLiveSSTables().stream().mapToInt(sstable -> Ints.checkedCast(sstable.getSSTableMetadata().totalRows)).sum();
            return new int[] { numPartitions, numRows };
        }
    });
}

Example 4 with SecondaryIndexBuilder

use of org.apache.cassandra.index.SecondaryIndexBuilder in project cassandra by apache.

the class CustomCassandraIndex method buildBlocking.

private void buildBlocking() {
    baseCfs.forceBlockingFlush();
    try (ColumnFamilyStore.RefViewFragment viewFragment = baseCfs.selectAndReference(View.selectFunction(SSTableSet.CANONICAL));
        Refs<SSTableReader> sstables = viewFragment.refs) {
        if (sstables.isEmpty()) {
            logger.info("No SSTable data for {}.{} to build index {} from, marking empty index as built", baseCfs.metadata.keyspace, baseCfs.metadata.name, metadata.name);
            return;
        }
        logger.info("Submitting index build of {} for data in {}", metadata.name, getSSTableNames(sstables));
        SecondaryIndexBuilder builder = new CollatedViewIndexBuilder(baseCfs, Collections.singleton(this), new ReducingKeyIterator(sstables), ImmutableSet.copyOf(sstables));
        Future<?> future = CompactionManager.instance.submitIndexBuild(builder);
        FBUtilities.waitOnFuture(future);
        indexCfs.forceBlockingFlush();
    }
    logger.info("Index build of {} complete", metadata.name);
}