Examples with CompactionAwareWriter org.apache.cassandra.db.compaction.writers.CompactionAwareWriter used on opensource projects

Example 1 with CompactionAwareWriter

use of org.apache.cassandra.db.compaction.writers.CompactionAwareWriter in project cassandra by apache.

the class CompactionAwareWriterTest method testSplittingSizeTieredCompactionWriter.

@Test
public void testSplittingSizeTieredCompactionWriter() throws Throwable {
    ColumnFamilyStore cfs = getColumnFamilyStore();
    cfs.disableAutoCompaction();
    int rowCount = 10000;
    populate(rowCount);
    LifecycleTransaction txn = cfs.getTracker().tryModify(cfs.getLiveSSTables(), OperationType.COMPACTION);
    long beforeSize = txn.originals().iterator().next().onDiskLength();
    CompactionAwareWriter writer = new SplittingSizeTieredCompactionWriter(cfs, cfs.getDirectories(), txn, txn.originals(), 0);
    int rows = compact(cfs, txn, writer);
    long expectedSize = beforeSize / 2;
    List<SSTableReader> sortedSSTables = new ArrayList<>(cfs.getLiveSSTables());
    Collections.sort(sortedSSTables, new Comparator<SSTableReader>() {

        @Override
        public int compare(SSTableReader o1, SSTableReader o2) {
            return Longs.compare(o2.onDiskLength(), o1.onDiskLength());
        }
    });
    for (SSTableReader sstable : sortedSSTables) {
        // we dont create smaller files than this, everything will be in the last file
        if (expectedSize > SplittingSizeTieredCompactionWriter.DEFAULT_SMALLEST_SSTABLE_BYTES)
            // allow 1% diff in estimated vs actual size
            assertEquals(expectedSize, sstable.onDiskLength(), expectedSize / 100);
        expectedSize /= 2;
    }
    assertEquals(rowCount, rows);
    validateData(cfs, rowCount);
    cfs.truncateBlocking();
}

Example 2 with CompactionAwareWriter

use of org.apache.cassandra.db.compaction.writers.CompactionAwareWriter in project cassandra by apache.

the class CompactionTask method runMayThrow.

/**
 * For internal use and testing only.  The rest of the system should go through the submit* methods,
 * which are properly serialized.
 * Caller is in charge of marking/unmarking the sstables as compacting.
 */
protected void runMayThrow() throws Exception {
    // it is not empty, it may compact down to nothing if all rows are deleted.
    assert transaction != null;
    if (transaction.originals().isEmpty())
        return;
    // Note that the current compaction strategy, is not necessarily the one this task was created under.
    // This should be harmless; see comments to CFS.maybeReloadCompactionStrategy.
    CompactionStrategyManager strategy = cfs.getCompactionStrategyManager();
    if (DatabaseDescriptor.isSnapshotBeforeCompaction()) {
        long epochMilli = currentTimeMillis();
        Instant creationTime = Instant.ofEpochMilli(epochMilli);
        cfs.snapshotWithoutFlush(epochMilli + "-compact-" + cfs.name, creationTime);
    }
    try (CompactionController controller = getCompactionController(transaction.originals())) {
        final Set<SSTableReader> fullyExpiredSSTables = controller.getFullyExpiredSSTables();
        // select SSTables to compact based on available disk space.
        buildCompactionCandidatesForAvailableDiskSpace(fullyExpiredSSTables);
        // sanity check: all sstables must belong to the same cfs
        assert !Iterables.any(transaction.originals(), new Predicate<SSTableReader>() {

            @Override
            public boolean apply(SSTableReader sstable) {
                return !sstable.descriptor.cfname.equals(cfs.name);
            }
        });
        UUID taskId = transaction.opId();
        // new sstables from flush can be added during a compaction, but only the compaction can remove them,
        // so in our single-threaded compaction world this is a valid way of determining if we're compacting
        // all the sstables (that existed when we started)
        StringBuilder ssTableLoggerMsg = new StringBuilder("[");
        for (SSTableReader sstr : transaction.originals()) {
            ssTableLoggerMsg.append(String.format("%s:level=%d, ", sstr.getFilename(), sstr.getSSTableLevel()));
        }
        ssTableLoggerMsg.append("]");
        logger.info("Compacting ({}) {}", taskId, ssTableLoggerMsg);
        RateLimiter limiter = CompactionManager.instance.getRateLimiter();
        long start = nanoTime();
        long startTime = currentTimeMillis();
        long totalKeysWritten = 0;
        long estimatedKeys = 0;
        long inputSizeBytes;
        long timeSpentWritingKeys;
        Set<SSTableReader> actuallyCompact = Sets.difference(transaction.originals(), fullyExpiredSSTables);
        Collection<SSTableReader> newSStables;
        long[] mergedRowCounts;
        long totalSourceCQLRows;
        int nowInSec = FBUtilities.nowInSeconds();
        try (Refs<SSTableReader> refs = Refs.ref(actuallyCompact);
            AbstractCompactionStrategy.ScannerList scanners = strategy.getScanners(actuallyCompact);
            CompactionIterator ci = new CompactionIterator(compactionType, scanners.scanners, controller, nowInSec, taskId)) {
            long lastCheckObsoletion = start;
            inputSizeBytes = scanners.getTotalCompressedSize();
            double compressionRatio = scanners.getCompressionRatio();
            if (compressionRatio == MetadataCollector.NO_COMPRESSION_RATIO)
                compressionRatio = 1.0;
            long lastBytesScanned = 0;
            activeCompactions.beginCompaction(ci);
            try (CompactionAwareWriter writer = getCompactionAwareWriter(cfs, getDirectories(), transaction, actuallyCompact)) {
                // block until the below exception is thrown and the transaction is cancelled.
                if (!controller.cfs.getCompactionStrategyManager().isActive())
                    throw new CompactionInterruptedException(ci.getCompactionInfo());
                estimatedKeys = writer.estimatedKeys();
                while (ci.hasNext()) {
                    if (writer.append(ci.next()))
                        totalKeysWritten++;
                    long bytesScanned = scanners.getTotalBytesScanned();
                    // Rate limit the scanners, and account for compression
                    CompactionManager.compactionRateLimiterAcquire(limiter, bytesScanned, lastBytesScanned, compressionRatio);
                    lastBytesScanned = bytesScanned;
                    if (nanoTime() - lastCheckObsoletion > TimeUnit.MINUTES.toNanos(1L)) {
                        controller.maybeRefreshOverlaps();
                        lastCheckObsoletion = nanoTime();
                    }
                }
                timeSpentWritingKeys = TimeUnit.NANOSECONDS.toMillis(nanoTime() - start);
                // point of no return
                newSStables = writer.finish();
            } finally {
                activeCompactions.finishCompaction(ci);
                mergedRowCounts = ci.getMergedRowCounts();
                totalSourceCQLRows = ci.getTotalSourceCQLRows();
            }
        }
        if (transaction.isOffline())
            return;
        // log a bunch of statistics about the result and save to system table compaction_history
        long durationInNano = nanoTime() - start;
        long dTime = TimeUnit.NANOSECONDS.toMillis(durationInNano);
        long startsize = inputSizeBytes;
        long endsize = SSTableReader.getTotalBytes(newSStables);
        double ratio = (double) endsize / (double) startsize;
        StringBuilder newSSTableNames = new StringBuilder();
        for (SSTableReader reader : newSStables) newSSTableNames.append(reader.descriptor.baseFilename()).append(",");
        long totalSourceRows = 0;
        for (int i = 0; i < mergedRowCounts.length; i++) totalSourceRows += mergedRowCounts[i] * (i + 1);
        String mergeSummary = updateCompactionHistory(cfs.keyspace.getName(), cfs.getTableName(), mergedRowCounts, startsize, endsize);
        logger.info(String.format("Compacted (%s) %d sstables to [%s] to level=%d.  %s to %s (~%d%% of original) in %,dms.  Read Throughput = %s, Write Throughput = %s, Row Throughput = ~%,d/s.  %,d total partitions merged to %,d.  Partition merge counts were {%s}. Time spent writing keys = %,dms", taskId, transaction.originals().size(), newSSTableNames.toString(), getLevel(), FBUtilities.prettyPrintMemory(startsize), FBUtilities.prettyPrintMemory(endsize), (int) (ratio * 100), dTime, FBUtilities.prettyPrintMemoryPerSecond(startsize, durationInNano), FBUtilities.prettyPrintMemoryPerSecond(endsize, durationInNano), (int) totalSourceCQLRows / (TimeUnit.NANOSECONDS.toSeconds(durationInNano) + 1), totalSourceRows, totalKeysWritten, mergeSummary, timeSpentWritingKeys));
        if (logger.isTraceEnabled()) {
            logger.trace("CF Total Bytes Compacted: {}", FBUtilities.prettyPrintMemory(CompactionTask.addToTotalBytesCompacted(endsize)));
            logger.trace("Actual #keys: {}, Estimated #keys:{}, Err%: {}", totalKeysWritten, estimatedKeys, ((double) (totalKeysWritten - estimatedKeys) / totalKeysWritten));
        }
        cfs.getCompactionStrategyManager().compactionLogger.compaction(startTime, transaction.originals(), currentTimeMillis(), newSStables);
        // update the metrics
        cfs.metric.compactionBytesWritten.inc(endsize);
    }
}

Example 3 with CompactionAwareWriter

use of org.apache.cassandra.db.compaction.writers.CompactionAwareWriter in project cassandra by apache.

the class CompactionAwareWriterTest method testMajorLeveledCompactionWriter.

@Test
public void testMajorLeveledCompactionWriter() throws Throwable {
    ColumnFamilyStore cfs = getColumnFamilyStore();
    cfs.disableAutoCompaction();
    int rowCount = 20000;
    int targetSSTableCount = 50;
    populate(rowCount);
    LifecycleTransaction txn = cfs.getTracker().tryModify(cfs.getLiveSSTables(), OperationType.COMPACTION);
    long beforeSize = txn.originals().iterator().next().onDiskLength();
    int sstableSize = (int) beforeSize / targetSSTableCount;
    CompactionAwareWriter writer = new MajorLeveledCompactionWriter(cfs, cfs.getDirectories(), txn, txn.originals(), sstableSize);
    int rows = compact(cfs, txn, writer);
    assertEquals(targetSSTableCount, cfs.getLiveSSTables().size());
    int[] levelCounts = new int[5];
    assertEquals(rowCount, rows);
    for (SSTableReader sstable : cfs.getLiveSSTables()) {
        levelCounts[sstable.getSSTableLevel()]++;
    }
    assertEquals(0, levelCounts[0]);
    assertEquals(10, levelCounts[1]);
    // note that if we want more levels, fix this
    assertEquals(targetSSTableCount - 10, levelCounts[2]);
    for (int i = 3; i < levelCounts.length; i++) assertEquals(0, levelCounts[i]);
    validateData(cfs, rowCount);
    cfs.truncateBlocking();
}

Example 4 with CompactionAwareWriter

use of org.apache.cassandra.db.compaction.writers.CompactionAwareWriter in project cassandra by apache.

the class CompactionAwareWriterTest method testMaxSSTableSizeWriter.

@Test
public void testMaxSSTableSizeWriter() throws Throwable {
    ColumnFamilyStore cfs = getColumnFamilyStore();
    cfs.disableAutoCompaction();
    int rowCount = 1000;
    populate(rowCount);
    LifecycleTransaction txn = cfs.getTracker().tryModify(cfs.getLiveSSTables(), OperationType.COMPACTION);
    long beforeSize = txn.originals().iterator().next().onDiskLength();
    int sstableSize = (int) beforeSize / 10;
    CompactionAwareWriter writer = new MaxSSTableSizeWriter(cfs, cfs.getDirectories(), txn, txn.originals(), sstableSize, 0);
    int rows = compact(cfs, txn, writer);
    assertEquals(10, cfs.getLiveSSTables().size());
    assertEquals(rowCount, rows);
    validateData(cfs, rowCount);
    cfs.truncateBlocking();
}

Example 5 with CompactionAwareWriter

use of org.apache.cassandra.db.compaction.writers.CompactionAwareWriter in project cassandra by apache.

the class CompactionAwareWriterTest method testDefaultCompactionWriter.

@Test
public void testDefaultCompactionWriter() throws Throwable {
    Keyspace ks = Keyspace.open(KEYSPACE);
    ColumnFamilyStore cfs = ks.getColumnFamilyStore(TABLE);
    int rowCount = 1000;
    cfs.disableAutoCompaction();
    populate(rowCount);
    LifecycleTransaction txn = cfs.getTracker().tryModify(cfs.getLiveSSTables(), OperationType.COMPACTION);
    long beforeSize = txn.originals().iterator().next().onDiskLength();
    CompactionAwareWriter writer = new DefaultCompactionWriter(cfs, cfs.getDirectories(), txn, txn.originals());
    int rows = compact(cfs, txn, writer);
    assertEquals(1, cfs.getLiveSSTables().size());
    assertEquals(rowCount, rows);
    assertEquals(beforeSize, cfs.getLiveSSTables().iterator().next().onDiskLength());
    validateData(cfs, rowCount);
    cfs.truncateBlocking();
}