Examples with SSTableRewriter org.apache.cassandra.io.sstable.SSTableRewriter used on opensource projects

Example 1 with SSTableRewriter

use of org.apache.cassandra.io.sstable.SSTableRewriter in project cassandra by apache.

the class CompactionManager method antiCompactGroup.

private int antiCompactGroup(ColumnFamilyStore cfs, Collection<Range<Token>> ranges, LifecycleTransaction anticompactionGroup, long repairedAt, UUID pendingRepair) {
    long groupMaxDataAge = -1;
    for (Iterator<SSTableReader> i = anticompactionGroup.originals().iterator(); i.hasNext(); ) {
        SSTableReader sstable = i.next();
        if (groupMaxDataAge < sstable.maxDataAge)
            groupMaxDataAge = sstable.maxDataAge;
    }
    if (anticompactionGroup.originals().size() == 0) {
        logger.info("No valid anticompactions for this group, All sstables were compacted and are no longer available");
        return 0;
    }
    logger.info("Anticompacting {}", anticompactionGroup);
    Set<SSTableReader> sstableAsSet = anticompactionGroup.originals();
    File destination = cfs.getDirectories().getWriteableLocationAsFile(cfs.getExpectedCompactedFileSize(sstableAsSet, OperationType.ANTICOMPACTION));
    long repairedKeyCount = 0;
    long unrepairedKeyCount = 0;
    int nowInSec = FBUtilities.nowInSeconds();
    CompactionStrategyManager strategy = cfs.getCompactionStrategyManager();
    try (SSTableRewriter repairedSSTableWriter = SSTableRewriter.constructWithoutEarlyOpening(anticompactionGroup, false, groupMaxDataAge);
        SSTableRewriter unRepairedSSTableWriter = SSTableRewriter.constructWithoutEarlyOpening(anticompactionGroup, false, groupMaxDataAge);
        AbstractCompactionStrategy.ScannerList scanners = strategy.getScanners(anticompactionGroup.originals());
        CompactionController controller = new CompactionController(cfs, sstableAsSet, getDefaultGcBefore(cfs, nowInSec));
        CompactionIterator ci = new CompactionIterator(OperationType.ANTICOMPACTION, scanners.scanners, controller, nowInSec, UUIDGen.getTimeUUID(), metrics)) {
        int expectedBloomFilterSize = Math.max(cfs.metadata().params.minIndexInterval, (int) (SSTableReader.getApproximateKeyCount(sstableAsSet)));
        repairedSSTableWriter.switchWriter(CompactionManager.createWriterForAntiCompaction(cfs, destination, expectedBloomFilterSize, repairedAt, pendingRepair, sstableAsSet, anticompactionGroup));
        unRepairedSSTableWriter.switchWriter(CompactionManager.createWriterForAntiCompaction(cfs, destination, expectedBloomFilterSize, ActiveRepairService.UNREPAIRED_SSTABLE, null, sstableAsSet, anticompactionGroup));
        Range.OrderedRangeContainmentChecker containmentChecker = new Range.OrderedRangeContainmentChecker(ranges);
        while (ci.hasNext()) {
            try (UnfilteredRowIterator partition = ci.next()) {
                // if current range from sstable is repaired, save it into the new repaired sstable
                if (containmentChecker.contains(partition.partitionKey().getToken())) {
                    repairedSSTableWriter.append(partition);
                    repairedKeyCount++;
                } else // otherwise save into the new 'non-repaired' table
                {
                    unRepairedSSTableWriter.append(partition);
                    unrepairedKeyCount++;
                }
            }
        }
        List<SSTableReader> anticompactedSSTables = new ArrayList<>();
        // since both writers are operating over the same Transaction, we cannot use the convenience Transactional.finish() method,
        // as on the second finish() we would prepareToCommit() on a Transaction that has already been committed, which is forbidden by the API
        // (since it indicates misuse). We call permitRedundantTransitions so that calls that transition to a state already occupied are permitted.
        anticompactionGroup.permitRedundantTransitions();
        repairedSSTableWriter.setRepairedAt(repairedAt).prepareToCommit();
        unRepairedSSTableWriter.prepareToCommit();
        anticompactedSSTables.addAll(repairedSSTableWriter.finished());
        anticompactedSSTables.addAll(unRepairedSSTableWriter.finished());
        repairedSSTableWriter.commit();
        unRepairedSSTableWriter.commit();
        logger.trace("Repaired {} keys out of {} for {}/{} in {}", repairedKeyCount, repairedKeyCount + unrepairedKeyCount, cfs.keyspace.getName(), cfs.getTableName(), anticompactionGroup);
        return anticompactedSSTables.size();
    } catch (Throwable e) {
        JVMStabilityInspector.inspectThrowable(e);
        logger.error("Error anticompacting " + anticompactionGroup, e);
    }
    return 0;
}

Example 2 with SSTableRewriter

use of org.apache.cassandra.io.sstable.SSTableRewriter in project cassandra by apache.

the class RealTransactionsTest method replaceSSTable.

private SSTableReader replaceSSTable(ColumnFamilyStore cfs, LifecycleTransaction txn, boolean fail) {
    List<SSTableReader> newsstables = null;
    int nowInSec = FBUtilities.nowInSeconds();
    try (CompactionController controller = new CompactionController(cfs, txn.originals(), cfs.gcBefore(FBUtilities.nowInSeconds()))) {
        try (SSTableRewriter rewriter = SSTableRewriter.constructKeepingOriginals(txn, false, 1000);
            AbstractCompactionStrategy.ScannerList scanners = cfs.getCompactionStrategyManager().getScanners(txn.originals());
            CompactionIterator ci = new CompactionIterator(txn.opType(), scanners.scanners, controller, nowInSec, txn.opId())) {
            long lastCheckObsoletion = System.nanoTime();
            File directory = txn.originals().iterator().next().descriptor.directory;
            Descriptor desc = cfs.newSSTableDescriptor(directory);
            TableMetadataRef metadata = Schema.instance.getTableMetadataRef(desc);
            rewriter.switchWriter(SSTableWriter.create(metadata, desc, 0, 0, null, 0, SerializationHeader.make(cfs.metadata(), txn.originals()), cfs.indexManager.listIndexes(), txn));
            while (ci.hasNext()) {
                rewriter.append(ci.next());
                if (System.nanoTime() - lastCheckObsoletion > TimeUnit.MINUTES.toNanos(1L)) {
                    controller.maybeRefreshOverlaps();
                    lastCheckObsoletion = System.nanoTime();
                }
            }
            if (!fail)
                newsstables = rewriter.finish();
            else
                rewriter.abort();
        }
    }
    assertTrue(fail || newsstables != null);
    if (newsstables != null) {
        Assert.assertEquals(1, newsstables.size());
        return newsstables.iterator().next();
    }
    return null;
}

Example 3 with SSTableRewriter

use of org.apache.cassandra.io.sstable.SSTableRewriter in project cassandra by apache.

the class CompactionManager method doCleanupOne.

/**
     * This function goes over a file and removes the keys that the node is not responsible for
     * and only keeps keys that this node is responsible for.
     *
     * @throws IOException
     */
private void doCleanupOne(final ColumnFamilyStore cfs, LifecycleTransaction txn, CleanupStrategy cleanupStrategy, Collection<Range<Token>> ranges, boolean hasIndexes) throws IOException {
    assert !cfs.isIndex();
    SSTableReader sstable = txn.onlyOne();
    if (!hasIndexes && !new Bounds<>(sstable.first.getToken(), sstable.last.getToken()).intersects(ranges)) {
        txn.obsoleteOriginals();
        txn.finish();
        return;
    }
    if (!needsCleanup(sstable, ranges)) {
        logger.trace("Skipping {} for cleanup; all rows should be kept", sstable);
        return;
    }
    long start = System.nanoTime();
    long totalkeysWritten = 0;
    long expectedBloomFilterSize = Math.max(cfs.metadata().params.minIndexInterval, SSTableReader.getApproximateKeyCount(txn.originals()));
    if (logger.isTraceEnabled())
        logger.trace("Expected bloom filter size : {}", expectedBloomFilterSize);
    logger.info("Cleaning up {}", sstable);
    File compactionFileLocation = sstable.descriptor.directory;
    RateLimiter limiter = getRateLimiter();
    double compressionRatio = sstable.getCompressionRatio();
    if (compressionRatio == MetadataCollector.NO_COMPRESSION_RATIO)
        compressionRatio = 1.0;
    List<SSTableReader> finished;
    int nowInSec = FBUtilities.nowInSeconds();
    try (SSTableRewriter writer = SSTableRewriter.construct(cfs, txn, false, sstable.maxDataAge);
        ISSTableScanner scanner = cleanupStrategy.getScanner(sstable);
        CompactionController controller = new CompactionController(cfs, txn.originals(), getDefaultGcBefore(cfs, nowInSec));
        CompactionIterator ci = new CompactionIterator(OperationType.CLEANUP, Collections.singletonList(scanner), controller, nowInSec, UUIDGen.getTimeUUID(), metrics)) {
        StatsMetadata metadata = sstable.getSSTableMetadata();
        writer.switchWriter(createWriter(cfs, compactionFileLocation, expectedBloomFilterSize, metadata.repairedAt, metadata.pendingRepair, sstable, txn));
        long lastBytesScanned = 0;
        while (ci.hasNext()) {
            if (ci.isStopRequested())
                throw new CompactionInterruptedException(ci.getCompactionInfo());
            try (UnfilteredRowIterator partition = ci.next();
                UnfilteredRowIterator notCleaned = cleanupStrategy.cleanup(partition)) {
                if (notCleaned == null)
                    continue;
                if (writer.append(notCleaned) != null)
                    totalkeysWritten++;
                long bytesScanned = scanner.getBytesScanned();
                compactionRateLimiterAcquire(limiter, bytesScanned, lastBytesScanned, compressionRatio);
                lastBytesScanned = bytesScanned;
            }
        }
        // flush to ensure we don't lose the tombstones on a restart, since they are not commitlog'd
        cfs.indexManager.flushAllIndexesBlocking();
        finished = writer.finish();
    }
    if (!finished.isEmpty()) {
        String format = "Cleaned up to %s.  %s to %s (~%d%% of original) for %,d keys.  Time: %,dms.";
        long dTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
        long startsize = sstable.onDiskLength();
        long endsize = 0;
        for (SSTableReader newSstable : finished) endsize += newSstable.onDiskLength();
        double ratio = (double) endsize / (double) startsize;
        logger.info(String.format(format, finished.get(0).getFilename(), FBUtilities.prettyPrintMemory(startsize), FBUtilities.prettyPrintMemory(endsize), (int) (ratio * 100), totalkeysWritten, dTime));
    }
}