Examples with ISSTableScanner org.apache.cassandra.io.sstable.ISSTableScanner used on opensource projects

Example 6 with ISSTableScanner

use of org.apache.cassandra.io.sstable.ISSTableScanner in project cassandra by apache.

the class TTLExpiryTest method testNoExpire.

@Test
public void testNoExpire() throws InterruptedException, IOException {
    ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore("Standard1");
    cfs.truncateBlocking();
    cfs.disableAutoCompaction();
    MigrationManager.announceTableUpdate(cfs.metadata().unbuild().gcGraceSeconds(0).build(), true);
    long timestamp = System.currentTimeMillis();
    String key = "ttl";
    new RowUpdateBuilder(cfs.metadata(), timestamp, 1, key).add("col", ByteBufferUtil.EMPTY_BYTE_BUFFER).add("col7", ByteBufferUtil.EMPTY_BYTE_BUFFER).build().applyUnsafe();
    cfs.forceBlockingFlush();
    new RowUpdateBuilder(cfs.metadata(), timestamp, 1, key).add("col2", ByteBufferUtil.EMPTY_BYTE_BUFFER).build().applyUnsafe();
    cfs.forceBlockingFlush();
    new RowUpdateBuilder(cfs.metadata(), timestamp, 1, key).add("col3", ByteBufferUtil.EMPTY_BYTE_BUFFER).build().applyUnsafe();
    cfs.forceBlockingFlush();
    String noTTLKey = "nottl";
    new RowUpdateBuilder(cfs.metadata(), timestamp, noTTLKey).add("col311", ByteBufferUtil.EMPTY_BYTE_BUFFER).build().applyUnsafe();
    cfs.forceBlockingFlush();
    // wait for ttl to expire
    Thread.sleep(2000);
    assertEquals(4, cfs.getLiveSSTables().size());
    cfs.enableAutoCompaction(true);
    assertEquals(1, cfs.getLiveSSTables().size());
    SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
    ISSTableScanner scanner = sstable.getScanner(ColumnFilter.all(cfs.metadata()), DataRange.allData(cfs.getPartitioner()));
    assertTrue(scanner.hasNext());
    while (scanner.hasNext()) {
        UnfilteredRowIterator iter = scanner.next();
        assertEquals(Util.dk(noTTLKey), iter.partitionKey());
    }
    scanner.close();
}

Example 7 with ISSTableScanner

use of org.apache.cassandra.io.sstable.ISSTableScanner in project cassandra by apache.

the class CompactionManager method doCleanupOne.

/**
     * This function goes over a file and removes the keys that the node is not responsible for
     * and only keeps keys that this node is responsible for.
     *
     * @throws IOException
     */
private void doCleanupOne(final ColumnFamilyStore cfs, LifecycleTransaction txn, CleanupStrategy cleanupStrategy, Collection<Range<Token>> ranges, boolean hasIndexes) throws IOException {
    assert !cfs.isIndex();
    SSTableReader sstable = txn.onlyOne();
    if (!hasIndexes && !new Bounds<>(sstable.first.getToken(), sstable.last.getToken()).intersects(ranges)) {
        txn.obsoleteOriginals();
        txn.finish();
        return;
    }
    if (!needsCleanup(sstable, ranges)) {
        logger.trace("Skipping {} for cleanup; all rows should be kept", sstable);
        return;
    }
    long start = System.nanoTime();
    long totalkeysWritten = 0;
    long expectedBloomFilterSize = Math.max(cfs.metadata().params.minIndexInterval, SSTableReader.getApproximateKeyCount(txn.originals()));
    if (logger.isTraceEnabled())
        logger.trace("Expected bloom filter size : {}", expectedBloomFilterSize);
    logger.info("Cleaning up {}", sstable);
    File compactionFileLocation = sstable.descriptor.directory;
    RateLimiter limiter = getRateLimiter();
    double compressionRatio = sstable.getCompressionRatio();
    if (compressionRatio == MetadataCollector.NO_COMPRESSION_RATIO)
        compressionRatio = 1.0;
    List<SSTableReader> finished;
    int nowInSec = FBUtilities.nowInSeconds();
    try (SSTableRewriter writer = SSTableRewriter.construct(cfs, txn, false, sstable.maxDataAge);
        ISSTableScanner scanner = cleanupStrategy.getScanner(sstable);
        CompactionController controller = new CompactionController(cfs, txn.originals(), getDefaultGcBefore(cfs, nowInSec));
        CompactionIterator ci = new CompactionIterator(OperationType.CLEANUP, Collections.singletonList(scanner), controller, nowInSec, UUIDGen.getTimeUUID(), metrics)) {
        StatsMetadata metadata = sstable.getSSTableMetadata();
        writer.switchWriter(createWriter(cfs, compactionFileLocation, expectedBloomFilterSize, metadata.repairedAt, metadata.pendingRepair, sstable, txn));
        long lastBytesScanned = 0;
        while (ci.hasNext()) {
            if (ci.isStopRequested())
                throw new CompactionInterruptedException(ci.getCompactionInfo());
            try (UnfilteredRowIterator partition = ci.next();
                UnfilteredRowIterator notCleaned = cleanupStrategy.cleanup(partition)) {
                if (notCleaned == null)
                    continue;
                if (writer.append(notCleaned) != null)
                    totalkeysWritten++;
                long bytesScanned = scanner.getBytesScanned();
                compactionRateLimiterAcquire(limiter, bytesScanned, lastBytesScanned, compressionRatio);
                lastBytesScanned = bytesScanned;
            }
        }
        // flush to ensure we don't lose the tombstones on a restart, since they are not commitlog'd
        cfs.indexManager.flushAllIndexesBlocking();
        finished = writer.finish();
    }
    if (!finished.isEmpty()) {
        String format = "Cleaned up to %s.  %s to %s (~%d%% of original) for %,d keys.  Time: %,dms.";
        long dTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
        long startsize = sstable.onDiskLength();
        long endsize = 0;
        for (SSTableReader newSstable : finished) endsize += newSstable.onDiskLength();
        double ratio = (double) endsize / (double) startsize;
        logger.info(String.format(format, finished.get(0).getFilename(), FBUtilities.prettyPrintMemory(startsize), FBUtilities.prettyPrintMemory(endsize), (int) (ratio * 100), totalkeysWritten, dTime));
    }
}

Example 8 with ISSTableScanner

use of org.apache.cassandra.io.sstable.ISSTableScanner in project cassandra by apache.

the class LeveledCompactionStrategy method getScanners.

public ScannerList getScanners(Collection<SSTableReader> sstables, Collection<Range<Token>> ranges) {
    Set<SSTableReader>[] sstablesPerLevel = manifest.getSStablesPerLevelSnapshot();
    Multimap<Integer, SSTableReader> byLevel = ArrayListMultimap.create();
    for (SSTableReader sstable : sstables) {
        int level = sstable.getSSTableLevel();
        // so we add it to level -1 and create exclusive scanners for it - see below (#9935)
        if (level >= sstablesPerLevel.length || !sstablesPerLevel[level].contains(sstable)) {
            logger.warn("Live sstable {} from level {} is not on corresponding level in the leveled manifest." + " This is not a problem per se, but may indicate an orphaned sstable due to a failed" + " compaction not cleaned up properly.", sstable.getFilename(), level);
            level = -1;
        }
        byLevel.get(level).add(sstable);
    }
    List<ISSTableScanner> scanners = new ArrayList<ISSTableScanner>(sstables.size());
    try {
        for (Integer level : byLevel.keySet()) {
            // since we don't know which level those sstable belong yet, we simply do the same as L0 sstables.
            if (level <= 0) {
                // L0 makes no guarantees about overlapping-ness.  Just create a direct scanner for each
                for (SSTableReader sstable : byLevel.get(level)) scanners.add(sstable.getScanner(ranges));
            } else {
                // Create a LeveledScanner that only opens one sstable at a time, in sorted order
                Collection<SSTableReader> intersecting = LeveledScanner.intersecting(byLevel.get(level), ranges);
                if (!intersecting.isEmpty()) {
                    // The ScannerList will be in charge of closing (and we close properly on errors)
                    @SuppressWarnings("resource") ISSTableScanner scanner = new LeveledScanner(cfs.metadata(), intersecting, ranges);
                    scanners.add(scanner);
                }
            }
        }
    } catch (Throwable t) {
        try {
            new ScannerList(scanners).close();
        } catch (Throwable t2) {
            t.addSuppressed(t2);
        }
        throw t;
    }
    return new ScannerList(scanners);
}

Example 9 with ISSTableScanner

use of org.apache.cassandra.io.sstable.ISSTableScanner in project cassandra by apache.

the class CompactionStrategyManager method getScanners.

/**
     * Create ISSTableScanners from the given sstables
     *
     * Delegates the call to the compaction strategies to allow LCS to create a scanner
     * @param sstables
     * @param ranges
     * @return
     */
@SuppressWarnings("resource")
public AbstractCompactionStrategy.ScannerList getScanners(Collection<SSTableReader> sstables, Collection<Range<Token>> ranges) {
    assert repaired.size() == unrepaired.size();
    assert repaired.size() == pendingRepairs.size();
    List<Set<SSTableReader>> pendingSSTables = new ArrayList<>();
    List<Set<SSTableReader>> repairedSSTables = new ArrayList<>();
    List<Set<SSTableReader>> unrepairedSSTables = new ArrayList<>();
    for (int i = 0; i < repaired.size(); i++) {
        pendingSSTables.add(new HashSet<>());
        repairedSSTables.add(new HashSet<>());
        unrepairedSSTables.add(new HashSet<>());
    }
    for (SSTableReader sstable : sstables) {
        int idx = getCompactionStrategyIndex(cfs, getDirectories(), sstable);
        if (sstable.isPendingRepair())
            pendingSSTables.get(idx).add(sstable);
        else if (sstable.isRepaired())
            repairedSSTables.get(idx).add(sstable);
        else
            unrepairedSSTables.get(idx).add(sstable);
    }
    List<ISSTableScanner> scanners = new ArrayList<>(sstables.size());
    readLock.lock();
    try {
        for (int i = 0; i < pendingSSTables.size(); i++) {
            if (!pendingSSTables.get(i).isEmpty())
                scanners.addAll(pendingRepairs.get(i).getScanners(pendingSSTables.get(i), ranges));
        }
        for (int i = 0; i < repairedSSTables.size(); i++) {
            if (!repairedSSTables.get(i).isEmpty())
                scanners.addAll(repaired.get(i).getScanners(repairedSSTables.get(i), ranges).scanners);
        }
        for (int i = 0; i < unrepairedSSTables.size(); i++) {
            if (!unrepairedSSTables.get(i).isEmpty())
                scanners.addAll(unrepaired.get(i).getScanners(unrepairedSSTables.get(i), ranges).scanners);
        }
        return new AbstractCompactionStrategy.ScannerList(scanners);
    } finally {
        readLock.unlock();
    }
}

Example 10 with ISSTableScanner

use of org.apache.cassandra.io.sstable.ISSTableScanner in project cassandra by apache.

the class PendingRepairManager method getScanners.

@SuppressWarnings("resource")
synchronized Set<ISSTableScanner> getScanners(Collection<SSTableReader> sstables, Collection<Range<Token>> ranges) {
    if (sstables.isEmpty()) {
        return Collections.emptySet();
    }
    Map<UUID, Set<SSTableReader>> sessionSSTables = new HashMap<>();
    for (SSTableReader sstable : sstables) {
        UUID sessionID = sstable.getSSTableMetadata().pendingRepair;
        assert sessionID != null;
        sessionSSTables.computeIfAbsent(sessionID, k -> new HashSet<>()).add(sstable);
    }
    Set<ISSTableScanner> scanners = new HashSet<>(sessionSSTables.size());
    for (Map.Entry<UUID, Set<SSTableReader>> entry : sessionSSTables.entrySet()) {
        scanners.addAll(get(entry.getKey()).getScanners(entry.getValue(), ranges).scanners);
    }
    return scanners;
}