Examples with SSTableReader org.apache.cassandra.io.sstable.SSTableReader used on opensource projects

Example 21 with SSTableReader

use of org.apache.cassandra.io.sstable.SSTableReader in project brisk by riptano.

the class BriskServer method getLocalSubBlock.

/**
 * Retrieves a local subBlock
 *
 * @param blockId row key
 * @param sblockId SubBlock column name
 * @param offset inside the sblock
 * @return a local sublock
 * @throws TException
 */
private LocalBlock getLocalSubBlock(String subBlockCFName, ByteBuffer blockId, ByteBuffer sblockId, int offset) throws TException {
    DecoratedKey<Token<?>> decoratedKey = new DecoratedKey<Token<?>>(StorageService.getPartitioner().getToken(blockId), blockId);
    Table table = Table.open(cfsKeyspace);
    ColumnFamilyStore sblockStore = table.getColumnFamilyStore(subBlockCFName);
    Collection<SSTableReader> sstables = sblockStore.getSSTables();
    for (SSTableReader sstable : sstables) {
        long position = sstable.getPosition(decoratedKey, Operator.EQ);
        if (position == -1)
            continue;
        String filename = sstable.descriptor.filenameFor(Component.DATA);
        RandomAccessFile raf = null;
        int mappedLength = -1;
        MappedByteBuffer mappedData = null;
        MappedFileDataInput file = null;
        try {
            raf = new RandomAccessFile(filename, "r");
            assert position < raf.length();
            mappedLength = (raf.length() - position) < Integer.MAX_VALUE ? (int) (raf.length() - position) : Integer.MAX_VALUE;
            mappedData = raf.getChannel().map(FileChannel.MapMode.READ_ONLY, position, mappedLength);
            file = new MappedFileDataInput(mappedData, filename, 0);
            if (file == null)
                continue;
            // Verify key was found in data file
            DecoratedKey keyInDisk = SSTableReader.decodeKey(sstable.partitioner, sstable.descriptor, ByteBufferUtil.readWithShortLength(file));
            assert keyInDisk.equals(decoratedKey) : String.format("%s != %s in %s", keyInDisk, decoratedKey, file.getPath());
            long rowSize = SSTableReader.readRowSize(file, sstable.descriptor);
            assert rowSize > 0;
            assert rowSize < mappedLength;
            Filter bf = IndexHelper.defreezeBloomFilter(file, sstable.descriptor.usesOldBloomFilter);
            // verify this column in in this version of the row.
            if (!bf.isPresent(sblockId))
                continue;
            List<IndexHelper.IndexInfo> indexList = IndexHelper.deserializeIndex(file);
            // we can stop early if bloom filter says none of the
            // columns actually exist -- but,
            // we can't stop before initializing the cf above, in
            // case there's a relevant tombstone
            ColumnFamilySerializer serializer = ColumnFamily.serializer();
            try {
                ColumnFamily cf = serializer.deserializeFromSSTableNoColumns(ColumnFamily.create(sstable.metadata), file);
                if (cf.isMarkedForDelete())
                    continue;
            } catch (Exception e) {
                e.printStackTrace();
                throw new IOException(serializer + " failed to deserialize " + sstable.getColumnFamilyName() + " with " + sstable.metadata + " from " + file, e);
            }
            Integer sblockLength = null;
            if (indexList == null)
                sblockLength = seekToSubColumn(sstable.metadata, file, sblockId);
            else
                sblockLength = seekToSubColumn(sstable.metadata, file, sblockId, indexList);
            if (sblockLength == null || sblockLength < 0)
                continue;
            int bytesReadFromStart = mappedLength - (int) file.bytesRemaining();
            if (logger.isDebugEnabled())
                logger.debug("BlockLength = " + sblockLength + " Availible " + file.bytesRemaining());
            assert offset <= sblockLength : String.format("%d > %d", offset, sblockLength);
            long dataOffset = position + bytesReadFromStart;
            if (file.bytesRemaining() == 0 || sblockLength == 0)
                continue;
            return new LocalBlock(file.getPath(), dataOffset + offset, sblockLength - offset);
        } catch (IOException e) {
            throw new TException(e);
        } finally {
            FileUtils.closeQuietly(raf);
        }
    }
    return null;
}

Example 22 with SSTableReader

use of org.apache.cassandra.io.sstable.SSTableReader in project eiger by wlloyd.

the class ColumnFamilyStoreTest method testSliceByNamesCommandOldMetatada.

@Test
public void testSliceByNamesCommandOldMetatada() throws Throwable {
    String tableName = "Keyspace1";
    String cfName = "Standard1";
    DecoratedKey key = Util.dk("slice-name-old-metadata");
    ByteBuffer cname = ByteBufferUtil.bytes("c1");
    Table table = Table.open(tableName);
    ColumnFamilyStore cfs = table.getColumnFamilyStore(cfName);
    cfs.clearUnsafe();
    // Create a column a 'high timestamp'
    putColsStandard(cfs, key, new Column(cname, ByteBufferUtil.bytes("a"), 2));
    cfs.forceBlockingFlush();
    // Nuke the metadata and reload that sstable
    Collection<SSTableReader> ssTables = cfs.getSSTables();
    assertEquals(1, ssTables.size());
    cfs.clearUnsafe();
    assertEquals(0, cfs.getSSTables().size());
    new File(ssTables.iterator().next().descriptor.filenameFor(SSTable.COMPONENT_STATS)).delete();
    cfs.loadNewSSTables();
    // Add another column with a lower timestamp
    putColsStandard(cfs, key, new Column(cname, ByteBufferUtil.bytes("b"), 1));
    // Test fetching the column by name returns the first column
    SliceByNamesReadCommand cmd = new SliceByNamesReadCommand(tableName, key.key, new QueryPath(cfName), Collections.singletonList(cname));
    ColumnFamily cf = cmd.getRow(table).cf;
    Column column = (Column) cf.getColumn(cname);
    assert column.value().equals(ByteBufferUtil.bytes("a")) : "expecting a, got " + ByteBufferUtil.string(column.value());
}

Example 23 with SSTableReader

use of org.apache.cassandra.io.sstable.SSTableReader in project eiger by wlloyd.

the class CompactionTask method execute.

/**
 * For internal use and testing only.  The rest of the system should go through the submit* methods,
 * which are properly serialized.
 * Caller is in charge of marking/unmarking the sstables as compacting.
 */
public int execute(CompactionExecutorStatsCollector collector) throws IOException {
    // it is not empty, it may compact down to nothing if all rows are deleted.
    assert sstables != null;
    Set<SSTableReader> toCompact = new HashSet<SSTableReader>(sstables);
    if (!isCompactionInteresting(toCompact))
        return 0;
    // If use defined, we don't want to "trust" our space estimation. If
    // there isn't enough room, it's the user problem
    long expectedSize = isUserDefined ? 0 : cfs.getExpectedCompactedFileSize(toCompact);
    File compactionFileLocation = cfs.directories.getDirectoryForNewSSTables(expectedSize);
    if (partialCompactionsAcceptable()) {
        // Try again w/o the largest one.
        if (compactionFileLocation == null) {
            while (compactionFileLocation == null && toCompact.size() > 1) {
                logger.warn("insufficient space to compact all requested files " + StringUtils.join(toCompact, ", "));
                // Note that we have removed files that are still marked as compacting. This suboptimal but ok since the caller will unmark all
                // the sstables at the end.
                toCompact.remove(cfs.getMaxSizeFile(toCompact));
                compactionFileLocation = cfs.directories.getDirectoryForNewSSTables(cfs.getExpectedCompactedFileSize(toCompact));
            }
        }
    }
    if (compactionFileLocation == null) {
        logger.warn("insufficient space to compact even the two smallest files, aborting");
        return 0;
    }
    if (DatabaseDescriptor.isSnapshotBeforeCompaction())
        cfs.table.snapshot(System.currentTimeMillis() + "-" + "compact-" + cfs.columnFamily);
    // sanity check: all sstables must belong to the same cfs
    for (SSTableReader sstable : toCompact) assert sstable.descriptor.cfname.equals(cfs.columnFamily);
    CompactionController controller = new CompactionController(cfs, toCompact, gcBefore, isUserDefined);
    // new sstables from flush can be added during a compaction, but only the compaction can remove them,
    // so in our single-threaded compaction world this is a valid way of determining if we're compacting
    // all the sstables (that existed when we started)
    logger.info("Compacting {}", toCompact);
    long startTime = System.currentTimeMillis();
    long totalkeysWritten = 0;
    long estimatedTotalKeys = Math.max(DatabaseDescriptor.getIndexInterval(), SSTableReader.getApproximateKeyCount(toCompact));
    long estimatedSSTables = Math.max(1, SSTable.getTotalBytes(toCompact) / cfs.getCompactionStrategy().getMaxSSTableSize());
    long keysPerSSTable = (long) Math.ceil((double) estimatedTotalKeys / estimatedSSTables);
    if (logger.isDebugEnabled())
        logger.debug("Expected bloom filter size : " + keysPerSSTable);
    AbstractCompactionIterable ci = DatabaseDescriptor.isMultithreadedCompaction() ? new ParallelCompactionIterable(OperationType.COMPACTION, toCompact, controller) : new CompactionIterable(OperationType.COMPACTION, toCompact, controller);
    CloseableIterator<AbstractCompactedRow> iter = ci.iterator();
    Iterator<AbstractCompactedRow> nni = Iterators.filter(iter, Predicates.notNull());
    Map<DecoratedKey, Long> cachedKeys = new HashMap<DecoratedKey, Long>();
    // we can't preheat until the tracker has been set. This doesn't happen until we tell the cfs to
    // replace the old entries.  Track entries to preheat here until then.
    Map<SSTableReader, Map<DecoratedKey, Long>> cachedKeyMap = new HashMap<SSTableReader, Map<DecoratedKey, Long>>();
    Collection<SSTableReader> sstables = new ArrayList<SSTableReader>();
    Collection<SSTableWriter> writers = new ArrayList<SSTableWriter>();
    if (collector != null)
        collector.beginCompaction(ci);
    try {
        if (!nni.hasNext()) {
            // don't mark compacted in the finally block, since if there _is_ nondeleted data,
            // we need to sync it (via closeAndOpen) first, so there is no period during which
            // a crash could cause data loss.
            cfs.markCompacted(toCompact);
            return 0;
        }
        SSTableWriter writer = cfs.createCompactionWriter(keysPerSSTable, compactionFileLocation, toCompact);
        writers.add(writer);
        while (nni.hasNext()) {
            if (ci.isStopped())
                throw new CompactionInterruptedException(ci.getCompactionInfo());
            AbstractCompactedRow row = nni.next();
            if (row.isEmpty())
                continue;
            long position = writer.append(row);
            totalkeysWritten++;
            if (DatabaseDescriptor.getPreheatKeyCache()) {
                for (SSTableReader sstable : toCompact) {
                    if (sstable.getCachedPosition(row.key, false) != null) {
                        cachedKeys.put(row.key, position);
                        break;
                    }
                }
            }
            if (!nni.hasNext() || newSSTableSegmentThresholdReached(writer, position)) {
                SSTableReader toIndex = writer.closeAndOpenReader(getMaxDataAge(toCompact));
                cachedKeyMap.put(toIndex, cachedKeys);
                sstables.add(toIndex);
                if (nni.hasNext()) {
                    writer = cfs.createCompactionWriter(keysPerSSTable, compactionFileLocation, toCompact);
                    writers.add(writer);
                    cachedKeys = new HashMap<DecoratedKey, Long>();
                }
            }
        }
    } catch (Exception e) {
        for (SSTableWriter writer : writers) writer.abort();
        throw FBUtilities.unchecked(e);
    } finally {
        iter.close();
        if (collector != null)
            collector.finishCompaction(ci);
    }
    cfs.replaceCompactedSSTables(toCompact, sstables);
    // TODO: this doesn't belong here, it should be part of the reader to load when the tracker is wired up
    for (Entry<SSTableReader, Map<DecoratedKey, Long>> ssTableReaderMapEntry : cachedKeyMap.entrySet()) {
        SSTableReader key = ssTableReaderMapEntry.getKey();
        for (Entry<DecoratedKey, Long> entry : ssTableReaderMapEntry.getValue().entrySet()) key.cacheKey(entry.getKey(), entry.getValue());
    }
    long dTime = System.currentTimeMillis() - startTime;
    long startsize = SSTable.getTotalBytes(toCompact);
    long endsize = SSTable.getTotalBytes(sstables);
    double ratio = (double) endsize / (double) startsize;
    StringBuilder builder = new StringBuilder();
    builder.append("[");
    for (SSTableReader reader : sstables) builder.append(reader.getFilename()).append(",");
    builder.append("]");
    double mbps = dTime > 0 ? (double) endsize / (1024 * 1024) / ((double) dTime / 1000) : 0;
    logger.info(String.format("Compacted to %s.  %,d to %,d (~%d%% of original) bytes for %,d keys at %fMB/s.  Time: %,dms.", builder.toString(), startsize, endsize, (int) (ratio * 100), totalkeysWritten, mbps, dTime));
    logger.debug(String.format("CF Total Bytes Compacted: %,d", CompactionTask.addToTotalBytesCompacted(endsize)));
    return toCompact.size();
}

Example 24 with SSTableReader

use of org.apache.cassandra.io.sstable.SSTableReader in project eiger by wlloyd.

the class LeveledManifest method load.

private static void load(ColumnFamilyStore cfs, LeveledManifest manifest) {
    File manifestFile = tryGetManifest(cfs);
    if (manifestFile == null)
        return;
    ObjectMapper m = new ObjectMapper();
    try {
        JsonNode rootNode = m.readValue(manifestFile, JsonNode.class);
        JsonNode generations = rootNode.get("generations");
        assert generations.isArray();
        for (JsonNode generation : generations) {
            int level = generation.get("generation").getIntValue();
            JsonNode generationValues = generation.get("members");
            for (JsonNode generationValue : generationValues) {
                for (SSTableReader ssTableReader : cfs.getSSTables()) {
                    if (ssTableReader.descriptor.generation == generationValue.getIntValue()) {
                        logger.debug("Loading {} at L{}", ssTableReader, level);
                        manifest.add(ssTableReader, level);
                    }
                }
            }
        }
    } catch (Exception e) {
        // TODO try to recover -old first
        logger.error("Manifest present but corrupt. Cassandra will compact levels from scratch", e);
    }
}

Example 25 with SSTableReader

use of org.apache.cassandra.io.sstable.SSTableReader in project eiger by wlloyd.

the class LeveledManifest method overlapping.

private static List<SSTableReader> overlapping(SSTableReader sstable, Iterable<SSTableReader> candidates) {
    List<SSTableReader> overlapped = new ArrayList<SSTableReader>();
    overlapped.add(sstable);
    Range<Token> promotedRange = new Range<Token>(sstable.first.token, sstable.last.token);
    for (SSTableReader candidate : candidates) {
        Range<Token> candidateRange = new Range<Token>(candidate.first.token, candidate.last.token);
        if (candidateRange.intersects(promotedRange))
            overlapped.add(candidate);
    }
    return overlapped;
}