Examples with SSTableReader org.apache.cassandra.io.sstable.SSTableReader used on opensource projects

Example 36 with SSTableReader

use of org.apache.cassandra.io.sstable.SSTableReader in project eiger by wlloyd.

the class DataTracker method getBloomFilterFalseRatio.

public double getBloomFilterFalseRatio() {
    long falseCount = 0L;
    long trueCount = 0L;
    for (SSTableReader sstable : getSSTables()) {
        falseCount += sstable.getBloomFilterFalsePositiveCount();
        trueCount += sstable.getBloomFilterTruePositiveCount();
    }
    if (falseCount == 0L && trueCount == 0L)
        return 0d;
    return (double) falseCount / (trueCount + falseCount);
}

Example 37 with SSTableReader

use of org.apache.cassandra.io.sstable.SSTableReader in project eiger by wlloyd.

the class Memtable method writeSortedContents.

private SSTableReader writeSortedContents(ReplayPosition context) throws IOException {
    logger.info("Writing " + this);
    long keySize = 0;
    for (RowPosition key : columnFamilies.keySet()) {
        // make sure we don't write non-sensical keys
        assert key instanceof DecoratedKey;
        keySize += ((DecoratedKey) key).key.remaining();
    }
    long estimatedSize = (long) ((// index entries
    keySize + // keys in data file
    keySize + // data
    currentThroughput.get()) * // bloom filter and row index overhead
    1.2);
    SSTableReader ssTable;
    // errors when creating the writer that may leave empty temp files.
    SSTableWriter writer = cfs.createFlushWriter(columnFamilies.size(), estimatedSize, context);
    try {
        // since the memtable is being used for queries in the "pending flush" category)
        for (Map.Entry<RowPosition, ColumnFamily> entry : columnFamilies.entrySet()) {
            ColumnFamily cf = entry.getValue();
            if (cf.isMarkedForDelete()) {
                // Pedantically, you could purge column level tombstones that are past GcGRace when writing to the SSTable.
                // But it can result in unexpected behaviour where deletes never make it to disk,
                // as they are lost and so cannot override existing column values. So we only remove deleted columns if there
                // is a CF level tombstone to ensure the delete makes it into an SSTable.
                ColumnFamilyStore.removeDeletedColumnsOnly(cf, Integer.MIN_VALUE);
            }
            writer.append((DecoratedKey) entry.getKey(), cf);
        }
        ssTable = writer.closeAndOpenReader();
    } catch (Exception e) {
        writer.abort();
        throw FBUtilities.unchecked(e);
    }
    logger.info(String.format("Completed flushing %s (%d bytes)", ssTable.getFilename(), new File(ssTable.getFilename()).length()));
    return ssTable;
}

Example 38 with SSTableReader

use of org.apache.cassandra.io.sstable.SSTableReader in project eiger by wlloyd.

the class CollationController method collectTimeOrderedData.

/**
 * Collects data in order of recency, using the sstable maxtimestamp data.
 * Once we have data for all requests columns that is newer than the newest remaining maxtimestamp,
 * we stop.
 */
private ColumnFamily collectTimeOrderedData() {
    logger.debug("collectTimeOrderedData");
    ISortedColumns.Factory factory = mutableColumns ? AtomicSortedColumns.factory() : TreeMapBackedSortedColumns.factory();
    ColumnFamily container = ColumnFamily.create(cfs.metadata, factory, filter.filter.isReversed());
    List<IColumnIterator> iterators = new ArrayList<IColumnIterator>();
    ColumnFamilyStore.ViewFragment view = cfs.markReferenced(filter.key);
    try {
        for (Memtable memtable : view.memtables) {
            IColumnIterator iter = filter.getMemtableColumnIterator(memtable);
            if (iter != null) {
                iterators.add(iter);
                container.delete(iter.getColumnFamily());
                while (iter.hasNext()) container.addColumn(iter.next());
            }
        }
        // avoid changing the filter columns of the original filter
        // (reduceNameFilter removes columns that are known to be irrelevant)
        TreeSet<ByteBuffer> filterColumns = new TreeSet<ByteBuffer>(((NamesQueryFilter) filter.filter).columns);
        QueryFilter reducedFilter = new QueryFilter(filter.key, filter.path, new NamesQueryFilter(filterColumns));
        /* add the SSTables on disk */
        Collections.sort(view.sstables, SSTable.maxTimestampComparator);
        // read sorted sstables
        for (SSTableReader sstable : view.sstables) {
            long currentMaxTs = sstable.getMaxTimestamp();
            reduceNameFilter(reducedFilter, container, currentMaxTs);
            if (((NamesQueryFilter) reducedFilter.filter).columns.isEmpty())
                break;
            IColumnIterator iter = reducedFilter.getSSTableColumnIterator(sstable);
            iterators.add(iter);
            if (iter.getColumnFamily() != null) {
                container.delete(iter.getColumnFamily());
                sstablesIterated++;
                while (iter.hasNext()) container.addColumn(iter.next());
            }
        }
        // and "there used to be data, but it's gone now" (we should cache the empty CF so we don't need to rebuild that slower)
        if (iterators.isEmpty())
            return null;
        // do a final collate.  toCollate is boilerplate required to provide a CloseableIterator
        final ColumnFamily c2 = container;
        CloseableIterator<IColumn> toCollate = new SimpleAbstractColumnIterator() {

            final Iterator<IColumn> iter = c2.iterator();

            protected IColumn computeNext() {
                return iter.hasNext() ? iter.next() : endOfData();
            }

            public ColumnFamily getColumnFamily() {
                return c2;
            }

            public DecoratedKey getKey() {
                return filter.key;
            }
        };
        ColumnFamily returnCF = container.cloneMeShallow();
        filter.collateColumns(returnCF, Collections.singletonList(toCollate), gcBefore);
        // "hoist up" the requested data into a more recent sstable
        if (sstablesIterated > cfs.getMinimumCompactionThreshold() && !cfs.isCompactionDisabled() && cfs.getCompactionStrategy() instanceof SizeTieredCompactionStrategy) {
            RowMutation rm = new RowMutation(cfs.table.name, new Row(filter.key, returnCF.cloneMe()));
            try {
                // skipping commitlog and index updates is fine since we're just de-fragmenting existing data
                Table.open(rm.getTable()).apply(rm, false, false);
            } catch (IOException e) {
                // log and allow the result to be returned
                logger.error("Error re-writing read results", e);
            }
        }
        // Caller is responsible for final removeDeletedCF.  This is important for cacheRow to work correctly:
        return returnCF;
    } finally {
        for (IColumnIterator iter : iterators) FileUtils.closeQuietly(iter);
        SSTableReader.releaseReferences(view.sstables);
    }
}

Example 39 with SSTableReader

use of org.apache.cassandra.io.sstable.SSTableReader in project eiger by wlloyd.

the class StreamingTransferTest method testTransferTableMultiple.

@Test
public void testTransferTableMultiple() throws Exception {
    // write temporary SSTables, but don't register them
    Set<String> content = new HashSet<String>();
    content.add("test");
    content.add("test2");
    content.add("test3");
    SSTableReader sstable = SSTableUtils.prepare().write(content);
    String tablename = sstable.getTableName();
    String cfname = sstable.getColumnFamilyName();
    content = new HashSet<String>();
    content.add("transfer1");
    content.add("transfer2");
    content.add("transfer3");
    SSTableReader sstable2 = SSTableUtils.prepare().write(content);
    // transfer the first and last key
    IPartitioner p = StorageService.getPartitioner();
    List<Range<Token>> ranges = new ArrayList<Range<Token>>();
    ranges.add(new Range<Token>(p.getMinimumToken(), p.getToken(ByteBufferUtil.bytes("test"))));
    ranges.add(new Range<Token>(p.getToken(ByteBufferUtil.bytes("transfer2")), p.getMinimumToken()));
    // Acquiring references, transferSSTables needs it
    sstable.acquireReference();
    sstable2.acquireReference();
    StreamOutSession session = StreamOutSession.create(tablename, LOCAL, null);
    StreamOut.transferSSTables(session, Arrays.asList(sstable, sstable2), ranges, OperationType.BOOTSTRAP);
    session.await();
    // confirm that the sstables were transferred and registered and that 2 keys arrived
    ColumnFamilyStore cfstore = Table.open(tablename).getColumnFamilyStore(cfname);
    List<Row> rows = Util.getRangeSlice(cfstore);
    assertEquals(2, rows.size());
    assert rows.get(0).key.key.equals(ByteBufferUtil.bytes("test"));
    assert rows.get(1).key.key.equals(ByteBufferUtil.bytes("transfer3"));
    assert rows.get(0).cf.getColumnCount() == 1;
    assert rows.get(1).cf.getColumnCount() == 1;
    // these keys fall outside of the ranges and should not be transferred
    assert null == cfstore.getColumnFamily(QueryFilter.getIdentityFilter(Util.dk("transfer1"), new QueryPath("Standard1")));
    assert null == cfstore.getColumnFamily(QueryFilter.getIdentityFilter(Util.dk("transfer2"), new QueryPath("Standard1")));
    assert null == cfstore.getColumnFamily(QueryFilter.getIdentityFilter(Util.dk("test2"), new QueryPath("Standard1")));
    assert null == cfstore.getColumnFamily(QueryFilter.getIdentityFilter(Util.dk("test3"), new QueryPath("Standard1")));
}

Example 40 with SSTableReader

use of org.apache.cassandra.io.sstable.SSTableReader in project eiger by wlloyd.

the class StreamingTransferTest method testTransferOfMultipleColumnFamilies.

@Test
public void testTransferOfMultipleColumnFamilies() throws Exception {
    String keyspace = "KeyCacheSpace";
    IPartitioner p = StorageService.getPartitioner();
    String[] columnFamilies = new String[] { "Standard1", "Standard2", "Standard3" };
    List<SSTableReader> ssTableReaders = new ArrayList<SSTableReader>();
    NavigableMap<DecoratedKey, String> keys = new TreeMap<DecoratedKey, String>();
    for (String cf : columnFamilies) {
        Set<String> content = new HashSet<String>();
        content.add("data-" + cf + "-1");
        content.add("data-" + cf + "-2");
        content.add("data-" + cf + "-3");
        SSTableUtils.Context context = SSTableUtils.prepare().ks(keyspace).cf(cf);
        ssTableReaders.add(context.write(content));
        // collect dks for each string key
        for (String str : content) keys.put(Util.dk(str), cf);
    }
    // transfer the first and last keys
    Map.Entry<DecoratedKey, String> first = keys.firstEntry();
    Map.Entry<DecoratedKey, String> last = keys.lastEntry();
    Map.Entry<DecoratedKey, String> secondtolast = keys.lowerEntry(last.getKey());
    List<Range<Token>> ranges = new ArrayList<Range<Token>>();
    ranges.add(new Range<Token>(p.getMinimumToken(), first.getKey().token));
    // the left hand side of the range is exclusive, so we transfer from the second-to-last token
    ranges.add(new Range<Token>(secondtolast.getKey().token, p.getMinimumToken()));
    // Acquiring references, transferSSTables needs it
    if (!SSTableReader.acquireReferences(ssTableReaders))
        throw new AssertionError();
    StreamOutSession session = StreamOutSession.create(keyspace, LOCAL, null);
    StreamOut.transferSSTables(session, ssTableReaders, ranges, OperationType.BOOTSTRAP);
    session.await();
    // check that only two keys were transferred
    for (Map.Entry<DecoratedKey, String> entry : Arrays.asList(first, last)) {
        ColumnFamilyStore store = Table.open(keyspace).getColumnFamilyStore(entry.getValue());
        List<Row> rows = Util.getRangeSlice(store);
        assertEquals(rows.toString(), 1, rows.size());
        assertEquals(entry.getKey(), rows.get(0).key);
    }
}